From 04cb19df889963f2998f8ebcb26b4b634a7bc1f9 Mon Sep 17 00:00:00 2001 From: Sunny Date: Sun, 17 Mar 2019 19:38:35 +0530 Subject: [PATCH 1/3] cluster: fix cluster status update Combine current cluster and deployment in to a single data type StorageOSCluster so that the deployment always points to the right cluster. Also, adds status members checks in e2e test and enables go race detector for the unit tests. --- Makefile | 2 +- pkg/controller/storageoscluster/cluster.go | 56 ++++++++++++++++++ .../storageoscluster_controller.go | 59 ++++++------------- pkg/storageos/deploy.go | 27 --------- pkg/storageos/deployment.go | 32 ++++++++++ test/e2e/util/cluster.go | 8 +++ 6 files changed, 114 insertions(+), 70 deletions(-) create mode 100644 pkg/controller/storageoscluster/cluster.go create mode 100644 pkg/storageos/deployment.go diff --git a/Makefile b/Makefile index f376d55ef..0ab6bd9ba 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ lint: go vet ./... unittest: - go test -v `go list -v ./... | grep -v test/e2e` + go test -v -race `go list -v ./... | grep -v test/e2e | grep -v test/olm` operator-sdk: # Download sdk only if it's not available. diff --git a/pkg/controller/storageoscluster/cluster.go b/pkg/controller/storageoscluster/cluster.go new file mode 100644 index 000000000..ed4941c91 --- /dev/null +++ b/pkg/controller/storageoscluster/cluster.go @@ -0,0 +1,56 @@ +package storageoscluster + +import ( + storageosv1alpha1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1alpha1" + "github.com/storageos/cluster-operator/pkg/storageos" +) + +// StorageOSCluster stores the current cluster's information. It binds the +// cluster and the deployment together, ensuring deployment interacts with the +// right cluster resource. +type StorageOSCluster struct { + cluster *storageosv1alpha1.StorageOSCluster + // deployment is the storageos.Deployment object. This is cached for a + // cluster to avoid recreating it without any change to the cluster object. + // Every new cluster will create their unique deployment. + deployment *storageos.Deployment +} + +// NewStorageOSCluster creates a new StorageOSCluster object. +func NewStorageOSCluster(cluster *storageosv1alpha1.StorageOSCluster) *StorageOSCluster { + return &StorageOSCluster{cluster: cluster} +} + +// SetDeployment creates a new StorageOS Deployment and sets it for the current +// StorageOSCluster. +func (c *StorageOSCluster) SetDeployment(r *ReconcileStorageOSCluster) { + // updateIfExists is set to false because we don't update any existing + // resources for now. May change in future. + // TODO: Change this after resolving the conflict between two way + // binding and upgrade. + updateIfExists := false + c.deployment = storageos.NewDeployment(r.client, c.cluster, r.recorder, r.scheme, r.k8sVersion, updateIfExists) +} + +// IsCurrentCluster compares the cluster attributes to check if the given +// cluster is the same as the current cluster. +func (c *StorageOSCluster) IsCurrentCluster(cluster *storageosv1alpha1.StorageOSCluster) bool { + if (c.cluster.GetName() == cluster.GetName()) && + (c.cluster.GetNamespace() == cluster.GetNamespace()) { + return true + } + return false +} + +// Deploy deploys the StorageOS cluster. +func (c *StorageOSCluster) Deploy(r *ReconcileStorageOSCluster) error { + if c.deployment == nil { + c.SetDeployment(r) + } + return c.deployment.Deploy() +} + +// DeleteDeployment deletes the StorageOS Cluster deployment. +func (c *StorageOSCluster) DeleteDeployment() error { + return c.deployment.Delete() +} diff --git a/pkg/controller/storageoscluster/storageoscluster_controller.go b/pkg/controller/storageoscluster/storageoscluster_controller.go index 9072658ad..c2925fc38 100644 --- a/pkg/controller/storageoscluster/storageoscluster_controller.go +++ b/pkg/controller/storageoscluster/storageoscluster_controller.go @@ -72,10 +72,9 @@ type ReconcileStorageOSCluster struct { // that reads objects from the cache and writes to the apiserver client client.Client scheme *runtime.Scheme - currentCluster *storageosv1alpha1.StorageOSCluster k8sVersion string recorder record.EventRecorder - deployment *storageos.Deployment + currentCluster *StorageOSCluster } // SetCurrentClusterIfNone checks if there's any existing current cluster and @@ -88,28 +87,12 @@ func (r *ReconcileStorageOSCluster) SetCurrentClusterIfNone(cluster *storageosv1 // SetCurrentCluster sets the currently active cluster in the controller. func (r *ReconcileStorageOSCluster) SetCurrentCluster(cluster *storageosv1alpha1.StorageOSCluster) { - r.currentCluster = cluster -} - -// IsCurrentCluster compares a given cluster with the current cluster to check -// if they are the same. -func (r *ReconcileStorageOSCluster) IsCurrentCluster(cluster *storageosv1alpha1.StorageOSCluster) bool { - if cluster == nil { - return false - } - - if (r.currentCluster.GetName() == cluster.GetName()) && (r.currentCluster.GetNamespace() == cluster.GetNamespace()) { - return true - } - return false + r.currentCluster = NewStorageOSCluster(cluster) } // ResetCurrentCluster resets the current cluster of the controller. func (r *ReconcileStorageOSCluster) ResetCurrentCluster() { r.currentCluster = nil - // Reset deployment as well. Deployments are specific to the current - // cluster. - r.deployment = nil } // Reconcile reads that state of the cluster for a StorageOSCluster object and makes changes based on the state read @@ -127,9 +110,10 @@ func (r *ReconcileStorageOSCluster) Reconcile(request reconcile.Request) (reconc err := r.client.Get(context.TODO(), request.NamespacedName, instance) if err != nil { if errors.IsNotFound(err) { - // Cluster instance not found. Delete the resources and reset the current cluster. + // Cluster instance not found. Delete the resources and reset the + // current cluster. if r.currentCluster != nil { - if err := getDeployment(r).Delete(); err != nil { + if err := r.currentCluster.DeleteDeployment(); err != nil { // Error deleting - requeue the request. return reconcileResult, err } @@ -149,10 +133,18 @@ func (r *ReconcileStorageOSCluster) Reconcile(request reconcile.Request) (reconc // If the event doesn't belongs to the current cluster, do not reconcile. // There must be only a single instance of storageos in a cluster. - if !r.IsCurrentCluster(instance) { + if !r.currentCluster.IsCurrentCluster(instance) { err := fmt.Errorf("can't create more than one storageos cluster") r.recorder.Event(instance, corev1.EventTypeWarning, "FailedCreation", err.Error()) return reconcileResult, err + } else if r.currentCluster.cluster.GetUID() != instance.GetUID() { + // If the cluster name and namespace match with the current cluster, but + // the resource UIDs are different, maybe the current cluster reset + // failed when the previous cluster was deleted. The same cluster was + // created again and has a different UID. Create and assign a new + // current cluster. + log.Printf("replacing current cluster UID: %s with new cluster UID: %s", r.currentCluster.cluster.GetUID(), instance.GetUID()) + r.SetCurrentCluster(instance) } if err := r.reconcile(instance); err != nil { @@ -163,8 +155,8 @@ func (r *ReconcileStorageOSCluster) Reconcile(request reconcile.Request) (reconc } func (r *ReconcileStorageOSCluster) reconcile(m *storageosv1alpha1.StorageOSCluster) error { - // Do not reconcile, the operator is paused for the cluster. if m.Spec.Pause { + // Do not reconcile, the operator is paused for the cluster. return nil } @@ -226,7 +218,7 @@ func (r *ReconcileStorageOSCluster) reconcile(m *storageosv1alpha1.StorageOSClus // r.SetCurrentCluster(m) // } - if err := getDeployment(r).Deploy(); err != nil { + if err := r.currentCluster.Deploy(r); err != nil { // Ignore "Operation cannot be fulfilled" error. It happens when the // actual state of object is different from what is known to the operator. // Operator would resync and retry the failed operation on its own. @@ -240,7 +232,7 @@ func (r *ReconcileStorageOSCluster) reconcile(m *storageosv1alpha1.StorageOSClus // resource. r.recorder.Event(m, corev1.EventTypeNormal, "Terminating", "Deleting all the resources...") - if err := getDeployment(r).Delete(); err != nil { + if err := r.currentCluster.DeleteDeployment(); err != nil { return err } @@ -338,20 +330,3 @@ func getMatchingTolerations(taints []corev1.Taint, tolerations []corev1.Tolerati } return true, result } - -// getDeployment returns an existing storageos deployment if found, else creates -// a new deployment object, stores in the reconcile object and returns the same. -func getDeployment(r *ReconcileStorageOSCluster) *storageos.Deployment { - if r.deployment != nil { - return r.deployment - } - - // update is set to false because we don't update any existing resources - // for now. May change in future. - // TODO: Change this after resolving the conflict between two way - // binding and upgrade. - updateIfExists := false - - r.deployment = storageos.NewDeployment(r.client, r.currentCluster, r.recorder, r.scheme, r.k8sVersion, updateIfExists) - return r.deployment -} diff --git a/pkg/storageos/deploy.go b/pkg/storageos/deploy.go index 40bb0b6c5..0ce1f972a 100644 --- a/pkg/storageos/deploy.go +++ b/pkg/storageos/deploy.go @@ -18,9 +18,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/tools/record" - - "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -110,30 +107,6 @@ const ( defaultPassword = "storageos" ) -// Deployment stores all the resource configuration and performs -// resource creation and update. -type Deployment struct { - client client.Client - stos *api.StorageOSCluster - recorder record.EventRecorder - k8sVersion string - scheme *runtime.Scheme - update bool -} - -// NewDeployment creates a new Deployment given a k8c client, storageos manifest -// and an event broadcast recorder. -func NewDeployment(client client.Client, stos *api.StorageOSCluster, recorder record.EventRecorder, scheme *runtime.Scheme, version string, update bool) *Deployment { - return &Deployment{ - client: client, - stos: stos, - recorder: recorder, - k8sVersion: version, - scheme: scheme, - update: update, - } -} - // Deploy deploys storageos by creating all the resources needed to run storageos. func (s *Deployment) Deploy() error { if err := s.createNamespace(); err != nil { diff --git a/pkg/storageos/deployment.go b/pkg/storageos/deployment.go new file mode 100644 index 000000000..398bd9118 --- /dev/null +++ b/pkg/storageos/deployment.go @@ -0,0 +1,32 @@ +package storageos + +import ( + api "github.com/storageos/cluster-operator/pkg/apis/storageos/v1alpha1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Deployment stores all the resource configuration and performs +// resource creation and update. +type Deployment struct { + client client.Client + stos *api.StorageOSCluster + recorder record.EventRecorder + k8sVersion string + scheme *runtime.Scheme + update bool +} + +// NewDeployment creates a new Deployment given a k8c client, storageos manifest +// and an event broadcast recorder. +func NewDeployment(client client.Client, stos *api.StorageOSCluster, recorder record.EventRecorder, scheme *runtime.Scheme, version string, update bool) *Deployment { + return &Deployment{ + client: client, + stos: stos, + recorder: recorder, + k8sVersion: version, + scheme: scheme, + update: update, + } +} diff --git a/test/e2e/util/cluster.go b/test/e2e/util/cluster.go index 348f3b942..f7dbcd609 100644 --- a/test/e2e/util/cluster.go +++ b/test/e2e/util/cluster.go @@ -92,6 +92,14 @@ func ClusterStatusCheck(t *testing.T, status storageos.StorageOSClusterStatus, n if status.Ready != wantReady { t.Errorf("unexpected Ready:\n\t(GOT) %s\n\t(WNT) %s", status.Ready, wantReady) } + + if len(status.Members.Ready) != nodes { + t.Errorf("unexpected number of ready members:\n\t(GOT) %d\n\t(WNT) %d", len(status.Members.Ready), nodes) + } + + if len(status.Members.Unready) != 0 { + t.Errorf("unexpected number of unready members:\n\t(GOT) %d\n\t(WNT) %d", len(status.Members.Unready), 0) + } } // DeployCluster creates a custom resource and checks if the From 7228c6baf96270e0c69c4066927798f3dbcd400b Mon Sep 17 00:00:00 2001 From: Sunny Date: Mon, 18 Mar 2019 14:24:39 +0530 Subject: [PATCH 2/3] pkg/storageos: separate into multiple files --- pkg/storageos/daemonset.go | 257 ++++++ pkg/storageos/delete.go | 85 ++ pkg/storageos/deploy.go | 1534 +-------------------------------- pkg/storageos/ingress.go | 62 ++ pkg/storageos/podspec.go | 278 ++++++ pkg/storageos/rbac.go | 321 +++++++ pkg/storageos/secret.go | 225 +++++ pkg/storageos/service.go | 97 +++ pkg/storageos/statefulset.go | 164 ++++ pkg/storageos/storageclass.go | 81 ++ 10 files changed, 1579 insertions(+), 1525 deletions(-) create mode 100644 pkg/storageos/daemonset.go create mode 100644 pkg/storageos/delete.go create mode 100644 pkg/storageos/ingress.go create mode 100644 pkg/storageos/podspec.go create mode 100644 pkg/storageos/rbac.go create mode 100644 pkg/storageos/secret.go create mode 100644 pkg/storageos/service.go create mode 100644 pkg/storageos/statefulset.go create mode 100644 pkg/storageos/storageclass.go diff --git a/pkg/storageos/daemonset.go b/pkg/storageos/daemonset.go new file mode 100644 index 000000000..633724988 --- /dev/null +++ b/pkg/storageos/daemonset.go @@ -0,0 +1,257 @@ +package storageos + +import ( + "strconv" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func (s *Deployment) createDaemonSet() error { + ls := labelsForDaemonSet(s.stos.Name) + privileged := true + mountPropagationBidirectional := corev1.MountPropagationBidirectional + allowPrivilegeEscalation := true + + dset := &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: daemonsetName, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": "storageos", + }, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: ls, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: ls, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "storageos-daemonset-sa", + HostPID: true, + HostNetwork: true, + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + InitContainers: []corev1.Container{ + { + Name: "enable-lio", + Image: s.stos.Spec.GetInitContainerImage(), + VolumeMounts: []corev1.VolumeMount{ + { + Name: "kernel-modules", + MountPath: "/lib/modules", + ReadOnly: true, + }, + { + Name: "sys", + MountPath: "/sys", + MountPropagation: &mountPropagationBidirectional, + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"SYS_ADMIN"}, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Image: s.stos.Spec.GetNodeContainerImage(), + Name: "storageos", + Args: []string{"server"}, + Ports: []corev1.ContainerPort{{ + ContainerPort: 5705, + Name: "api", + }}, + LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: int32(65), + TimeoutSeconds: int32(10), + FailureThreshold: int32(5), + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/health", + Port: intstr.IntOrString{Type: intstr.String, StrVal: "api"}, + }, + }, + }, + ReadinessProbe: &corev1.Probe{ + InitialDelaySeconds: int32(65), + TimeoutSeconds: int32(10), + FailureThreshold: int32(5), + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/health", + Port: intstr.IntOrString{Type: intstr.String, StrVal: "api"}, + }, + }, + }, + Env: []corev1.EnvVar{ + { + Name: hostnameEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: adminUsernameEnvVar, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: initSecretName, + }, + Key: "username", + }, + }, + }, + { + Name: adminPasswordEnvVar, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: initSecretName, + }, + Key: "password", + }, + }, + }, + { + Name: joinEnvVar, + Value: s.stos.Spec.Join, + }, + { + Name: advertiseIPEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + Name: namespaceEnvVar, + Value: s.stos.Spec.GetResourceNS(), + }, + { + Name: disableTelemetryEnvVar, + Value: strconv.FormatBool(s.stos.Spec.DisableTelemetry), + }, + { + Name: csiVersionEnvVar, + Value: s.stos.Spec.GetCSIVersion(CSIV1Supported(s.k8sVersion)), + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{sysAdminCap}, + }, + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "fuse", + MountPath: "/dev/fuse", + }, + { + Name: "sys", + MountPath: "/sys", + }, + { + Name: "state", + MountPath: "/var/lib/storageos", + MountPropagation: &mountPropagationBidirectional, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "kernel-modules", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/lib/modules", + }, + }, + }, + { + Name: "fuse", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/fuse", + }, + }, + }, + { + Name: "sys", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/sys", + }, + }, + }, + { + Name: "state", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/lib/storageos", + }, + }, + }, + }, + }, + }, + }, + } + + podSpec := &dset.Spec.Template.Spec + nodeContainer := &podSpec.Containers[0] + + s.addNodeAffinity(podSpec) + + if err := s.addTolerations(podSpec); err != nil { + return err + } + + nodeContainer.Env = s.addKVBackendEnvVars(nodeContainer.Env) + + nodeContainer.Env = s.addDebugEnvVars(nodeContainer.Env) + + s.addNodeContainerResources(nodeContainer) + + s.addSharedDir(podSpec) + + s.addCSI(podSpec) + + return s.createOrUpdateObject(dset) +} + +func (s *Deployment) deleteDaemonSet(name string) error { + return s.deleteObject(s.getDaemonSet(name)) +} + +func (s *Deployment) getDaemonSet(name string) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": "storageos", + }, + }, + } +} diff --git a/pkg/storageos/delete.go b/pkg/storageos/delete.go new file mode 100644 index 000000000..5089643a5 --- /dev/null +++ b/pkg/storageos/delete.go @@ -0,0 +1,85 @@ +package storageos + +// Delete deletes all the storageos resources. +// This explicit delete is implemented instead of depending on the garbage +// collector because sometimes the garbage collector deletes the resources +// with owner reference as a CRD without the parent being deleted. This happens +// especially when a cluster reboots. Althrough the operator re-creates the +// resources, we want to avoid this behavior by implementing an explcit delete. +func (s *Deployment) Delete() error { + + if err := s.deleteStorageClass("fast"); err != nil { + return err + } + + if err := s.deleteService(s.stos.Spec.GetServiceName()); err != nil { + return err + } + + if err := s.deleteDaemonSet(daemonsetName); err != nil { + return err + } + + if err := s.deleteSecret(initSecretName); err != nil { + return err + } + + if err := s.deleteRoleBinding(keyManagementBindingName); err != nil { + return err + } + + if err := s.deleteRole(keyManagementRoleName); err != nil { + return err + } + + if err := s.deleteServiceAccount("storageos-daemonset-sa"); err != nil { + return err + } + + if s.stos.Spec.CSI.Enable { + if err := s.deleteStatefulSet(statefulsetName); err != nil { + return err + } + + if err := s.deleteClusterRoleBinding("csi-attacher-binding"); err != nil { + return err + } + + if err := s.deleteClusterRoleBinding("csi-provisioner-binding"); err != nil { + return err + } + + if err := s.deleteClusterRole("csi-attacher-role"); err != nil { + return err + } + + if err := s.deleteClusterRole("csi-provisioner-role"); err != nil { + return err + } + + if err := s.deleteServiceAccount("storageos-statefulset-sa"); err != nil { + return err + } + + if err := s.deleteClusterRoleBinding("k8s-driver-registrar-binding"); err != nil { + return err + } + + if err := s.deleteClusterRoleBinding("driver-registrar-binding"); err != nil { + return err + } + + if err := s.deleteClusterRole("driver-registrar-role"); err != nil { + return err + } + + if err := s.deleteCSISecrets(); err != nil { + return err + } + } + + // NOTE: Do not delete the namespace. Namespace can have some resources + // created by the control plane. They must not be deleted. + + return nil +} diff --git a/pkg/storageos/deploy.go b/pkg/storageos/deploy.go index 0ce1f972a..cfa752bb6 100644 --- a/pkg/storageos/deploy.go +++ b/pkg/storageos/deploy.go @@ -4,20 +4,13 @@ import ( "context" "fmt" "log" - "strconv" "github.com/blang/semver" api "github.com/storageos/cluster-operator/pkg/apis/storageos/v1alpha1" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/api/extensions/v1beta1" - rbacv1 "k8s.io/api/rbac/v1" - storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -203,90 +196,6 @@ func (s *Deployment) Deploy() error { return s.updateStorageOSStatus(status) } -// Delete deletes all the storageos resources. -// This explicit delete is implemented instead of depending on the garbage -// collector because sometimes the garbage collector deletes the resources -// with owner reference as a CRD without the parent being deleted. This happens -// especially when a cluster reboots. Althrough the operator re-creates the -// resources, we want to avoid this behavior by implementing an explcit delete. -func (s *Deployment) Delete() error { - - if err := s.deleteStorageClass("fast"); err != nil { - return err - } - - if err := s.deleteService(s.stos.Spec.GetServiceName()); err != nil { - return err - } - - if err := s.deleteDaemonSet(daemonsetName); err != nil { - return err - } - - if err := s.deleteSecret(initSecretName); err != nil { - return err - } - - if err := s.deleteRoleBinding(keyManagementBindingName); err != nil { - return err - } - - if err := s.deleteRole(keyManagementRoleName); err != nil { - return err - } - - if err := s.deleteServiceAccount("storageos-daemonset-sa"); err != nil { - return err - } - - if s.stos.Spec.CSI.Enable { - if err := s.deleteStatefulSet(statefulsetName); err != nil { - return err - } - - if err := s.deleteClusterRoleBinding("csi-attacher-binding"); err != nil { - return err - } - - if err := s.deleteClusterRoleBinding("csi-provisioner-binding"); err != nil { - return err - } - - if err := s.deleteClusterRole("csi-attacher-role"); err != nil { - return err - } - - if err := s.deleteClusterRole("csi-provisioner-role"); err != nil { - return err - } - - if err := s.deleteServiceAccount("storageos-statefulset-sa"); err != nil { - return err - } - - if err := s.deleteClusterRoleBinding("k8s-driver-registrar-binding"); err != nil { - return err - } - - if err := s.deleteClusterRoleBinding("driver-registrar-binding"); err != nil { - return err - } - - if err := s.deleteClusterRole("driver-registrar-role"); err != nil { - return err - } - - if err := s.deleteCSISecrets(); err != nil { - return err - } - } - - // NOTE: Do not delete the namespace. Namespace can have some resources - // created by the control plane. They must not be deleted. - - return nil -} - func (s *Deployment) createNamespace() error { ns := &corev1.Namespace{ TypeMeta: metav1.TypeMeta{ @@ -304,572 +213,6 @@ func (s *Deployment) createNamespace() error { return s.createOrUpdateObject(ns) } -func (s *Deployment) createServiceAccount(name string) error { - sa := s.getServiceAccount(name) - return s.createOrUpdateObject(sa) -} - -func (s *Deployment) deleteServiceAccount(name string) error { - return s.deleteObject(s.getServiceAccount(name)) -} - -// getServiceAccount creates a generic service account object with the given -// name and returns it. -func (s *Deployment) getServiceAccount(name string) *corev1.ServiceAccount { - return &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ServiceAccount", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createServiceAccountForDaemonSet() error { - return s.createServiceAccount("storageos-daemonset-sa") -} - -func (s *Deployment) createServiceAccountForStatefulSet() error { - return s.createServiceAccount("storageos-statefulset-sa") -} - -func (s *Deployment) createRoleForKeyMgmt() error { - role := s.getRole(keyManagementRoleName) - role.Rules = []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "create", "delete"}, - }, - } - - return s.createOrUpdateObject(role) -} - -func (s *Deployment) deleteRole(name string) error { - return s.deleteObject(s.getRole(keyManagementRoleName)) -} - -// getRole creates a generic role object with the given name and returns it. -func (s *Deployment) getRole(name string) *rbacv1.Role { - return &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "Role", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createClusterRole(name string, rules []rbacv1.PolicyRule) error { - role := s.getClusterRole(name) - role.Rules = rules - return s.createOrUpdateObject(role) -} - -func (s *Deployment) deleteClusterRole(name string) error { - return s.deleteObject(s.getClusterRole(name)) -} - -func (s *Deployment) getClusterRole(name string) *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "ClusterRole", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createClusterRoleForDriverRegistrar() error { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list", "watch", "create", "update", "patch"}, - }, - { - APIGroups: []string{"apiextensions.k8s.io"}, - Resources: []string{"customresourcedefinitions"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"csi.storage.k8s.io"}, - Resources: []string{"csidrivers"}, - Verbs: []string{"create"}, - }, - } - return s.createClusterRole("driver-registrar-role", rules) -} - -func (s *Deployment) createClusterRoleForProvisioner() error { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"persistentvolumes"}, - Verbs: []string{"list", "watch", "create", "delete"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"persistentvolumeclaims"}, - Verbs: []string{"get", "list", "watch", "update"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - Verbs: []string{"list", "watch", "get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list", "watch", "create", "update", "patch"}, - }, - } - return s.createClusterRole("csi-provisioner-role", rules) -} - -func (s *Deployment) createClusterRoleForAttacher() error { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"persistentvolumes"}, - Verbs: []string{"get", "list", "watch", "update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - Verbs: []string{"list", "watch", "get"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"volumeattachments"}, - Verbs: []string{"get", "list", "watch", "update"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"csinodeinfos"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list", "watch", "create", "update", "patch"}, - }, - } - return s.createClusterRole("csi-attacher-role", rules) -} - -func (s *Deployment) createRoleBindingForKeyMgmt() error { - roleBinding := s.getRoleBinding(keyManagementBindingName) - roleBinding.Subjects = []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "storageos-daemonset-sa", - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleBinding.RoleRef = rbacv1.RoleRef{ - Kind: "Role", - Name: keyManagementRoleName, - APIGroup: "rbac.authorization.k8s.io", - } - return s.createOrUpdateObject(roleBinding) -} - -func (s *Deployment) deleteRoleBinding(name string) error { - return s.deleteObject(s.getRoleBinding(name)) -} - -func (s *Deployment) getRoleBinding(name string) *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "RoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createClusterRoleBinding(name string, subjects []rbacv1.Subject, roleRef rbacv1.RoleRef) error { - roleBinding := s.getClusterRoleBinding(name) - roleBinding.Subjects = subjects - roleBinding.RoleRef = roleRef - return s.createOrUpdateObject(roleBinding) -} - -func (s *Deployment) deleteClusterRoleBinding(name string) error { - return s.deleteObject(s.getClusterRoleBinding(name)) -} - -func (s *Deployment) getClusterRoleBinding(name string) *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "ClusterRoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createClusterRoleBindingForDriverRegistrar() error { - subjects := []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "storageos-daemonset-sa", - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleRef := rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "driver-registrar-role", - APIGroup: "rbac.authorization.k8s.io", - } - return s.createClusterRoleBinding("driver-registrar-binding", subjects, roleRef) -} - -func (s *Deployment) createClusterRoleBindingForK8SDriverRegistrar() error { - subjects := []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "storageos-statefulset-sa", - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleRef := rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "driver-registrar-role", - APIGroup: "rbac.authorization.k8s.io", - } - return s.createClusterRoleBinding("k8s-driver-registrar-binding", subjects, roleRef) -} - -func (s *Deployment) createClusterRoleBindingForProvisioner() error { - subjects := []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "storageos-statefulset-sa", - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleRef := rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "csi-provisioner-role", - APIGroup: "rbac.authorization.k8s.io", - } - return s.createClusterRoleBinding("csi-provisioner-binding", subjects, roleRef) -} - -func (s *Deployment) createClusterRoleBindingForAttacher() error { - subjects := []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "storageos-statefulset-sa", - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleRef := rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "csi-attacher-role", - APIGroup: "rbac.authorization.k8s.io", - } - return s.createClusterRoleBinding("csi-attacher-binding", subjects, roleRef) -} - -func (s *Deployment) createDaemonSet() error { - ls := labelsForDaemonSet(s.stos.Name) - privileged := true - mountPropagationBidirectional := corev1.MountPropagationBidirectional - allowPrivilegeEscalation := true - - dset := &appsv1.DaemonSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: daemonsetName, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": "storageos", - }, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: ls, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: ls, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: "storageos-daemonset-sa", - HostPID: true, - HostNetwork: true, - DNSPolicy: corev1.DNSClusterFirstWithHostNet, - InitContainers: []corev1.Container{ - { - Name: "enable-lio", - Image: s.stos.Spec.GetInitContainerImage(), - VolumeMounts: []corev1.VolumeMount{ - { - Name: "kernel-modules", - MountPath: "/lib/modules", - ReadOnly: true, - }, - { - Name: "sys", - MountPath: "/sys", - MountPropagation: &mountPropagationBidirectional, - }, - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: &privileged, - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"SYS_ADMIN"}, - }, - }, - }, - }, - Containers: []corev1.Container{ - { - Image: s.stos.Spec.GetNodeContainerImage(), - Name: "storageos", - Args: []string{"server"}, - Ports: []corev1.ContainerPort{{ - ContainerPort: 5705, - Name: "api", - }}, - LivenessProbe: &corev1.Probe{ - InitialDelaySeconds: int32(65), - TimeoutSeconds: int32(10), - FailureThreshold: int32(5), - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/v1/health", - Port: intstr.IntOrString{Type: intstr.String, StrVal: "api"}, - }, - }, - }, - ReadinessProbe: &corev1.Probe{ - InitialDelaySeconds: int32(65), - TimeoutSeconds: int32(10), - FailureThreshold: int32(5), - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/v1/health", - Port: intstr.IntOrString{Type: intstr.String, StrVal: "api"}, - }, - }, - }, - Env: []corev1.EnvVar{ - { - Name: hostnameEnvVar, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - { - Name: adminUsernameEnvVar, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: initSecretName, - }, - Key: "username", - }, - }, - }, - { - Name: adminPasswordEnvVar, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: initSecretName, - }, - Key: "password", - }, - }, - }, - { - Name: joinEnvVar, - Value: s.stos.Spec.Join, - // ValueFrom: &corev1.EnvVarSource{ - // FieldRef: &corev1.ObjectFieldSelector{ - // FieldPath: "status.podIP", - // }, - // }, - }, - { - Name: advertiseIPEnvVar, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }, - { - Name: namespaceEnvVar, - Value: s.stos.Spec.GetResourceNS(), - }, - { - Name: disableTelemetryEnvVar, - Value: strconv.FormatBool(s.stos.Spec.DisableTelemetry), - }, - { - Name: csiVersionEnvVar, - Value: s.stos.Spec.GetCSIVersion(CSIV1Supported(s.k8sVersion)), - }, - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: &privileged, - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{sysAdminCap}, - }, - AllowPrivilegeEscalation: &allowPrivilegeEscalation, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "fuse", - MountPath: "/dev/fuse", - }, - { - Name: "sys", - MountPath: "/sys", - }, - { - Name: "state", - MountPath: "/var/lib/storageos", - MountPropagation: &mountPropagationBidirectional, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "kernel-modules", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/lib/modules", - }, - }, - }, - { - Name: "fuse", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/dev/fuse", - }, - }, - }, - { - Name: "sys", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/sys", - }, - }, - }, - { - Name: "state", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/storageos", - }, - }, - }, - }, - }, - }, - }, - } - - podSpec := &dset.Spec.Template.Spec - nodeContainer := &podSpec.Containers[0] - - s.addNodeAffinity(podSpec) - - if err := s.addTolerations(podSpec); err != nil { - return err - } - - nodeContainer.Env = s.addKVBackendEnvVars(nodeContainer.Env) - - nodeContainer.Env = s.addDebugEnvVars(nodeContainer.Env) - - s.addNodeContainerResources(nodeContainer) - - s.addSharedDir(podSpec) - - s.addCSI(podSpec) - - return s.createOrUpdateObject(dset) -} - -func (s *Deployment) deleteDaemonSet(name string) error { - return s.deleteObject(s.getDaemonSet(name)) -} - -func (s *Deployment) getDaemonSet(name string) *appsv1.DaemonSet { - return &appsv1.DaemonSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": "storageos", - }, - }, - } -} - // addNodeContainerResources adds resource requirements for the node containers. func (s *Deployment) addNodeContainerResources(nodeContainer *corev1.Container) { if s.stos.Spec.Resources.Limits != nil || @@ -951,286 +294,15 @@ func (s *Deployment) addDebugEnvVars(env []corev1.EnvVar) []corev1.EnvVar { return env } -// addSharedDir adds env var and volumes for shared dir when running kubelet in -// a container. -func (s *Deployment) addSharedDir(podSpec *corev1.PodSpec) { - mountPropagationBidirectional := corev1.MountPropagationBidirectional - nodeContainer := &podSpec.Containers[0] - - // If kubelet is running in a container, sharedDir should be set. - if s.stos.Spec.SharedDir != "" { - envVar := corev1.EnvVar{ - Name: deviceDirEnvVar, - Value: fmt.Sprintf("%s/devices", s.stos.Spec.SharedDir), - } - nodeContainer.Env = append(nodeContainer.Env, envVar) - - sharedDir := corev1.Volume{ - Name: "shared", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.SharedDir, - }, - }, - } - podSpec.Volumes = append(podSpec.Volumes, sharedDir) - - volMnt := corev1.VolumeMount{ - Name: "shared", - MountPath: s.stos.Spec.SharedDir, - MountPropagation: &mountPropagationBidirectional, - } - nodeContainer.VolumeMounts = append(nodeContainer.VolumeMounts, volMnt) - } -} - -// addCSI adds the CSI env vars, volumes and containers to the provided podSpec. -func (s *Deployment) addCSI(podSpec *corev1.PodSpec) { - hostpathDirOrCreate := corev1.HostPathDirectoryOrCreate - hostpathDir := corev1.HostPathDirectory - mountPropagationBidirectional := corev1.MountPropagationBidirectional - - nodeContainer := &podSpec.Containers[0] - - // Add CSI specific configurations if enabled. - if s.stos.Spec.CSI.Enable { - vols := []corev1.Volume{ - { - Name: "registrar-socket-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIRegistrarSocketDir(), - Type: &hostpathDirOrCreate, - }, - }, - }, - { - Name: "kubelet-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIKubeletDir(), - Type: &hostpathDir, - }, - }, - }, - { - Name: "plugin-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), - Type: &hostpathDirOrCreate, - }, - }, - }, - { - Name: "device-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIDeviceDir(), - Type: &hostpathDir, - }, - }, - }, - { - Name: "registration-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIRegistrationDir(CSIV1Supported(s.k8sVersion)), - Type: &hostpathDir, - }, - }, - }, - } - - podSpec.Volumes = append(podSpec.Volumes, vols...) - - volMnts := []corev1.VolumeMount{ - { - Name: "kubelet-dir", - MountPath: s.stos.Spec.GetCSIKubeletDir(), - MountPropagation: &mountPropagationBidirectional, - }, - { - Name: "plugin-dir", - MountPath: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), - }, - { - Name: "device-dir", - MountPath: s.stos.Spec.GetCSIDeviceDir(), - }, - } - - // Append volume mounts to the first container, the only container is the node container, at this point. - nodeContainer.VolumeMounts = append(nodeContainer.VolumeMounts, volMnts...) - - envVar := []corev1.EnvVar{ - { - Name: csiEndpointEnvVar, - Value: s.stos.Spec.GetCSIEndpoint(CSIV1Supported(s.k8sVersion)), - }, - } - - // Append CSI Provision Creds env var if enabled. - if s.stos.Spec.CSI.EnableProvisionCreds { - envVar = append( - envVar, - corev1.EnvVar{ - Name: csiRequireCredsCreateEnvVar, - Value: "true", - }, - corev1.EnvVar{ - Name: csiRequireCredsDeleteEnvVar, - Value: "true", - }, - getCSICredsEnvVar(csiProvisionCredsUsernameEnvVar, csiProvisionerSecretName, "username"), - getCSICredsEnvVar(csiProvisionCredsPasswordEnvVar, csiProvisionerSecretName, "password"), - ) - } - - // Append CSI Controller Publish env var if enabled. - if s.stos.Spec.CSI.EnableControllerPublishCreds { - envVar = append( - envVar, - corev1.EnvVar{ - Name: csiRequireCredsCtrlPubEnvVar, - Value: "true", - }, - corev1.EnvVar{ - Name: csiRequireCredsCtrlUnpubEnvVar, - Value: "true", - }, - getCSICredsEnvVar(csiControllerPubCredsUsernameEnvVar, csiControllerPublishSecretName, "username"), - getCSICredsEnvVar(csiControllerPubCredsPasswordEnvVar, csiControllerPublishSecretName, "password"), - ) - } - - // Append CSI Node Publish env var if enabled. - if s.stos.Spec.CSI.EnableNodePublishCreds { - envVar = append( - envVar, - corev1.EnvVar{ - Name: csiRequireCredsNodePubEnvVar, - Value: "true", - }, - getCSICredsEnvVar(csiNodePubCredsUsernameEnvVar, csiNodePublishSecretName, "username"), - getCSICredsEnvVar(csiNodePubCredsPasswordEnvVar, csiNodePublishSecretName, "password"), - ) - } - - // Append env vars to the first container, node container. - nodeContainer.Env = append(nodeContainer.Env, envVar...) - - driverReg := corev1.Container{ - Image: s.stos.Spec.GetCSINodeDriverRegistrarImage(CSIV1Supported(s.k8sVersion)), - Name: "csi-driver-registrar", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--v=5", - "--csi-address=$(ADDRESS)", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - { - Name: kubeNodeNameEnvVar, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "spec.nodeName", - }, - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - { - Name: "registrar-socket-dir", - MountPath: "/var/lib/csi/sockets/", - }, - { - Name: "registration-dir", - MountPath: "/registration", - }, - }, - } - - // Add extra flags to activate node-register mode if kubelet plugins - // watcher is supported. - if kubeletPluginsWatcherSupported(s.k8sVersion) { - driverReg.Args = append( - driverReg.Args, - fmt.Sprintf("--kubelet-registration-path=%s", s.stos.Spec.GetCSIKubeletRegistrationPath(CSIV1Supported(s.k8sVersion)))) - } - podSpec.Containers = append(podSpec.Containers, driverReg) - - if CSIV1Supported(s.k8sVersion) { - livenessProbe := corev1.Container{ - Image: s.stos.Spec.GetCSILivenessProbeImage(), - Name: "csi-liveness-probe", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--csi-address=$(ADDRESS)", - "--connection-timeout=3s", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - }, - } - podSpec.Containers = append(podSpec.Containers, livenessProbe) - } - } -} - -// addNodeAffinity adds node affinity to the given pod spec from the cluster -// spec NodeSelectorLabel. -func (s *Deployment) addNodeAffinity(podSpec *corev1.PodSpec) { - if len(s.stos.Spec.NodeSelectorTerms) > 0 { - podSpec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: s.stos.Spec.NodeSelectorTerms, - }, - }} - } -} - -// addTolerations adds tolerations to the given pod spec from cluster -// spec Tolerations. -func (s *Deployment) addTolerations(podSpec *corev1.PodSpec) error { - tolerations := s.stos.Spec.Tolerations - for i := range tolerations { - if tolerations[i].Operator == corev1.TolerationOpExists && tolerations[i].Value != "" { - return fmt.Errorf("key(%s): toleration value must be empty when `operator` is 'Exists'", tolerations[i].Key) - } - } - if len(tolerations) > 0 { - podSpec.Tolerations = s.stos.Spec.Tolerations - } - return nil -} - -// getCSICredsEnvVar returns a corev1.EnvVar object with value from a secret key -// reference, given env var name, reference secret name and key in the secret. -func getCSICredsEnvVar(envVarName, secretName, key string) corev1.EnvVar { - return corev1.EnvVar{ - Name: envVarName, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretName, +// getCSICredsEnvVar returns a corev1.EnvVar object with value from a secret key +// reference, given env var name, reference secret name and key in the secret. +func getCSICredsEnvVar(envVarName, secretName, key string) corev1.EnvVar { + return corev1.EnvVar{ + Name: envVarName, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, }, Key: key, }, @@ -1238,594 +310,6 @@ func getCSICredsEnvVar(envVarName, secretName, key string) corev1.EnvVar { } } -func (s *Deployment) createStatefulSet() error { - ls := labelsForStatefulSet(s.stos.Name) - replicas := int32(1) - hostpathDirOrCreate := corev1.HostPathDirectoryOrCreate - - sset := &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: statefulsetName, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": "storageos", - }, - }, - Spec: appsv1.StatefulSetSpec{ - ServiceName: "storageos", - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: ls, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: ls, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: "storageos-statefulset-sa", - Containers: []corev1.Container{ - { - Image: s.stos.Spec.GetCSIExternalProvisionerImage(CSIV1Supported(s.k8sVersion)), - Name: "csi-external-provisioner", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--v=5", - "--provisioner=storageos", - "--csi-address=$(ADDRESS)", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - }, - }, - { - Image: s.stos.Spec.GetCSIExternalAttacherImage(CSIV1Supported(s.k8sVersion)), - Name: "csi-external-attacher", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--v=5", - "--csi-address=$(ADDRESS)", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "plugin-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), - Type: &hostpathDirOrCreate, - }, - }, - }, - }, - }, - }, - }, - } - - if CSIV1Supported(s.k8sVersion) { - driverReg := corev1.Container{ - Image: s.stos.Spec.GetCSIClusterDriverRegistrarImage(), - Name: "csi-driver-k8s-registrar", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--v=5", - "--csi-address=$(ADDRESS)", - "--pod-info-mount-version=v1", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - { - Name: kubeNodeNameEnvVar, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "spec.nodeName", - }, - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - }, - } - - sset.Spec.Template.Spec.Containers = append(sset.Spec.Template.Spec.Containers, driverReg) - } - - podSpec := &sset.Spec.Template.Spec - - s.addNodeAffinity(podSpec) - - if err := s.addTolerations(podSpec); err != nil { - return err - } - - return s.createOrUpdateObject(sset) -} - -func (s *Deployment) deleteStatefulSet(name string) error { - return s.deleteObject(s.getStatefulSet(name)) -} - -func (s *Deployment) getStatefulSet(name string) *appsv1.StatefulSet { - return &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": "storageos", - }, - }, - } -} - -func (s *Deployment) createService() error { - svc := s.getService(s.stos.Spec.GetServiceName()) - svc.Spec = corev1.ServiceSpec{ - Type: corev1.ServiceType(s.stos.Spec.GetServiceType()), - Ports: []corev1.ServicePort{ - { - Name: s.stos.Spec.GetServiceName(), - Protocol: "TCP", - Port: int32(s.stos.Spec.GetServiceInternalPort()), - TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: int32(s.stos.Spec.GetServiceExternalPort())}, - }, - }, - Selector: map[string]string{ - "app": appName, - "kind": daemonsetKind, - }, - } - - if err := s.client.Create(context.Background(), svc); err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create %s: %v", svc.GroupVersionKind().Kind, err) - } - // if err := s.createOrUpdateObject(svc); err != nil { - // return err - // } - - // Patch storageos-api secret with above service IP in apiAddress. - if !s.stos.Spec.CSI.Enable { - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: s.stos.Spec.SecretRefName, - Namespace: s.stos.Spec.SecretRefNamespace, - }, - } - nsNameSecret := types.NamespacedName{ - Namespace: secret.ObjectMeta.GetNamespace(), - Name: secret.ObjectMeta.GetName(), - } - if err := s.client.Get(context.Background(), nsNameSecret, secret); err != nil { - return err - } - - nsNameService := types.NamespacedName{ - Namespace: svc.ObjectMeta.GetNamespace(), - Name: svc.ObjectMeta.GetName(), - } - if err := s.client.Get(context.Background(), nsNameService, svc); err != nil { - return err - } - - apiAddress := fmt.Sprintf("tcp://%s:5705", svc.Spec.ClusterIP) - secret.Data[apiAddressKey] = []byte(apiAddress) - - if err := s.client.Update(context.Background(), secret); err != nil { - return err - } - } - - return nil -} - -func (s *Deployment) deleteService(name string) error { - return s.deleteObject(s.getService(name)) -} - -func (s *Deployment) getService(name string) *corev1.Service { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Service", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - Annotations: s.stos.Spec.Service.Annotations, - }, - } -} - -func (s *Deployment) createIngress() error { - ingress := &v1beta1.Ingress{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "extensions/v1beta1", - Kind: "Ingress", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "storageos-ingress", - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - Annotations: s.stos.Spec.Ingress.Annotations, - }, - Spec: v1beta1.IngressSpec{ - Backend: &v1beta1.IngressBackend{ - ServiceName: s.stos.Spec.GetServiceName(), - ServicePort: intstr.IntOrString{Type: intstr.Int, IntVal: int32(s.stos.Spec.GetServiceExternalPort())}, - }, - }, - } - - if s.stos.Spec.Ingress.TLS { - ingress.Spec.TLS = []v1beta1.IngressTLS{ - v1beta1.IngressTLS{ - Hosts: []string{s.stos.Spec.Ingress.Hostname}, - SecretName: tlsSecretName, - }, - } - } - - return s.createOrUpdateObject(ingress) -} - -func (s *Deployment) deleteIngress(name string) error { - return s.deleteObject(s.getIngress(name)) -} - -func (s *Deployment) getIngress(name string) *v1beta1.Ingress { - return &v1beta1.Ingress{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "extensions/v1beta1", - Kind: "Ingress", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - Annotations: s.stos.Spec.Ingress.Annotations, - }, - } -} - -func (s *Deployment) createTLSSecret() error { - cert, key, err := s.getTLSData() - if err != nil { - return err - } - - secret := s.getSecret(tlsSecretName) - secret.Type = corev1.SecretType(tlsSecretType) - secret.Data = map[string][]byte{ - tlsCertKey: cert, - tlsKeyKey: key, - } - return s.createOrUpdateObject(secret) -} - -func (s *Deployment) deleteSecret(name string) error { - return s.deleteObject(s.getSecret(name)) -} - -func (s *Deployment) getSecret(name string) *corev1.Secret { - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createInitSecret() error { - username, password, err := s.getAdminCreds() - if err != nil { - return err - } - if err := s.createCredSecret(initSecretName, username, password); err != nil { - return err - } - return nil -} - -func (s *Deployment) getAdminCreds() ([]byte, []byte, error) { - var username, password []byte - if s.stos.Spec.SecretRefName != "" && s.stos.Spec.SecretRefNamespace != "" { - se := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: s.stos.Spec.SecretRefName, - Namespace: s.stos.Spec.SecretRefNamespace, - }, - } - nsName := types.NamespacedName{ - Name: se.ObjectMeta.GetName(), - Namespace: se.ObjectMeta.GetNamespace(), - } - if err := s.client.Get(context.Background(), nsName, se); err != nil { - return nil, nil, err - } - - username = se.Data[apiUsernameKey] - password = se.Data[apiPasswordKey] - } else { - // Use the default credentials. - username = []byte(defaultUsername) - password = []byte(defaultPassword) - } - - return username, password, nil -} - -func (s *Deployment) getTLSData() ([]byte, []byte, error) { - var cert, key []byte - if s.stos.Spec.SecretRefName != "" && s.stos.Spec.SecretRefNamespace != "" { - se := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: s.stos.Spec.SecretRefName, - Namespace: s.stos.Spec.SecretRefNamespace, - }, - } - nsName := types.NamespacedName{ - Name: se.ObjectMeta.GetName(), - Namespace: se.ObjectMeta.GetNamespace(), - } - if err := s.client.Get(context.Background(), nsName, se); err != nil { - return nil, nil, err - } - - cert = se.Data[tlsCertKey] - key = se.Data[tlsKeyKey] - } else { - cert = []byte("") - key = []byte("") - } - - return cert, key, nil -} - -// createCSISecrets checks which CSI creds are enabled and creates secret for -// those components. -func (s *Deployment) createCSISecrets() error { - // Create Provision Secret. - if s.stos.Spec.CSI.EnableProvisionCreds { - username, password, err := s.getCSICreds(csiProvisionUsernameKey, csiProvisionPasswordKey) - if err != nil { - return err - } - if err := s.createCredSecret(csiProvisionerSecretName, username, password); err != nil { - return err - } - } - - // Create Controller Publish Secret. - if s.stos.Spec.CSI.EnableControllerPublishCreds { - username, password, err := s.getCSICreds(csiControllerPublishUsernameKey, csiControllerPublishPasswordKey) - if err != nil { - return err - } - if err := s.createCredSecret(csiControllerPublishSecretName, username, password); err != nil { - return err - } - } - - // Create Node Publish Secret. - if s.stos.Spec.CSI.EnableNodePublishCreds { - username, password, err := s.getCSICreds(csiNodePublishUsernameKey, csiNodePublishPasswordKey) - if err != nil { - return err - } - if err := s.createCredSecret(csiNodePublishSecretName, username, password); err != nil { - return err - } - } - - return nil -} - -// deleteCSISecrets deletes all the CSI related secrets. -func (s *Deployment) deleteCSISecrets() error { - if err := s.deleteSecret(csiProvisionerSecretName); err != nil { - return err - } - - if err := s.deleteSecret(csiControllerPublishSecretName); err != nil { - return err - } - - if err := s.deleteSecret(csiNodePublishSecretName); err != nil { - return err - } - - return nil -} - -func (s *Deployment) createCredSecret(name string, username, password []byte) error { - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - }, - Type: corev1.SecretType(corev1.SecretTypeOpaque), - Data: map[string][]byte{ - "username": username, - "password": password, - }, - } - - return s.createOrUpdateObject(secret) -} - -// getCSICreds - given username and password keys, it fetches the creds from -// storageos-api secret and returns them. -func (s *Deployment) getCSICreds(usernameKey, passwordKey string) (username []byte, password []byte, err error) { - // Get the username and password from storageos-api secret object. - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: s.stos.Spec.SecretRefName, - Namespace: s.stos.Spec.SecretRefNamespace, - }, - } - nsName := types.NamespacedName{ - Name: secret.ObjectMeta.GetName(), - Namespace: secret.ObjectMeta.GetNamespace(), - } - if err := s.client.Get(context.Background(), nsName, secret); err != nil { - return nil, nil, err - } - - username = secret.Data[usernameKey] - password = secret.Data[passwordKey] - - return username, password, err -} - -func (s *Deployment) createStorageClass() error { - // Provisioner name for in-tree storage plugin. - provisioner := intreeProvisionerName - - if s.stos.Spec.CSI.Enable { - provisioner = csiProvisionerName - } - - sc := s.getStorageClass("fast") - sc.Provisioner = provisioner - sc.Parameters = map[string]string{ - "pool": "default", - } - - if s.stos.Spec.CSI.Enable { - // Add CSI creds secrets in parameters. - if CSIV1Supported(s.k8sVersion) { - // New CSI secret parameter keys were introduced in CSI v1. - sc.Parameters[csiV1FSType] = defaultFSType - if s.stos.Spec.CSI.EnableProvisionCreds { - sc.Parameters[csiV1ProvisionerSecretNameKey] = csiProvisionerSecretName - sc.Parameters[csiV1ProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableControllerPublishCreds { - sc.Parameters[csiV1ControllerPublishSecretNameKey] = csiControllerPublishSecretName - sc.Parameters[csiV1ControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableNodePublishCreds { - sc.Parameters[csiV1NodePublishSecretNameKey] = csiNodePublishSecretName - sc.Parameters[csiV1NodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - } else { - sc.Parameters[fsType] = defaultFSType - if s.stos.Spec.CSI.EnableProvisionCreds { - sc.Parameters[csiV0ProvisionerSecretNameKey] = csiProvisionerSecretName - sc.Parameters[csiV0ProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableControllerPublishCreds { - sc.Parameters[csiV0ControllerPublishSecretNameKey] = csiControllerPublishSecretName - sc.Parameters[csiV0ControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableNodePublishCreds { - sc.Parameters[csiV0NodePublishSecretNameKey] = csiNodePublishSecretName - sc.Parameters[csiV0NodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - } - } else { - sc.Parameters[fsType] = defaultFSType - // Add StorageOS admin secrets name and namespace. - sc.Parameters[secretNamespaceKey] = s.stos.Spec.SecretRefNamespace - sc.Parameters[secretNameKey] = s.stos.Spec.SecretRefName - } - - return s.createOrUpdateObject(sc) -} - -func (s *Deployment) deleteStorageClass(name string) error { - return s.deleteObject(s.getStorageClass(name)) -} - -func (s *Deployment) getStorageClass(name string) *storagev1.StorageClass { - return &storagev1.StorageClass{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "storage.k8s.io/v1", - Kind: "StorageClass", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - // createOrUpdateObject attempts to create a given object. If the object already // exists and `Deployment.update` is false, no change is made. If update is true, // the existing object is updated. diff --git a/pkg/storageos/ingress.go b/pkg/storageos/ingress.go new file mode 100644 index 000000000..49272bcea --- /dev/null +++ b/pkg/storageos/ingress.go @@ -0,0 +1,62 @@ +package storageos + +import ( + "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func (s *Deployment) createIngress() error { + ingress := &v1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "extensions/v1beta1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "storageos-ingress", + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + Annotations: s.stos.Spec.Ingress.Annotations, + }, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: s.stos.Spec.GetServiceName(), + ServicePort: intstr.IntOrString{Type: intstr.Int, IntVal: int32(s.stos.Spec.GetServiceExternalPort())}, + }, + }, + } + + if s.stos.Spec.Ingress.TLS { + ingress.Spec.TLS = []v1beta1.IngressTLS{ + v1beta1.IngressTLS{ + Hosts: []string{s.stos.Spec.Ingress.Hostname}, + SecretName: tlsSecretName, + }, + } + } + + return s.createOrUpdateObject(ingress) +} + +func (s *Deployment) deleteIngress(name string) error { + return s.deleteObject(s.getIngress(name)) +} + +func (s *Deployment) getIngress(name string) *v1beta1.Ingress { + return &v1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "extensions/v1beta1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + Annotations: s.stos.Spec.Ingress.Annotations, + }, + } +} diff --git a/pkg/storageos/podspec.go b/pkg/storageos/podspec.go new file mode 100644 index 000000000..11840d9e7 --- /dev/null +++ b/pkg/storageos/podspec.go @@ -0,0 +1,278 @@ +package storageos + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" +) + +// addSharedDir adds env var and volumes for shared dir when running kubelet in +// a container. +func (s *Deployment) addSharedDir(podSpec *corev1.PodSpec) { + mountPropagationBidirectional := corev1.MountPropagationBidirectional + nodeContainer := &podSpec.Containers[0] + + // If kubelet is running in a container, sharedDir should be set. + if s.stos.Spec.SharedDir != "" { + envVar := corev1.EnvVar{ + Name: deviceDirEnvVar, + Value: fmt.Sprintf("%s/devices", s.stos.Spec.SharedDir), + } + nodeContainer.Env = append(nodeContainer.Env, envVar) + + sharedDir := corev1.Volume{ + Name: "shared", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.SharedDir, + }, + }, + } + podSpec.Volumes = append(podSpec.Volumes, sharedDir) + + volMnt := corev1.VolumeMount{ + Name: "shared", + MountPath: s.stos.Spec.SharedDir, + MountPropagation: &mountPropagationBidirectional, + } + nodeContainer.VolumeMounts = append(nodeContainer.VolumeMounts, volMnt) + } +} + +// addCSI adds the CSI env vars, volumes and containers to the provided podSpec. +func (s *Deployment) addCSI(podSpec *corev1.PodSpec) { + hostpathDirOrCreate := corev1.HostPathDirectoryOrCreate + hostpathDir := corev1.HostPathDirectory + mountPropagationBidirectional := corev1.MountPropagationBidirectional + + nodeContainer := &podSpec.Containers[0] + + // Add CSI specific configurations if enabled. + if s.stos.Spec.CSI.Enable { + vols := []corev1.Volume{ + { + Name: "registrar-socket-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIRegistrarSocketDir(), + Type: &hostpathDirOrCreate, + }, + }, + }, + { + Name: "kubelet-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIKubeletDir(), + Type: &hostpathDir, + }, + }, + }, + { + Name: "plugin-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), + Type: &hostpathDirOrCreate, + }, + }, + }, + { + Name: "device-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIDeviceDir(), + Type: &hostpathDir, + }, + }, + }, + { + Name: "registration-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIRegistrationDir(CSIV1Supported(s.k8sVersion)), + Type: &hostpathDir, + }, + }, + }, + } + + podSpec.Volumes = append(podSpec.Volumes, vols...) + + volMnts := []corev1.VolumeMount{ + { + Name: "kubelet-dir", + MountPath: s.stos.Spec.GetCSIKubeletDir(), + MountPropagation: &mountPropagationBidirectional, + }, + { + Name: "plugin-dir", + MountPath: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), + }, + { + Name: "device-dir", + MountPath: s.stos.Spec.GetCSIDeviceDir(), + }, + } + + // Append volume mounts to the first container, the only container is the node container, at this point. + nodeContainer.VolumeMounts = append(nodeContainer.VolumeMounts, volMnts...) + + envVar := []corev1.EnvVar{ + { + Name: csiEndpointEnvVar, + Value: s.stos.Spec.GetCSIEndpoint(CSIV1Supported(s.k8sVersion)), + }, + } + + // Append CSI Provision Creds env var if enabled. + if s.stos.Spec.CSI.EnableProvisionCreds { + envVar = append( + envVar, + corev1.EnvVar{ + Name: csiRequireCredsCreateEnvVar, + Value: "true", + }, + corev1.EnvVar{ + Name: csiRequireCredsDeleteEnvVar, + Value: "true", + }, + getCSICredsEnvVar(csiProvisionCredsUsernameEnvVar, csiProvisionerSecretName, "username"), + getCSICredsEnvVar(csiProvisionCredsPasswordEnvVar, csiProvisionerSecretName, "password"), + ) + } + + // Append CSI Controller Publish env var if enabled. + if s.stos.Spec.CSI.EnableControllerPublishCreds { + envVar = append( + envVar, + corev1.EnvVar{ + Name: csiRequireCredsCtrlPubEnvVar, + Value: "true", + }, + corev1.EnvVar{ + Name: csiRequireCredsCtrlUnpubEnvVar, + Value: "true", + }, + getCSICredsEnvVar(csiControllerPubCredsUsernameEnvVar, csiControllerPublishSecretName, "username"), + getCSICredsEnvVar(csiControllerPubCredsPasswordEnvVar, csiControllerPublishSecretName, "password"), + ) + } + + // Append CSI Node Publish env var if enabled. + if s.stos.Spec.CSI.EnableNodePublishCreds { + envVar = append( + envVar, + corev1.EnvVar{ + Name: csiRequireCredsNodePubEnvVar, + Value: "true", + }, + getCSICredsEnvVar(csiNodePubCredsUsernameEnvVar, csiNodePublishSecretName, "username"), + getCSICredsEnvVar(csiNodePubCredsPasswordEnvVar, csiNodePublishSecretName, "password"), + ) + } + + // Append env vars to the first container, node container. + nodeContainer.Env = append(nodeContainer.Env, envVar...) + + driverReg := corev1.Container{ + Image: s.stos.Spec.GetCSINodeDriverRegistrarImage(CSIV1Supported(s.k8sVersion)), + Name: "csi-driver-registrar", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--v=5", + "--csi-address=$(ADDRESS)", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + { + Name: kubeNodeNameEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + { + Name: "registrar-socket-dir", + MountPath: "/var/lib/csi/sockets/", + }, + { + Name: "registration-dir", + MountPath: "/registration", + }, + }, + } + + // Add extra flags to activate node-register mode if kubelet plugins + // watcher is supported. + if kubeletPluginsWatcherSupported(s.k8sVersion) { + driverReg.Args = append( + driverReg.Args, + fmt.Sprintf("--kubelet-registration-path=%s", s.stos.Spec.GetCSIKubeletRegistrationPath(CSIV1Supported(s.k8sVersion)))) + } + podSpec.Containers = append(podSpec.Containers, driverReg) + + if CSIV1Supported(s.k8sVersion) { + livenessProbe := corev1.Container{ + Image: s.stos.Spec.GetCSILivenessProbeImage(), + Name: "csi-liveness-probe", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--csi-address=$(ADDRESS)", + "--connection-timeout=3s", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + }, + } + podSpec.Containers = append(podSpec.Containers, livenessProbe) + } + } +} + +// addNodeAffinity adds node affinity to the given pod spec from the cluster +// spec NodeSelectorLabel. +func (s *Deployment) addNodeAffinity(podSpec *corev1.PodSpec) { + if len(s.stos.Spec.NodeSelectorTerms) > 0 { + podSpec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: s.stos.Spec.NodeSelectorTerms, + }, + }} + } +} + +// addTolerations adds tolerations to the given pod spec from cluster +// spec Tolerations. +func (s *Deployment) addTolerations(podSpec *corev1.PodSpec) error { + tolerations := s.stos.Spec.Tolerations + for i := range tolerations { + if tolerations[i].Operator == corev1.TolerationOpExists && tolerations[i].Value != "" { + return fmt.Errorf("key(%s): toleration value must be empty when `operator` is 'Exists'", tolerations[i].Key) + } + } + if len(tolerations) > 0 { + podSpec.Tolerations = s.stos.Spec.Tolerations + } + return nil +} diff --git a/pkg/storageos/rbac.go b/pkg/storageos/rbac.go new file mode 100644 index 000000000..7c063e6ac --- /dev/null +++ b/pkg/storageos/rbac.go @@ -0,0 +1,321 @@ +package storageos + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (s *Deployment) createServiceAccount(name string) error { + sa := s.getServiceAccount(name) + return s.createOrUpdateObject(sa) +} + +func (s *Deployment) deleteServiceAccount(name string) error { + return s.deleteObject(s.getServiceAccount(name)) +} + +// getServiceAccount creates a generic service account object with the given +// name and returns it. +func (s *Deployment) getServiceAccount(name string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createServiceAccountForDaemonSet() error { + return s.createServiceAccount("storageos-daemonset-sa") +} + +func (s *Deployment) createServiceAccountForStatefulSet() error { + return s.createServiceAccount("storageos-statefulset-sa") +} + +func (s *Deployment) createRoleForKeyMgmt() error { + role := s.getRole(keyManagementRoleName) + role.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "create", "delete"}, + }, + } + + return s.createOrUpdateObject(role) +} + +func (s *Deployment) deleteRole(name string) error { + return s.deleteObject(s.getRole(keyManagementRoleName)) +} + +// getRole creates a generic role object with the given name and returns it. +func (s *Deployment) getRole(name string) *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "Role", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createClusterRole(name string, rules []rbacv1.PolicyRule) error { + role := s.getClusterRole(name) + role.Rules = rules + return s.createOrUpdateObject(role) +} + +func (s *Deployment) deleteClusterRole(name string) error { + return s.deleteObject(s.getClusterRole(name)) +} + +func (s *Deployment) getClusterRole(name string) *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRole", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createClusterRoleForDriverRegistrar() error { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list", "watch", "create", "update", "patch"}, + }, + { + APIGroups: []string{"apiextensions.k8s.io"}, + Resources: []string{"customresourcedefinitions"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"csi.storage.k8s.io"}, + Resources: []string{"csidrivers"}, + Verbs: []string{"create"}, + }, + } + return s.createClusterRole("driver-registrar-role", rules) +} + +func (s *Deployment) createClusterRoleForProvisioner() error { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumes"}, + Verbs: []string{"list", "watch", "create", "delete"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumeclaims"}, + Verbs: []string{"get", "list", "watch", "update"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"storageclasses"}, + Verbs: []string{"list", "watch", "get"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list", "watch", "create", "update", "patch"}, + }, + } + return s.createClusterRole("csi-provisioner-role", rules) +} + +func (s *Deployment) createClusterRoleForAttacher() error { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumes"}, + Verbs: []string{"get", "list", "watch", "update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"storageclasses"}, + Verbs: []string{"list", "watch", "get"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"volumeattachments"}, + Verbs: []string{"get", "list", "watch", "update"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"csinodeinfos"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list", "watch", "create", "update", "patch"}, + }, + } + return s.createClusterRole("csi-attacher-role", rules) +} + +func (s *Deployment) createRoleBindingForKeyMgmt() error { + roleBinding := s.getRoleBinding(keyManagementBindingName) + roleBinding.Subjects = []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "storageos-daemonset-sa", + Namespace: s.stos.Spec.GetResourceNS(), + }, + } + roleBinding.RoleRef = rbacv1.RoleRef{ + Kind: "Role", + Name: keyManagementRoleName, + APIGroup: "rbac.authorization.k8s.io", + } + return s.createOrUpdateObject(roleBinding) +} + +func (s *Deployment) deleteRoleBinding(name string) error { + return s.deleteObject(s.getRoleBinding(name)) +} + +func (s *Deployment) getRoleBinding(name string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createClusterRoleBinding(name string, subjects []rbacv1.Subject, roleRef rbacv1.RoleRef) error { + roleBinding := s.getClusterRoleBinding(name) + roleBinding.Subjects = subjects + roleBinding.RoleRef = roleRef + return s.createOrUpdateObject(roleBinding) +} + +func (s *Deployment) deleteClusterRoleBinding(name string) error { + return s.deleteObject(s.getClusterRoleBinding(name)) +} + +func (s *Deployment) getClusterRoleBinding(name string) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createClusterRoleBindingForDriverRegistrar() error { + subjects := []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "storageos-daemonset-sa", + Namespace: s.stos.Spec.GetResourceNS(), + }, + } + roleRef := rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "driver-registrar-role", + APIGroup: "rbac.authorization.k8s.io", + } + return s.createClusterRoleBinding("driver-registrar-binding", subjects, roleRef) +} + +func (s *Deployment) createClusterRoleBindingForK8SDriverRegistrar() error { + subjects := []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "storageos-statefulset-sa", + Namespace: s.stos.Spec.GetResourceNS(), + }, + } + roleRef := rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "driver-registrar-role", + APIGroup: "rbac.authorization.k8s.io", + } + return s.createClusterRoleBinding("k8s-driver-registrar-binding", subjects, roleRef) +} + +func (s *Deployment) createClusterRoleBindingForProvisioner() error { + subjects := []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "storageos-statefulset-sa", + Namespace: s.stos.Spec.GetResourceNS(), + }, + } + roleRef := rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "csi-provisioner-role", + APIGroup: "rbac.authorization.k8s.io", + } + return s.createClusterRoleBinding("csi-provisioner-binding", subjects, roleRef) +} + +func (s *Deployment) createClusterRoleBindingForAttacher() error { + subjects := []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "storageos-statefulset-sa", + Namespace: s.stos.Spec.GetResourceNS(), + }, + } + roleRef := rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "csi-attacher-role", + APIGroup: "rbac.authorization.k8s.io", + } + return s.createClusterRoleBinding("csi-attacher-binding", subjects, roleRef) +} diff --git a/pkg/storageos/secret.go b/pkg/storageos/secret.go new file mode 100644 index 000000000..670eb8379 --- /dev/null +++ b/pkg/storageos/secret.go @@ -0,0 +1,225 @@ +package storageos + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func (s *Deployment) deleteSecret(name string) error { + return s.deleteObject(s.getSecret(name)) +} + +func (s *Deployment) getSecret(name string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createInitSecret() error { + username, password, err := s.getAdminCreds() + if err != nil { + return err + } + if err := s.createCredSecret(initSecretName, username, password); err != nil { + return err + } + return nil +} + +func (s *Deployment) createTLSSecret() error { + cert, key, err := s.getTLSData() + if err != nil { + return err + } + + secret := s.getSecret(tlsSecretName) + secret.Type = corev1.SecretType(tlsSecretType) + secret.Data = map[string][]byte{ + tlsCertKey: cert, + tlsKeyKey: key, + } + return s.createOrUpdateObject(secret) +} + +func (s *Deployment) getAdminCreds() ([]byte, []byte, error) { + var username, password []byte + if s.stos.Spec.SecretRefName != "" && s.stos.Spec.SecretRefNamespace != "" { + se := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: s.stos.Spec.SecretRefName, + Namespace: s.stos.Spec.SecretRefNamespace, + }, + } + nsName := types.NamespacedName{ + Name: se.ObjectMeta.GetName(), + Namespace: se.ObjectMeta.GetNamespace(), + } + if err := s.client.Get(context.Background(), nsName, se); err != nil { + return nil, nil, err + } + + username = se.Data[apiUsernameKey] + password = se.Data[apiPasswordKey] + } else { + // Use the default credentials. + username = []byte(defaultUsername) + password = []byte(defaultPassword) + } + + return username, password, nil +} + +func (s *Deployment) getTLSData() ([]byte, []byte, error) { + var cert, key []byte + if s.stos.Spec.SecretRefName != "" && s.stos.Spec.SecretRefNamespace != "" { + se := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: s.stos.Spec.SecretRefName, + Namespace: s.stos.Spec.SecretRefNamespace, + }, + } + nsName := types.NamespacedName{ + Name: se.ObjectMeta.GetName(), + Namespace: se.ObjectMeta.GetNamespace(), + } + if err := s.client.Get(context.Background(), nsName, se); err != nil { + return nil, nil, err + } + + cert = se.Data[tlsCertKey] + key = se.Data[tlsKeyKey] + } else { + cert = []byte("") + key = []byte("") + } + + return cert, key, nil +} + +// createCSISecrets checks which CSI creds are enabled and creates secret for +// those components. +func (s *Deployment) createCSISecrets() error { + // Create Provision Secret. + if s.stos.Spec.CSI.EnableProvisionCreds { + username, password, err := s.getCSICreds(csiProvisionUsernameKey, csiProvisionPasswordKey) + if err != nil { + return err + } + if err := s.createCredSecret(csiProvisionerSecretName, username, password); err != nil { + return err + } + } + + // Create Controller Publish Secret. + if s.stos.Spec.CSI.EnableControllerPublishCreds { + username, password, err := s.getCSICreds(csiControllerPublishUsernameKey, csiControllerPublishPasswordKey) + if err != nil { + return err + } + if err := s.createCredSecret(csiControllerPublishSecretName, username, password); err != nil { + return err + } + } + + // Create Node Publish Secret. + if s.stos.Spec.CSI.EnableNodePublishCreds { + username, password, err := s.getCSICreds(csiNodePublishUsernameKey, csiNodePublishPasswordKey) + if err != nil { + return err + } + if err := s.createCredSecret(csiNodePublishSecretName, username, password); err != nil { + return err + } + } + + return nil +} + +// deleteCSISecrets deletes all the CSI related secrets. +func (s *Deployment) deleteCSISecrets() error { + if err := s.deleteSecret(csiProvisionerSecretName); err != nil { + return err + } + + if err := s.deleteSecret(csiControllerPublishSecretName); err != nil { + return err + } + + if err := s.deleteSecret(csiNodePublishSecretName); err != nil { + return err + } + + return nil +} + +func (s *Deployment) createCredSecret(name string, username, password []byte) error { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + }, + Type: corev1.SecretType(corev1.SecretTypeOpaque), + Data: map[string][]byte{ + "username": username, + "password": password, + }, + } + + return s.createOrUpdateObject(secret) +} + +// getCSICreds - given username and password keys, it fetches the creds from +// storageos-api secret and returns them. +func (s *Deployment) getCSICreds(usernameKey, passwordKey string) (username []byte, password []byte, err error) { + // Get the username and password from storageos-api secret object. + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: s.stos.Spec.SecretRefName, + Namespace: s.stos.Spec.SecretRefNamespace, + }, + } + nsName := types.NamespacedName{ + Name: secret.ObjectMeta.GetName(), + Namespace: secret.ObjectMeta.GetNamespace(), + } + if err := s.client.Get(context.Background(), nsName, secret); err != nil { + return nil, nil, err + } + + username = secret.Data[usernameKey] + password = secret.Data[passwordKey] + + return username, password, err +} diff --git a/pkg/storageos/service.go b/pkg/storageos/service.go new file mode 100644 index 000000000..7effdca55 --- /dev/null +++ b/pkg/storageos/service.go @@ -0,0 +1,97 @@ +package storageos + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func (s *Deployment) createService() error { + svc := s.getService(s.stos.Spec.GetServiceName()) + svc.Spec = corev1.ServiceSpec{ + Type: corev1.ServiceType(s.stos.Spec.GetServiceType()), + Ports: []corev1.ServicePort{ + { + Name: s.stos.Spec.GetServiceName(), + Protocol: "TCP", + Port: int32(s.stos.Spec.GetServiceInternalPort()), + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: int32(s.stos.Spec.GetServiceExternalPort())}, + }, + }, + Selector: map[string]string{ + "app": appName, + "kind": daemonsetKind, + }, + } + + if err := s.client.Create(context.Background(), svc); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create %s: %v", svc.GroupVersionKind().Kind, err) + } + // if err := s.createOrUpdateObject(svc); err != nil { + // return err + // } + + // Patch storageos-api secret with above service IP in apiAddress. + if !s.stos.Spec.CSI.Enable { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: s.stos.Spec.SecretRefName, + Namespace: s.stos.Spec.SecretRefNamespace, + }, + } + nsNameSecret := types.NamespacedName{ + Namespace: secret.ObjectMeta.GetNamespace(), + Name: secret.ObjectMeta.GetName(), + } + if err := s.client.Get(context.Background(), nsNameSecret, secret); err != nil { + return err + } + + nsNameService := types.NamespacedName{ + Namespace: svc.ObjectMeta.GetNamespace(), + Name: svc.ObjectMeta.GetName(), + } + if err := s.client.Get(context.Background(), nsNameService, svc); err != nil { + return err + } + + apiAddress := fmt.Sprintf("tcp://%s:5705", svc.Spec.ClusterIP) + secret.Data[apiAddressKey] = []byte(apiAddress) + + if err := s.client.Update(context.Background(), secret); err != nil { + return err + } + } + + return nil +} + +func (s *Deployment) deleteService(name string) error { + return s.deleteObject(s.getService(name)) +} + +func (s *Deployment) getService(name string) *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + Annotations: s.stos.Spec.Service.Annotations, + }, + } +} diff --git a/pkg/storageos/statefulset.go b/pkg/storageos/statefulset.go new file mode 100644 index 000000000..201aa1475 --- /dev/null +++ b/pkg/storageos/statefulset.go @@ -0,0 +1,164 @@ +package storageos + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (s *Deployment) createStatefulSet() error { + ls := labelsForStatefulSet(s.stos.Name) + replicas := int32(1) + hostpathDirOrCreate := corev1.HostPathDirectoryOrCreate + + sset := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: statefulsetName, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": "storageos", + }, + }, + Spec: appsv1.StatefulSetSpec{ + ServiceName: "storageos", + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: ls, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: ls, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "storageos-statefulset-sa", + Containers: []corev1.Container{ + { + Image: s.stos.Spec.GetCSIExternalProvisionerImage(CSIV1Supported(s.k8sVersion)), + Name: "csi-external-provisioner", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--v=5", + "--provisioner=storageos", + "--csi-address=$(ADDRESS)", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + }, + }, + { + Image: s.stos.Spec.GetCSIExternalAttacherImage(CSIV1Supported(s.k8sVersion)), + Name: "csi-external-attacher", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--v=5", + "--csi-address=$(ADDRESS)", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "plugin-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), + Type: &hostpathDirOrCreate, + }, + }, + }, + }, + }, + }, + }, + } + + if CSIV1Supported(s.k8sVersion) { + driverReg := corev1.Container{ + Image: s.stos.Spec.GetCSIClusterDriverRegistrarImage(), + Name: "csi-driver-k8s-registrar", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--v=5", + "--csi-address=$(ADDRESS)", + "--pod-info-mount-version=v1", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + { + Name: kubeNodeNameEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + }, + } + + sset.Spec.Template.Spec.Containers = append(sset.Spec.Template.Spec.Containers, driverReg) + } + + podSpec := &sset.Spec.Template.Spec + + s.addNodeAffinity(podSpec) + + if err := s.addTolerations(podSpec); err != nil { + return err + } + + return s.createOrUpdateObject(sset) +} + +func (s *Deployment) deleteStatefulSet(name string) error { + return s.deleteObject(s.getStatefulSet(name)) +} + +func (s *Deployment) getStatefulSet(name string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": "storageos", + }, + }, + } +} diff --git a/pkg/storageos/storageclass.go b/pkg/storageos/storageclass.go new file mode 100644 index 000000000..551e60076 --- /dev/null +++ b/pkg/storageos/storageclass.go @@ -0,0 +1,81 @@ +package storageos + +import ( + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (s *Deployment) createStorageClass() error { + // Provisioner name for in-tree storage plugin. + provisioner := intreeProvisionerName + + if s.stos.Spec.CSI.Enable { + provisioner = csiProvisionerName + } + + sc := s.getStorageClass("fast") + sc.Provisioner = provisioner + sc.Parameters = map[string]string{ + "pool": "default", + } + + if s.stos.Spec.CSI.Enable { + // Add CSI creds secrets in parameters. + if CSIV1Supported(s.k8sVersion) { + // New CSI secret parameter keys were introduced in CSI v1. + sc.Parameters[csiV1FSType] = defaultFSType + if s.stos.Spec.CSI.EnableProvisionCreds { + sc.Parameters[csiV1ProvisionerSecretNameKey] = csiProvisionerSecretName + sc.Parameters[csiV1ProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + if s.stos.Spec.CSI.EnableControllerPublishCreds { + sc.Parameters[csiV1ControllerPublishSecretNameKey] = csiControllerPublishSecretName + sc.Parameters[csiV1ControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + if s.stos.Spec.CSI.EnableNodePublishCreds { + sc.Parameters[csiV1NodePublishSecretNameKey] = csiNodePublishSecretName + sc.Parameters[csiV1NodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + } else { + sc.Parameters[fsType] = defaultFSType + if s.stos.Spec.CSI.EnableProvisionCreds { + sc.Parameters[csiV0ProvisionerSecretNameKey] = csiProvisionerSecretName + sc.Parameters[csiV0ProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + if s.stos.Spec.CSI.EnableControllerPublishCreds { + sc.Parameters[csiV0ControllerPublishSecretNameKey] = csiControllerPublishSecretName + sc.Parameters[csiV0ControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + if s.stos.Spec.CSI.EnableNodePublishCreds { + sc.Parameters[csiV0NodePublishSecretNameKey] = csiNodePublishSecretName + sc.Parameters[csiV0NodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + } + } else { + sc.Parameters[fsType] = defaultFSType + // Add StorageOS admin secrets name and namespace. + sc.Parameters[secretNamespaceKey] = s.stos.Spec.SecretRefNamespace + sc.Parameters[secretNameKey] = s.stos.Spec.SecretRefName + } + + return s.createOrUpdateObject(sc) +} + +func (s *Deployment) deleteStorageClass(name string) error { + return s.deleteObject(s.getStorageClass(name)) +} + +func (s *Deployment) getStorageClass(name string) *storagev1.StorageClass { + return &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "storage.k8s.io/v1", + Kind: "StorageClass", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "app": appName, + }, + }, + } +} From 3796b20715e0383d16aa46f3e877f43e232b0c64 Mon Sep 17 00:00:00 2001 From: Sunny Date: Mon, 18 Mar 2019 15:19:13 +0530 Subject: [PATCH 3/3] cluster-ctrl: Define Deployment interface Deployment interface defines a cluster deployment for StorageOSCluster. --- pkg/controller/storageoscluster/cluster.go | 8 ++++---- pkg/controller/storageoscluster/deployment.go | 9 +++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 pkg/controller/storageoscluster/deployment.go diff --git a/pkg/controller/storageoscluster/cluster.go b/pkg/controller/storageoscluster/cluster.go index ed4941c91..bdc1fd9e2 100644 --- a/pkg/controller/storageoscluster/cluster.go +++ b/pkg/controller/storageoscluster/cluster.go @@ -10,10 +10,10 @@ import ( // right cluster resource. type StorageOSCluster struct { cluster *storageosv1alpha1.StorageOSCluster - // deployment is the storageos.Deployment object. This is cached for a - // cluster to avoid recreating it without any change to the cluster object. - // Every new cluster will create their unique deployment. - deployment *storageos.Deployment + // deployment implements storageoscluster.Deployment interface. This is + // cached for a cluster to avoid recreating it without any change to the + // cluster object. Every new cluster will create its unique deployment. + deployment Deployment } // NewStorageOSCluster creates a new StorageOSCluster object. diff --git a/pkg/controller/storageoscluster/deployment.go b/pkg/controller/storageoscluster/deployment.go new file mode 100644 index 000000000..eb64145c5 --- /dev/null +++ b/pkg/controller/storageoscluster/deployment.go @@ -0,0 +1,9 @@ +package storageoscluster + +// Deployment is an interface for deployment of a cluster. +type Deployment interface { + // Deploy deploys a cluster. + Deploy() error + // Delete deletes a deployed cluster. + Delete() error +}