diff --git a/Makefile b/Makefile index f376d55ef..0ab6bd9ba 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ lint: go vet ./... unittest: - go test -v `go list -v ./... | grep -v test/e2e` + go test -v -race `go list -v ./... | grep -v test/e2e | grep -v test/olm` operator-sdk: # Download sdk only if it's not available. diff --git a/pkg/controller/storageoscluster/cluster.go b/pkg/controller/storageoscluster/cluster.go new file mode 100644 index 000000000..bdc1fd9e2 --- /dev/null +++ b/pkg/controller/storageoscluster/cluster.go @@ -0,0 +1,56 @@ +package storageoscluster + +import ( + storageosv1alpha1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1alpha1" + "github.com/storageos/cluster-operator/pkg/storageos" +) + +// StorageOSCluster stores the current cluster's information. It binds the +// cluster and the deployment together, ensuring deployment interacts with the +// right cluster resource. +type StorageOSCluster struct { + cluster *storageosv1alpha1.StorageOSCluster + // deployment implements storageoscluster.Deployment interface. This is + // cached for a cluster to avoid recreating it without any change to the + // cluster object. Every new cluster will create its unique deployment. + deployment Deployment +} + +// NewStorageOSCluster creates a new StorageOSCluster object. +func NewStorageOSCluster(cluster *storageosv1alpha1.StorageOSCluster) *StorageOSCluster { + return &StorageOSCluster{cluster: cluster} +} + +// SetDeployment creates a new StorageOS Deployment and sets it for the current +// StorageOSCluster. +func (c *StorageOSCluster) SetDeployment(r *ReconcileStorageOSCluster) { + // updateIfExists is set to false because we don't update any existing + // resources for now. May change in future. + // TODO: Change this after resolving the conflict between two way + // binding and upgrade. + updateIfExists := false + c.deployment = storageos.NewDeployment(r.client, c.cluster, r.recorder, r.scheme, r.k8sVersion, updateIfExists) +} + +// IsCurrentCluster compares the cluster attributes to check if the given +// cluster is the same as the current cluster. +func (c *StorageOSCluster) IsCurrentCluster(cluster *storageosv1alpha1.StorageOSCluster) bool { + if (c.cluster.GetName() == cluster.GetName()) && + (c.cluster.GetNamespace() == cluster.GetNamespace()) { + return true + } + return false +} + +// Deploy deploys the StorageOS cluster. +func (c *StorageOSCluster) Deploy(r *ReconcileStorageOSCluster) error { + if c.deployment == nil { + c.SetDeployment(r) + } + return c.deployment.Deploy() +} + +// DeleteDeployment deletes the StorageOS Cluster deployment. +func (c *StorageOSCluster) DeleteDeployment() error { + return c.deployment.Delete() +} diff --git a/pkg/controller/storageoscluster/deployment.go b/pkg/controller/storageoscluster/deployment.go new file mode 100644 index 000000000..eb64145c5 --- /dev/null +++ b/pkg/controller/storageoscluster/deployment.go @@ -0,0 +1,9 @@ +package storageoscluster + +// Deployment is an interface for deployment of a cluster. +type Deployment interface { + // Deploy deploys a cluster. + Deploy() error + // Delete deletes a deployed cluster. + Delete() error +} diff --git a/pkg/controller/storageoscluster/storageoscluster_controller.go b/pkg/controller/storageoscluster/storageoscluster_controller.go index 9072658ad..c2925fc38 100644 --- a/pkg/controller/storageoscluster/storageoscluster_controller.go +++ b/pkg/controller/storageoscluster/storageoscluster_controller.go @@ -72,10 +72,9 @@ type ReconcileStorageOSCluster struct { // that reads objects from the cache and writes to the apiserver client client.Client scheme *runtime.Scheme - currentCluster *storageosv1alpha1.StorageOSCluster k8sVersion string recorder record.EventRecorder - deployment *storageos.Deployment + currentCluster *StorageOSCluster } // SetCurrentClusterIfNone checks if there's any existing current cluster and @@ -88,28 +87,12 @@ func (r *ReconcileStorageOSCluster) SetCurrentClusterIfNone(cluster *storageosv1 // SetCurrentCluster sets the currently active cluster in the controller. func (r *ReconcileStorageOSCluster) SetCurrentCluster(cluster *storageosv1alpha1.StorageOSCluster) { - r.currentCluster = cluster -} - -// IsCurrentCluster compares a given cluster with the current cluster to check -// if they are the same. -func (r *ReconcileStorageOSCluster) IsCurrentCluster(cluster *storageosv1alpha1.StorageOSCluster) bool { - if cluster == nil { - return false - } - - if (r.currentCluster.GetName() == cluster.GetName()) && (r.currentCluster.GetNamespace() == cluster.GetNamespace()) { - return true - } - return false + r.currentCluster = NewStorageOSCluster(cluster) } // ResetCurrentCluster resets the current cluster of the controller. func (r *ReconcileStorageOSCluster) ResetCurrentCluster() { r.currentCluster = nil - // Reset deployment as well. Deployments are specific to the current - // cluster. - r.deployment = nil } // Reconcile reads that state of the cluster for a StorageOSCluster object and makes changes based on the state read @@ -127,9 +110,10 @@ func (r *ReconcileStorageOSCluster) Reconcile(request reconcile.Request) (reconc err := r.client.Get(context.TODO(), request.NamespacedName, instance) if err != nil { if errors.IsNotFound(err) { - // Cluster instance not found. Delete the resources and reset the current cluster. + // Cluster instance not found. Delete the resources and reset the + // current cluster. if r.currentCluster != nil { - if err := getDeployment(r).Delete(); err != nil { + if err := r.currentCluster.DeleteDeployment(); err != nil { // Error deleting - requeue the request. return reconcileResult, err } @@ -149,10 +133,18 @@ func (r *ReconcileStorageOSCluster) Reconcile(request reconcile.Request) (reconc // If the event doesn't belongs to the current cluster, do not reconcile. // There must be only a single instance of storageos in a cluster. - if !r.IsCurrentCluster(instance) { + if !r.currentCluster.IsCurrentCluster(instance) { err := fmt.Errorf("can't create more than one storageos cluster") r.recorder.Event(instance, corev1.EventTypeWarning, "FailedCreation", err.Error()) return reconcileResult, err + } else if r.currentCluster.cluster.GetUID() != instance.GetUID() { + // If the cluster name and namespace match with the current cluster, but + // the resource UIDs are different, maybe the current cluster reset + // failed when the previous cluster was deleted. The same cluster was + // created again and has a different UID. Create and assign a new + // current cluster. + log.Printf("replacing current cluster UID: %s with new cluster UID: %s", r.currentCluster.cluster.GetUID(), instance.GetUID()) + r.SetCurrentCluster(instance) } if err := r.reconcile(instance); err != nil { @@ -163,8 +155,8 @@ func (r *ReconcileStorageOSCluster) Reconcile(request reconcile.Request) (reconc } func (r *ReconcileStorageOSCluster) reconcile(m *storageosv1alpha1.StorageOSCluster) error { - // Do not reconcile, the operator is paused for the cluster. if m.Spec.Pause { + // Do not reconcile, the operator is paused for the cluster. return nil } @@ -226,7 +218,7 @@ func (r *ReconcileStorageOSCluster) reconcile(m *storageosv1alpha1.StorageOSClus // r.SetCurrentCluster(m) // } - if err := getDeployment(r).Deploy(); err != nil { + if err := r.currentCluster.Deploy(r); err != nil { // Ignore "Operation cannot be fulfilled" error. It happens when the // actual state of object is different from what is known to the operator. // Operator would resync and retry the failed operation on its own. @@ -240,7 +232,7 @@ func (r *ReconcileStorageOSCluster) reconcile(m *storageosv1alpha1.StorageOSClus // resource. r.recorder.Event(m, corev1.EventTypeNormal, "Terminating", "Deleting all the resources...") - if err := getDeployment(r).Delete(); err != nil { + if err := r.currentCluster.DeleteDeployment(); err != nil { return err } @@ -338,20 +330,3 @@ func getMatchingTolerations(taints []corev1.Taint, tolerations []corev1.Tolerati } return true, result } - -// getDeployment returns an existing storageos deployment if found, else creates -// a new deployment object, stores in the reconcile object and returns the same. -func getDeployment(r *ReconcileStorageOSCluster) *storageos.Deployment { - if r.deployment != nil { - return r.deployment - } - - // update is set to false because we don't update any existing resources - // for now. May change in future. - // TODO: Change this after resolving the conflict between two way - // binding and upgrade. - updateIfExists := false - - r.deployment = storageos.NewDeployment(r.client, r.currentCluster, r.recorder, r.scheme, r.k8sVersion, updateIfExists) - return r.deployment -} diff --git a/pkg/storageos/daemonset.go b/pkg/storageos/daemonset.go new file mode 100644 index 000000000..633724988 --- /dev/null +++ b/pkg/storageos/daemonset.go @@ -0,0 +1,257 @@ +package storageos + +import ( + "strconv" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func (s *Deployment) createDaemonSet() error { + ls := labelsForDaemonSet(s.stos.Name) + privileged := true + mountPropagationBidirectional := corev1.MountPropagationBidirectional + allowPrivilegeEscalation := true + + dset := &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: daemonsetName, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": "storageos", + }, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: ls, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: ls, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "storageos-daemonset-sa", + HostPID: true, + HostNetwork: true, + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + InitContainers: []corev1.Container{ + { + Name: "enable-lio", + Image: s.stos.Spec.GetInitContainerImage(), + VolumeMounts: []corev1.VolumeMount{ + { + Name: "kernel-modules", + MountPath: "/lib/modules", + ReadOnly: true, + }, + { + Name: "sys", + MountPath: "/sys", + MountPropagation: &mountPropagationBidirectional, + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"SYS_ADMIN"}, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Image: s.stos.Spec.GetNodeContainerImage(), + Name: "storageos", + Args: []string{"server"}, + Ports: []corev1.ContainerPort{{ + ContainerPort: 5705, + Name: "api", + }}, + LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: int32(65), + TimeoutSeconds: int32(10), + FailureThreshold: int32(5), + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/health", + Port: intstr.IntOrString{Type: intstr.String, StrVal: "api"}, + }, + }, + }, + ReadinessProbe: &corev1.Probe{ + InitialDelaySeconds: int32(65), + TimeoutSeconds: int32(10), + FailureThreshold: int32(5), + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v1/health", + Port: intstr.IntOrString{Type: intstr.String, StrVal: "api"}, + }, + }, + }, + Env: []corev1.EnvVar{ + { + Name: hostnameEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: adminUsernameEnvVar, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: initSecretName, + }, + Key: "username", + }, + }, + }, + { + Name: adminPasswordEnvVar, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: initSecretName, + }, + Key: "password", + }, + }, + }, + { + Name: joinEnvVar, + Value: s.stos.Spec.Join, + }, + { + Name: advertiseIPEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + Name: namespaceEnvVar, + Value: s.stos.Spec.GetResourceNS(), + }, + { + Name: disableTelemetryEnvVar, + Value: strconv.FormatBool(s.stos.Spec.DisableTelemetry), + }, + { + Name: csiVersionEnvVar, + Value: s.stos.Spec.GetCSIVersion(CSIV1Supported(s.k8sVersion)), + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{sysAdminCap}, + }, + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "fuse", + MountPath: "/dev/fuse", + }, + { + Name: "sys", + MountPath: "/sys", + }, + { + Name: "state", + MountPath: "/var/lib/storageos", + MountPropagation: &mountPropagationBidirectional, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "kernel-modules", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/lib/modules", + }, + }, + }, + { + Name: "fuse", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/fuse", + }, + }, + }, + { + Name: "sys", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/sys", + }, + }, + }, + { + Name: "state", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/lib/storageos", + }, + }, + }, + }, + }, + }, + }, + } + + podSpec := &dset.Spec.Template.Spec + nodeContainer := &podSpec.Containers[0] + + s.addNodeAffinity(podSpec) + + if err := s.addTolerations(podSpec); err != nil { + return err + } + + nodeContainer.Env = s.addKVBackendEnvVars(nodeContainer.Env) + + nodeContainer.Env = s.addDebugEnvVars(nodeContainer.Env) + + s.addNodeContainerResources(nodeContainer) + + s.addSharedDir(podSpec) + + s.addCSI(podSpec) + + return s.createOrUpdateObject(dset) +} + +func (s *Deployment) deleteDaemonSet(name string) error { + return s.deleteObject(s.getDaemonSet(name)) +} + +func (s *Deployment) getDaemonSet(name string) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": "storageos", + }, + }, + } +} diff --git a/pkg/storageos/delete.go b/pkg/storageos/delete.go new file mode 100644 index 000000000..5089643a5 --- /dev/null +++ b/pkg/storageos/delete.go @@ -0,0 +1,85 @@ +package storageos + +// Delete deletes all the storageos resources. +// This explicit delete is implemented instead of depending on the garbage +// collector because sometimes the garbage collector deletes the resources +// with owner reference as a CRD without the parent being deleted. This happens +// especially when a cluster reboots. Althrough the operator re-creates the +// resources, we want to avoid this behavior by implementing an explcit delete. +func (s *Deployment) Delete() error { + + if err := s.deleteStorageClass("fast"); err != nil { + return err + } + + if err := s.deleteService(s.stos.Spec.GetServiceName()); err != nil { + return err + } + + if err := s.deleteDaemonSet(daemonsetName); err != nil { + return err + } + + if err := s.deleteSecret(initSecretName); err != nil { + return err + } + + if err := s.deleteRoleBinding(keyManagementBindingName); err != nil { + return err + } + + if err := s.deleteRole(keyManagementRoleName); err != nil { + return err + } + + if err := s.deleteServiceAccount("storageos-daemonset-sa"); err != nil { + return err + } + + if s.stos.Spec.CSI.Enable { + if err := s.deleteStatefulSet(statefulsetName); err != nil { + return err + } + + if err := s.deleteClusterRoleBinding("csi-attacher-binding"); err != nil { + return err + } + + if err := s.deleteClusterRoleBinding("csi-provisioner-binding"); err != nil { + return err + } + + if err := s.deleteClusterRole("csi-attacher-role"); err != nil { + return err + } + + if err := s.deleteClusterRole("csi-provisioner-role"); err != nil { + return err + } + + if err := s.deleteServiceAccount("storageos-statefulset-sa"); err != nil { + return err + } + + if err := s.deleteClusterRoleBinding("k8s-driver-registrar-binding"); err != nil { + return err + } + + if err := s.deleteClusterRoleBinding("driver-registrar-binding"); err != nil { + return err + } + + if err := s.deleteClusterRole("driver-registrar-role"); err != nil { + return err + } + + if err := s.deleteCSISecrets(); err != nil { + return err + } + } + + // NOTE: Do not delete the namespace. Namespace can have some resources + // created by the control plane. They must not be deleted. + + return nil +} diff --git a/pkg/storageos/deploy.go b/pkg/storageos/deploy.go index 40bb0b6c5..cfa752bb6 100644 --- a/pkg/storageos/deploy.go +++ b/pkg/storageos/deploy.go @@ -4,23 +4,13 @@ import ( "context" "fmt" "log" - "strconv" "github.com/blang/semver" api "github.com/storageos/cluster-operator/pkg/apis/storageos/v1alpha1" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/api/extensions/v1beta1" - rbacv1 "k8s.io/api/rbac/v1" - storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/tools/record" - - "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -110,30 +100,6 @@ const ( defaultPassword = "storageos" ) -// Deployment stores all the resource configuration and performs -// resource creation and update. -type Deployment struct { - client client.Client - stos *api.StorageOSCluster - recorder record.EventRecorder - k8sVersion string - scheme *runtime.Scheme - update bool -} - -// NewDeployment creates a new Deployment given a k8c client, storageos manifest -// and an event broadcast recorder. -func NewDeployment(client client.Client, stos *api.StorageOSCluster, recorder record.EventRecorder, scheme *runtime.Scheme, version string, update bool) *Deployment { - return &Deployment{ - client: client, - stos: stos, - recorder: recorder, - k8sVersion: version, - scheme: scheme, - update: update, - } -} - // Deploy deploys storageos by creating all the resources needed to run storageos. func (s *Deployment) Deploy() error { if err := s.createNamespace(); err != nil { @@ -230,90 +196,6 @@ func (s *Deployment) Deploy() error { return s.updateStorageOSStatus(status) } -// Delete deletes all the storageos resources. -// This explicit delete is implemented instead of depending on the garbage -// collector because sometimes the garbage collector deletes the resources -// with owner reference as a CRD without the parent being deleted. This happens -// especially when a cluster reboots. Althrough the operator re-creates the -// resources, we want to avoid this behavior by implementing an explcit delete. -func (s *Deployment) Delete() error { - - if err := s.deleteStorageClass("fast"); err != nil { - return err - } - - if err := s.deleteService(s.stos.Spec.GetServiceName()); err != nil { - return err - } - - if err := s.deleteDaemonSet(daemonsetName); err != nil { - return err - } - - if err := s.deleteSecret(initSecretName); err != nil { - return err - } - - if err := s.deleteRoleBinding(keyManagementBindingName); err != nil { - return err - } - - if err := s.deleteRole(keyManagementRoleName); err != nil { - return err - } - - if err := s.deleteServiceAccount("storageos-daemonset-sa"); err != nil { - return err - } - - if s.stos.Spec.CSI.Enable { - if err := s.deleteStatefulSet(statefulsetName); err != nil { - return err - } - - if err := s.deleteClusterRoleBinding("csi-attacher-binding"); err != nil { - return err - } - - if err := s.deleteClusterRoleBinding("csi-provisioner-binding"); err != nil { - return err - } - - if err := s.deleteClusterRole("csi-attacher-role"); err != nil { - return err - } - - if err := s.deleteClusterRole("csi-provisioner-role"); err != nil { - return err - } - - if err := s.deleteServiceAccount("storageos-statefulset-sa"); err != nil { - return err - } - - if err := s.deleteClusterRoleBinding("k8s-driver-registrar-binding"); err != nil { - return err - } - - if err := s.deleteClusterRoleBinding("driver-registrar-binding"); err != nil { - return err - } - - if err := s.deleteClusterRole("driver-registrar-role"); err != nil { - return err - } - - if err := s.deleteCSISecrets(); err != nil { - return err - } - } - - // NOTE: Do not delete the namespace. Namespace can have some resources - // created by the control plane. They must not be deleted. - - return nil -} - func (s *Deployment) createNamespace() error { ns := &corev1.Namespace{ TypeMeta: metav1.TypeMeta{ @@ -331,572 +213,6 @@ func (s *Deployment) createNamespace() error { return s.createOrUpdateObject(ns) } -func (s *Deployment) createServiceAccount(name string) error { - sa := s.getServiceAccount(name) - return s.createOrUpdateObject(sa) -} - -func (s *Deployment) deleteServiceAccount(name string) error { - return s.deleteObject(s.getServiceAccount(name)) -} - -// getServiceAccount creates a generic service account object with the given -// name and returns it. -func (s *Deployment) getServiceAccount(name string) *corev1.ServiceAccount { - return &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ServiceAccount", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createServiceAccountForDaemonSet() error { - return s.createServiceAccount("storageos-daemonset-sa") -} - -func (s *Deployment) createServiceAccountForStatefulSet() error { - return s.createServiceAccount("storageos-statefulset-sa") -} - -func (s *Deployment) createRoleForKeyMgmt() error { - role := s.getRole(keyManagementRoleName) - role.Rules = []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "create", "delete"}, - }, - } - - return s.createOrUpdateObject(role) -} - -func (s *Deployment) deleteRole(name string) error { - return s.deleteObject(s.getRole(keyManagementRoleName)) -} - -// getRole creates a generic role object with the given name and returns it. -func (s *Deployment) getRole(name string) *rbacv1.Role { - return &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "Role", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createClusterRole(name string, rules []rbacv1.PolicyRule) error { - role := s.getClusterRole(name) - role.Rules = rules - return s.createOrUpdateObject(role) -} - -func (s *Deployment) deleteClusterRole(name string) error { - return s.deleteObject(s.getClusterRole(name)) -} - -func (s *Deployment) getClusterRole(name string) *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "ClusterRole", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createClusterRoleForDriverRegistrar() error { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list", "watch", "create", "update", "patch"}, - }, - { - APIGroups: []string{"apiextensions.k8s.io"}, - Resources: []string{"customresourcedefinitions"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"csi.storage.k8s.io"}, - Resources: []string{"csidrivers"}, - Verbs: []string{"create"}, - }, - } - return s.createClusterRole("driver-registrar-role", rules) -} - -func (s *Deployment) createClusterRoleForProvisioner() error { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"persistentvolumes"}, - Verbs: []string{"list", "watch", "create", "delete"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"persistentvolumeclaims"}, - Verbs: []string{"get", "list", "watch", "update"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - Verbs: []string{"list", "watch", "get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list", "watch", "create", "update", "patch"}, - }, - } - return s.createClusterRole("csi-provisioner-role", rules) -} - -func (s *Deployment) createClusterRoleForAttacher() error { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"persistentvolumes"}, - Verbs: []string{"get", "list", "watch", "update"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - Verbs: []string{"list", "watch", "get"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"volumeattachments"}, - Verbs: []string{"get", "list", "watch", "update"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"csinodeinfos"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list", "watch", "create", "update", "patch"}, - }, - } - return s.createClusterRole("csi-attacher-role", rules) -} - -func (s *Deployment) createRoleBindingForKeyMgmt() error { - roleBinding := s.getRoleBinding(keyManagementBindingName) - roleBinding.Subjects = []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "storageos-daemonset-sa", - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleBinding.RoleRef = rbacv1.RoleRef{ - Kind: "Role", - Name: keyManagementRoleName, - APIGroup: "rbac.authorization.k8s.io", - } - return s.createOrUpdateObject(roleBinding) -} - -func (s *Deployment) deleteRoleBinding(name string) error { - return s.deleteObject(s.getRoleBinding(name)) -} - -func (s *Deployment) getRoleBinding(name string) *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "RoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createClusterRoleBinding(name string, subjects []rbacv1.Subject, roleRef rbacv1.RoleRef) error { - roleBinding := s.getClusterRoleBinding(name) - roleBinding.Subjects = subjects - roleBinding.RoleRef = roleRef - return s.createOrUpdateObject(roleBinding) -} - -func (s *Deployment) deleteClusterRoleBinding(name string) error { - return s.deleteObject(s.getClusterRoleBinding(name)) -} - -func (s *Deployment) getClusterRoleBinding(name string) *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "ClusterRoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createClusterRoleBindingForDriverRegistrar() error { - subjects := []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "storageos-daemonset-sa", - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleRef := rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "driver-registrar-role", - APIGroup: "rbac.authorization.k8s.io", - } - return s.createClusterRoleBinding("driver-registrar-binding", subjects, roleRef) -} - -func (s *Deployment) createClusterRoleBindingForK8SDriverRegistrar() error { - subjects := []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "storageos-statefulset-sa", - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleRef := rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "driver-registrar-role", - APIGroup: "rbac.authorization.k8s.io", - } - return s.createClusterRoleBinding("k8s-driver-registrar-binding", subjects, roleRef) -} - -func (s *Deployment) createClusterRoleBindingForProvisioner() error { - subjects := []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "storageos-statefulset-sa", - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleRef := rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "csi-provisioner-role", - APIGroup: "rbac.authorization.k8s.io", - } - return s.createClusterRoleBinding("csi-provisioner-binding", subjects, roleRef) -} - -func (s *Deployment) createClusterRoleBindingForAttacher() error { - subjects := []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "storageos-statefulset-sa", - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleRef := rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "csi-attacher-role", - APIGroup: "rbac.authorization.k8s.io", - } - return s.createClusterRoleBinding("csi-attacher-binding", subjects, roleRef) -} - -func (s *Deployment) createDaemonSet() error { - ls := labelsForDaemonSet(s.stos.Name) - privileged := true - mountPropagationBidirectional := corev1.MountPropagationBidirectional - allowPrivilegeEscalation := true - - dset := &appsv1.DaemonSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: daemonsetName, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": "storageos", - }, - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: ls, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: ls, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: "storageos-daemonset-sa", - HostPID: true, - HostNetwork: true, - DNSPolicy: corev1.DNSClusterFirstWithHostNet, - InitContainers: []corev1.Container{ - { - Name: "enable-lio", - Image: s.stos.Spec.GetInitContainerImage(), - VolumeMounts: []corev1.VolumeMount{ - { - Name: "kernel-modules", - MountPath: "/lib/modules", - ReadOnly: true, - }, - { - Name: "sys", - MountPath: "/sys", - MountPropagation: &mountPropagationBidirectional, - }, - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: &privileged, - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"SYS_ADMIN"}, - }, - }, - }, - }, - Containers: []corev1.Container{ - { - Image: s.stos.Spec.GetNodeContainerImage(), - Name: "storageos", - Args: []string{"server"}, - Ports: []corev1.ContainerPort{{ - ContainerPort: 5705, - Name: "api", - }}, - LivenessProbe: &corev1.Probe{ - InitialDelaySeconds: int32(65), - TimeoutSeconds: int32(10), - FailureThreshold: int32(5), - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/v1/health", - Port: intstr.IntOrString{Type: intstr.String, StrVal: "api"}, - }, - }, - }, - ReadinessProbe: &corev1.Probe{ - InitialDelaySeconds: int32(65), - TimeoutSeconds: int32(10), - FailureThreshold: int32(5), - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/v1/health", - Port: intstr.IntOrString{Type: intstr.String, StrVal: "api"}, - }, - }, - }, - Env: []corev1.EnvVar{ - { - Name: hostnameEnvVar, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - { - Name: adminUsernameEnvVar, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: initSecretName, - }, - Key: "username", - }, - }, - }, - { - Name: adminPasswordEnvVar, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: initSecretName, - }, - Key: "password", - }, - }, - }, - { - Name: joinEnvVar, - Value: s.stos.Spec.Join, - // ValueFrom: &corev1.EnvVarSource{ - // FieldRef: &corev1.ObjectFieldSelector{ - // FieldPath: "status.podIP", - // }, - // }, - }, - { - Name: advertiseIPEnvVar, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }, - { - Name: namespaceEnvVar, - Value: s.stos.Spec.GetResourceNS(), - }, - { - Name: disableTelemetryEnvVar, - Value: strconv.FormatBool(s.stos.Spec.DisableTelemetry), - }, - { - Name: csiVersionEnvVar, - Value: s.stos.Spec.GetCSIVersion(CSIV1Supported(s.k8sVersion)), - }, - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: &privileged, - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{sysAdminCap}, - }, - AllowPrivilegeEscalation: &allowPrivilegeEscalation, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "fuse", - MountPath: "/dev/fuse", - }, - { - Name: "sys", - MountPath: "/sys", - }, - { - Name: "state", - MountPath: "/var/lib/storageos", - MountPropagation: &mountPropagationBidirectional, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "kernel-modules", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/lib/modules", - }, - }, - }, - { - Name: "fuse", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/dev/fuse", - }, - }, - }, - { - Name: "sys", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/sys", - }, - }, - }, - { - Name: "state", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/storageos", - }, - }, - }, - }, - }, - }, - }, - } - - podSpec := &dset.Spec.Template.Spec - nodeContainer := &podSpec.Containers[0] - - s.addNodeAffinity(podSpec) - - if err := s.addTolerations(podSpec); err != nil { - return err - } - - nodeContainer.Env = s.addKVBackendEnvVars(nodeContainer.Env) - - nodeContainer.Env = s.addDebugEnvVars(nodeContainer.Env) - - s.addNodeContainerResources(nodeContainer) - - s.addSharedDir(podSpec) - - s.addCSI(podSpec) - - return s.createOrUpdateObject(dset) -} - -func (s *Deployment) deleteDaemonSet(name string) error { - return s.deleteObject(s.getDaemonSet(name)) -} - -func (s *Deployment) getDaemonSet(name string) *appsv1.DaemonSet { - return &appsv1.DaemonSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": "storageos", - }, - }, - } -} - // addNodeContainerResources adds resource requirements for the node containers. func (s *Deployment) addNodeContainerResources(nodeContainer *corev1.Container) { if s.stos.Spec.Resources.Limits != nil || @@ -978,286 +294,15 @@ func (s *Deployment) addDebugEnvVars(env []corev1.EnvVar) []corev1.EnvVar { return env } -// addSharedDir adds env var and volumes for shared dir when running kubelet in -// a container. -func (s *Deployment) addSharedDir(podSpec *corev1.PodSpec) { - mountPropagationBidirectional := corev1.MountPropagationBidirectional - nodeContainer := &podSpec.Containers[0] - - // If kubelet is running in a container, sharedDir should be set. - if s.stos.Spec.SharedDir != "" { - envVar := corev1.EnvVar{ - Name: deviceDirEnvVar, - Value: fmt.Sprintf("%s/devices", s.stos.Spec.SharedDir), - } - nodeContainer.Env = append(nodeContainer.Env, envVar) - - sharedDir := corev1.Volume{ - Name: "shared", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.SharedDir, - }, - }, - } - podSpec.Volumes = append(podSpec.Volumes, sharedDir) - - volMnt := corev1.VolumeMount{ - Name: "shared", - MountPath: s.stos.Spec.SharedDir, - MountPropagation: &mountPropagationBidirectional, - } - nodeContainer.VolumeMounts = append(nodeContainer.VolumeMounts, volMnt) - } -} - -// addCSI adds the CSI env vars, volumes and containers to the provided podSpec. -func (s *Deployment) addCSI(podSpec *corev1.PodSpec) { - hostpathDirOrCreate := corev1.HostPathDirectoryOrCreate - hostpathDir := corev1.HostPathDirectory - mountPropagationBidirectional := corev1.MountPropagationBidirectional - - nodeContainer := &podSpec.Containers[0] - - // Add CSI specific configurations if enabled. - if s.stos.Spec.CSI.Enable { - vols := []corev1.Volume{ - { - Name: "registrar-socket-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIRegistrarSocketDir(), - Type: &hostpathDirOrCreate, - }, - }, - }, - { - Name: "kubelet-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIKubeletDir(), - Type: &hostpathDir, - }, - }, - }, - { - Name: "plugin-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), - Type: &hostpathDirOrCreate, - }, - }, - }, - { - Name: "device-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIDeviceDir(), - Type: &hostpathDir, - }, - }, - }, - { - Name: "registration-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIRegistrationDir(CSIV1Supported(s.k8sVersion)), - Type: &hostpathDir, - }, - }, - }, - } - - podSpec.Volumes = append(podSpec.Volumes, vols...) - - volMnts := []corev1.VolumeMount{ - { - Name: "kubelet-dir", - MountPath: s.stos.Spec.GetCSIKubeletDir(), - MountPropagation: &mountPropagationBidirectional, - }, - { - Name: "plugin-dir", - MountPath: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), - }, - { - Name: "device-dir", - MountPath: s.stos.Spec.GetCSIDeviceDir(), - }, - } - - // Append volume mounts to the first container, the only container is the node container, at this point. - nodeContainer.VolumeMounts = append(nodeContainer.VolumeMounts, volMnts...) - - envVar := []corev1.EnvVar{ - { - Name: csiEndpointEnvVar, - Value: s.stos.Spec.GetCSIEndpoint(CSIV1Supported(s.k8sVersion)), - }, - } - - // Append CSI Provision Creds env var if enabled. - if s.stos.Spec.CSI.EnableProvisionCreds { - envVar = append( - envVar, - corev1.EnvVar{ - Name: csiRequireCredsCreateEnvVar, - Value: "true", - }, - corev1.EnvVar{ - Name: csiRequireCredsDeleteEnvVar, - Value: "true", - }, - getCSICredsEnvVar(csiProvisionCredsUsernameEnvVar, csiProvisionerSecretName, "username"), - getCSICredsEnvVar(csiProvisionCredsPasswordEnvVar, csiProvisionerSecretName, "password"), - ) - } - - // Append CSI Controller Publish env var if enabled. - if s.stos.Spec.CSI.EnableControllerPublishCreds { - envVar = append( - envVar, - corev1.EnvVar{ - Name: csiRequireCredsCtrlPubEnvVar, - Value: "true", - }, - corev1.EnvVar{ - Name: csiRequireCredsCtrlUnpubEnvVar, - Value: "true", - }, - getCSICredsEnvVar(csiControllerPubCredsUsernameEnvVar, csiControllerPublishSecretName, "username"), - getCSICredsEnvVar(csiControllerPubCredsPasswordEnvVar, csiControllerPublishSecretName, "password"), - ) - } - - // Append CSI Node Publish env var if enabled. - if s.stos.Spec.CSI.EnableNodePublishCreds { - envVar = append( - envVar, - corev1.EnvVar{ - Name: csiRequireCredsNodePubEnvVar, - Value: "true", - }, - getCSICredsEnvVar(csiNodePubCredsUsernameEnvVar, csiNodePublishSecretName, "username"), - getCSICredsEnvVar(csiNodePubCredsPasswordEnvVar, csiNodePublishSecretName, "password"), - ) - } - - // Append env vars to the first container, node container. - nodeContainer.Env = append(nodeContainer.Env, envVar...) - - driverReg := corev1.Container{ - Image: s.stos.Spec.GetCSINodeDriverRegistrarImage(CSIV1Supported(s.k8sVersion)), - Name: "csi-driver-registrar", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--v=5", - "--csi-address=$(ADDRESS)", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - { - Name: kubeNodeNameEnvVar, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "spec.nodeName", - }, - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - { - Name: "registrar-socket-dir", - MountPath: "/var/lib/csi/sockets/", - }, - { - Name: "registration-dir", - MountPath: "/registration", - }, - }, - } - - // Add extra flags to activate node-register mode if kubelet plugins - // watcher is supported. - if kubeletPluginsWatcherSupported(s.k8sVersion) { - driverReg.Args = append( - driverReg.Args, - fmt.Sprintf("--kubelet-registration-path=%s", s.stos.Spec.GetCSIKubeletRegistrationPath(CSIV1Supported(s.k8sVersion)))) - } - podSpec.Containers = append(podSpec.Containers, driverReg) - - if CSIV1Supported(s.k8sVersion) { - livenessProbe := corev1.Container{ - Image: s.stos.Spec.GetCSILivenessProbeImage(), - Name: "csi-liveness-probe", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--csi-address=$(ADDRESS)", - "--connection-timeout=3s", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - }, - } - podSpec.Containers = append(podSpec.Containers, livenessProbe) - } - } -} - -// addNodeAffinity adds node affinity to the given pod spec from the cluster -// spec NodeSelectorLabel. -func (s *Deployment) addNodeAffinity(podSpec *corev1.PodSpec) { - if len(s.stos.Spec.NodeSelectorTerms) > 0 { - podSpec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: s.stos.Spec.NodeSelectorTerms, - }, - }} - } -} - -// addTolerations adds tolerations to the given pod spec from cluster -// spec Tolerations. -func (s *Deployment) addTolerations(podSpec *corev1.PodSpec) error { - tolerations := s.stos.Spec.Tolerations - for i := range tolerations { - if tolerations[i].Operator == corev1.TolerationOpExists && tolerations[i].Value != "" { - return fmt.Errorf("key(%s): toleration value must be empty when `operator` is 'Exists'", tolerations[i].Key) - } - } - if len(tolerations) > 0 { - podSpec.Tolerations = s.stos.Spec.Tolerations - } - return nil -} - -// getCSICredsEnvVar returns a corev1.EnvVar object with value from a secret key -// reference, given env var name, reference secret name and key in the secret. -func getCSICredsEnvVar(envVarName, secretName, key string) corev1.EnvVar { - return corev1.EnvVar{ - Name: envVarName, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretName, +// getCSICredsEnvVar returns a corev1.EnvVar object with value from a secret key +// reference, given env var name, reference secret name and key in the secret. +func getCSICredsEnvVar(envVarName, secretName, key string) corev1.EnvVar { + return corev1.EnvVar{ + Name: envVarName, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, }, Key: key, }, @@ -1265,594 +310,6 @@ func getCSICredsEnvVar(envVarName, secretName, key string) corev1.EnvVar { } } -func (s *Deployment) createStatefulSet() error { - ls := labelsForStatefulSet(s.stos.Name) - replicas := int32(1) - hostpathDirOrCreate := corev1.HostPathDirectoryOrCreate - - sset := &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: statefulsetName, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": "storageos", - }, - }, - Spec: appsv1.StatefulSetSpec{ - ServiceName: "storageos", - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: ls, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: ls, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: "storageos-statefulset-sa", - Containers: []corev1.Container{ - { - Image: s.stos.Spec.GetCSIExternalProvisionerImage(CSIV1Supported(s.k8sVersion)), - Name: "csi-external-provisioner", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--v=5", - "--provisioner=storageos", - "--csi-address=$(ADDRESS)", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - }, - }, - { - Image: s.stos.Spec.GetCSIExternalAttacherImage(CSIV1Supported(s.k8sVersion)), - Name: "csi-external-attacher", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--v=5", - "--csi-address=$(ADDRESS)", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "plugin-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), - Type: &hostpathDirOrCreate, - }, - }, - }, - }, - }, - }, - }, - } - - if CSIV1Supported(s.k8sVersion) { - driverReg := corev1.Container{ - Image: s.stos.Spec.GetCSIClusterDriverRegistrarImage(), - Name: "csi-driver-k8s-registrar", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--v=5", - "--csi-address=$(ADDRESS)", - "--pod-info-mount-version=v1", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - { - Name: kubeNodeNameEnvVar, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "spec.nodeName", - }, - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - }, - } - - sset.Spec.Template.Spec.Containers = append(sset.Spec.Template.Spec.Containers, driverReg) - } - - podSpec := &sset.Spec.Template.Spec - - s.addNodeAffinity(podSpec) - - if err := s.addTolerations(podSpec); err != nil { - return err - } - - return s.createOrUpdateObject(sset) -} - -func (s *Deployment) deleteStatefulSet(name string) error { - return s.deleteObject(s.getStatefulSet(name)) -} - -func (s *Deployment) getStatefulSet(name string) *appsv1.StatefulSet { - return &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": "storageos", - }, - }, - } -} - -func (s *Deployment) createService() error { - svc := s.getService(s.stos.Spec.GetServiceName()) - svc.Spec = corev1.ServiceSpec{ - Type: corev1.ServiceType(s.stos.Spec.GetServiceType()), - Ports: []corev1.ServicePort{ - { - Name: s.stos.Spec.GetServiceName(), - Protocol: "TCP", - Port: int32(s.stos.Spec.GetServiceInternalPort()), - TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: int32(s.stos.Spec.GetServiceExternalPort())}, - }, - }, - Selector: map[string]string{ - "app": appName, - "kind": daemonsetKind, - }, - } - - if err := s.client.Create(context.Background(), svc); err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create %s: %v", svc.GroupVersionKind().Kind, err) - } - // if err := s.createOrUpdateObject(svc); err != nil { - // return err - // } - - // Patch storageos-api secret with above service IP in apiAddress. - if !s.stos.Spec.CSI.Enable { - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: s.stos.Spec.SecretRefName, - Namespace: s.stos.Spec.SecretRefNamespace, - }, - } - nsNameSecret := types.NamespacedName{ - Namespace: secret.ObjectMeta.GetNamespace(), - Name: secret.ObjectMeta.GetName(), - } - if err := s.client.Get(context.Background(), nsNameSecret, secret); err != nil { - return err - } - - nsNameService := types.NamespacedName{ - Namespace: svc.ObjectMeta.GetNamespace(), - Name: svc.ObjectMeta.GetName(), - } - if err := s.client.Get(context.Background(), nsNameService, svc); err != nil { - return err - } - - apiAddress := fmt.Sprintf("tcp://%s:5705", svc.Spec.ClusterIP) - secret.Data[apiAddressKey] = []byte(apiAddress) - - if err := s.client.Update(context.Background(), secret); err != nil { - return err - } - } - - return nil -} - -func (s *Deployment) deleteService(name string) error { - return s.deleteObject(s.getService(name)) -} - -func (s *Deployment) getService(name string) *corev1.Service { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Service", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - Annotations: s.stos.Spec.Service.Annotations, - }, - } -} - -func (s *Deployment) createIngress() error { - ingress := &v1beta1.Ingress{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "extensions/v1beta1", - Kind: "Ingress", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "storageos-ingress", - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - Annotations: s.stos.Spec.Ingress.Annotations, - }, - Spec: v1beta1.IngressSpec{ - Backend: &v1beta1.IngressBackend{ - ServiceName: s.stos.Spec.GetServiceName(), - ServicePort: intstr.IntOrString{Type: intstr.Int, IntVal: int32(s.stos.Spec.GetServiceExternalPort())}, - }, - }, - } - - if s.stos.Spec.Ingress.TLS { - ingress.Spec.TLS = []v1beta1.IngressTLS{ - v1beta1.IngressTLS{ - Hosts: []string{s.stos.Spec.Ingress.Hostname}, - SecretName: tlsSecretName, - }, - } - } - - return s.createOrUpdateObject(ingress) -} - -func (s *Deployment) deleteIngress(name string) error { - return s.deleteObject(s.getIngress(name)) -} - -func (s *Deployment) getIngress(name string) *v1beta1.Ingress { - return &v1beta1.Ingress{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "extensions/v1beta1", - Kind: "Ingress", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - Annotations: s.stos.Spec.Ingress.Annotations, - }, - } -} - -func (s *Deployment) createTLSSecret() error { - cert, key, err := s.getTLSData() - if err != nil { - return err - } - - secret := s.getSecret(tlsSecretName) - secret.Type = corev1.SecretType(tlsSecretType) - secret.Data = map[string][]byte{ - tlsCertKey: cert, - tlsKeyKey: key, - } - return s.createOrUpdateObject(secret) -} - -func (s *Deployment) deleteSecret(name string) error { - return s.deleteObject(s.getSecret(name)) -} - -func (s *Deployment) getSecret(name string) *corev1.Secret { - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - -func (s *Deployment) createInitSecret() error { - username, password, err := s.getAdminCreds() - if err != nil { - return err - } - if err := s.createCredSecret(initSecretName, username, password); err != nil { - return err - } - return nil -} - -func (s *Deployment) getAdminCreds() ([]byte, []byte, error) { - var username, password []byte - if s.stos.Spec.SecretRefName != "" && s.stos.Spec.SecretRefNamespace != "" { - se := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: s.stos.Spec.SecretRefName, - Namespace: s.stos.Spec.SecretRefNamespace, - }, - } - nsName := types.NamespacedName{ - Name: se.ObjectMeta.GetName(), - Namespace: se.ObjectMeta.GetNamespace(), - } - if err := s.client.Get(context.Background(), nsName, se); err != nil { - return nil, nil, err - } - - username = se.Data[apiUsernameKey] - password = se.Data[apiPasswordKey] - } else { - // Use the default credentials. - username = []byte(defaultUsername) - password = []byte(defaultPassword) - } - - return username, password, nil -} - -func (s *Deployment) getTLSData() ([]byte, []byte, error) { - var cert, key []byte - if s.stos.Spec.SecretRefName != "" && s.stos.Spec.SecretRefNamespace != "" { - se := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: s.stos.Spec.SecretRefName, - Namespace: s.stos.Spec.SecretRefNamespace, - }, - } - nsName := types.NamespacedName{ - Name: se.ObjectMeta.GetName(), - Namespace: se.ObjectMeta.GetNamespace(), - } - if err := s.client.Get(context.Background(), nsName, se); err != nil { - return nil, nil, err - } - - cert = se.Data[tlsCertKey] - key = se.Data[tlsKeyKey] - } else { - cert = []byte("") - key = []byte("") - } - - return cert, key, nil -} - -// createCSISecrets checks which CSI creds are enabled and creates secret for -// those components. -func (s *Deployment) createCSISecrets() error { - // Create Provision Secret. - if s.stos.Spec.CSI.EnableProvisionCreds { - username, password, err := s.getCSICreds(csiProvisionUsernameKey, csiProvisionPasswordKey) - if err != nil { - return err - } - if err := s.createCredSecret(csiProvisionerSecretName, username, password); err != nil { - return err - } - } - - // Create Controller Publish Secret. - if s.stos.Spec.CSI.EnableControllerPublishCreds { - username, password, err := s.getCSICreds(csiControllerPublishUsernameKey, csiControllerPublishPasswordKey) - if err != nil { - return err - } - if err := s.createCredSecret(csiControllerPublishSecretName, username, password); err != nil { - return err - } - } - - // Create Node Publish Secret. - if s.stos.Spec.CSI.EnableNodePublishCreds { - username, password, err := s.getCSICreds(csiNodePublishUsernameKey, csiNodePublishPasswordKey) - if err != nil { - return err - } - if err := s.createCredSecret(csiNodePublishSecretName, username, password); err != nil { - return err - } - } - - return nil -} - -// deleteCSISecrets deletes all the CSI related secrets. -func (s *Deployment) deleteCSISecrets() error { - if err := s.deleteSecret(csiProvisionerSecretName); err != nil { - return err - } - - if err := s.deleteSecret(csiControllerPublishSecretName); err != nil { - return err - } - - if err := s.deleteSecret(csiNodePublishSecretName); err != nil { - return err - } - - return nil -} - -func (s *Deployment) createCredSecret(name string, username, password []byte) error { - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: s.stos.Spec.GetResourceNS(), - Labels: map[string]string{ - "app": appName, - }, - }, - Type: corev1.SecretType(corev1.SecretTypeOpaque), - Data: map[string][]byte{ - "username": username, - "password": password, - }, - } - - return s.createOrUpdateObject(secret) -} - -// getCSICreds - given username and password keys, it fetches the creds from -// storageos-api secret and returns them. -func (s *Deployment) getCSICreds(usernameKey, passwordKey string) (username []byte, password []byte, err error) { - // Get the username and password from storageos-api secret object. - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: s.stos.Spec.SecretRefName, - Namespace: s.stos.Spec.SecretRefNamespace, - }, - } - nsName := types.NamespacedName{ - Name: secret.ObjectMeta.GetName(), - Namespace: secret.ObjectMeta.GetNamespace(), - } - if err := s.client.Get(context.Background(), nsName, secret); err != nil { - return nil, nil, err - } - - username = secret.Data[usernameKey] - password = secret.Data[passwordKey] - - return username, password, err -} - -func (s *Deployment) createStorageClass() error { - // Provisioner name for in-tree storage plugin. - provisioner := intreeProvisionerName - - if s.stos.Spec.CSI.Enable { - provisioner = csiProvisionerName - } - - sc := s.getStorageClass("fast") - sc.Provisioner = provisioner - sc.Parameters = map[string]string{ - "pool": "default", - } - - if s.stos.Spec.CSI.Enable { - // Add CSI creds secrets in parameters. - if CSIV1Supported(s.k8sVersion) { - // New CSI secret parameter keys were introduced in CSI v1. - sc.Parameters[csiV1FSType] = defaultFSType - if s.stos.Spec.CSI.EnableProvisionCreds { - sc.Parameters[csiV1ProvisionerSecretNameKey] = csiProvisionerSecretName - sc.Parameters[csiV1ProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableControllerPublishCreds { - sc.Parameters[csiV1ControllerPublishSecretNameKey] = csiControllerPublishSecretName - sc.Parameters[csiV1ControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableNodePublishCreds { - sc.Parameters[csiV1NodePublishSecretNameKey] = csiNodePublishSecretName - sc.Parameters[csiV1NodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - } else { - sc.Parameters[fsType] = defaultFSType - if s.stos.Spec.CSI.EnableProvisionCreds { - sc.Parameters[csiV0ProvisionerSecretNameKey] = csiProvisionerSecretName - sc.Parameters[csiV0ProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableControllerPublishCreds { - sc.Parameters[csiV0ControllerPublishSecretNameKey] = csiControllerPublishSecretName - sc.Parameters[csiV0ControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableNodePublishCreds { - sc.Parameters[csiV0NodePublishSecretNameKey] = csiNodePublishSecretName - sc.Parameters[csiV0NodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - } - } else { - sc.Parameters[fsType] = defaultFSType - // Add StorageOS admin secrets name and namespace. - sc.Parameters[secretNamespaceKey] = s.stos.Spec.SecretRefNamespace - sc.Parameters[secretNameKey] = s.stos.Spec.SecretRefName - } - - return s.createOrUpdateObject(sc) -} - -func (s *Deployment) deleteStorageClass(name string) error { - return s.deleteObject(s.getStorageClass(name)) -} - -func (s *Deployment) getStorageClass(name string) *storagev1.StorageClass { - return &storagev1.StorageClass{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "storage.k8s.io/v1", - Kind: "StorageClass", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "app": appName, - }, - }, - } -} - // createOrUpdateObject attempts to create a given object. If the object already // exists and `Deployment.update` is false, no change is made. If update is true, // the existing object is updated. diff --git a/pkg/storageos/deployment.go b/pkg/storageos/deployment.go new file mode 100644 index 000000000..398bd9118 --- /dev/null +++ b/pkg/storageos/deployment.go @@ -0,0 +1,32 @@ +package storageos + +import ( + api "github.com/storageos/cluster-operator/pkg/apis/storageos/v1alpha1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Deployment stores all the resource configuration and performs +// resource creation and update. +type Deployment struct { + client client.Client + stos *api.StorageOSCluster + recorder record.EventRecorder + k8sVersion string + scheme *runtime.Scheme + update bool +} + +// NewDeployment creates a new Deployment given a k8c client, storageos manifest +// and an event broadcast recorder. +func NewDeployment(client client.Client, stos *api.StorageOSCluster, recorder record.EventRecorder, scheme *runtime.Scheme, version string, update bool) *Deployment { + return &Deployment{ + client: client, + stos: stos, + recorder: recorder, + k8sVersion: version, + scheme: scheme, + update: update, + } +} diff --git a/pkg/storageos/ingress.go b/pkg/storageos/ingress.go new file mode 100644 index 000000000..49272bcea --- /dev/null +++ b/pkg/storageos/ingress.go @@ -0,0 +1,62 @@ +package storageos + +import ( + "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func (s *Deployment) createIngress() error { + ingress := &v1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "extensions/v1beta1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "storageos-ingress", + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + Annotations: s.stos.Spec.Ingress.Annotations, + }, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: s.stos.Spec.GetServiceName(), + ServicePort: intstr.IntOrString{Type: intstr.Int, IntVal: int32(s.stos.Spec.GetServiceExternalPort())}, + }, + }, + } + + if s.stos.Spec.Ingress.TLS { + ingress.Spec.TLS = []v1beta1.IngressTLS{ + v1beta1.IngressTLS{ + Hosts: []string{s.stos.Spec.Ingress.Hostname}, + SecretName: tlsSecretName, + }, + } + } + + return s.createOrUpdateObject(ingress) +} + +func (s *Deployment) deleteIngress(name string) error { + return s.deleteObject(s.getIngress(name)) +} + +func (s *Deployment) getIngress(name string) *v1beta1.Ingress { + return &v1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "extensions/v1beta1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + Annotations: s.stos.Spec.Ingress.Annotations, + }, + } +} diff --git a/pkg/storageos/podspec.go b/pkg/storageos/podspec.go new file mode 100644 index 000000000..11840d9e7 --- /dev/null +++ b/pkg/storageos/podspec.go @@ -0,0 +1,278 @@ +package storageos + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" +) + +// addSharedDir adds env var and volumes for shared dir when running kubelet in +// a container. +func (s *Deployment) addSharedDir(podSpec *corev1.PodSpec) { + mountPropagationBidirectional := corev1.MountPropagationBidirectional + nodeContainer := &podSpec.Containers[0] + + // If kubelet is running in a container, sharedDir should be set. + if s.stos.Spec.SharedDir != "" { + envVar := corev1.EnvVar{ + Name: deviceDirEnvVar, + Value: fmt.Sprintf("%s/devices", s.stos.Spec.SharedDir), + } + nodeContainer.Env = append(nodeContainer.Env, envVar) + + sharedDir := corev1.Volume{ + Name: "shared", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.SharedDir, + }, + }, + } + podSpec.Volumes = append(podSpec.Volumes, sharedDir) + + volMnt := corev1.VolumeMount{ + Name: "shared", + MountPath: s.stos.Spec.SharedDir, + MountPropagation: &mountPropagationBidirectional, + } + nodeContainer.VolumeMounts = append(nodeContainer.VolumeMounts, volMnt) + } +} + +// addCSI adds the CSI env vars, volumes and containers to the provided podSpec. +func (s *Deployment) addCSI(podSpec *corev1.PodSpec) { + hostpathDirOrCreate := corev1.HostPathDirectoryOrCreate + hostpathDir := corev1.HostPathDirectory + mountPropagationBidirectional := corev1.MountPropagationBidirectional + + nodeContainer := &podSpec.Containers[0] + + // Add CSI specific configurations if enabled. + if s.stos.Spec.CSI.Enable { + vols := []corev1.Volume{ + { + Name: "registrar-socket-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIRegistrarSocketDir(), + Type: &hostpathDirOrCreate, + }, + }, + }, + { + Name: "kubelet-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIKubeletDir(), + Type: &hostpathDir, + }, + }, + }, + { + Name: "plugin-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), + Type: &hostpathDirOrCreate, + }, + }, + }, + { + Name: "device-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIDeviceDir(), + Type: &hostpathDir, + }, + }, + }, + { + Name: "registration-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIRegistrationDir(CSIV1Supported(s.k8sVersion)), + Type: &hostpathDir, + }, + }, + }, + } + + podSpec.Volumes = append(podSpec.Volumes, vols...) + + volMnts := []corev1.VolumeMount{ + { + Name: "kubelet-dir", + MountPath: s.stos.Spec.GetCSIKubeletDir(), + MountPropagation: &mountPropagationBidirectional, + }, + { + Name: "plugin-dir", + MountPath: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), + }, + { + Name: "device-dir", + MountPath: s.stos.Spec.GetCSIDeviceDir(), + }, + } + + // Append volume mounts to the first container, the only container is the node container, at this point. + nodeContainer.VolumeMounts = append(nodeContainer.VolumeMounts, volMnts...) + + envVar := []corev1.EnvVar{ + { + Name: csiEndpointEnvVar, + Value: s.stos.Spec.GetCSIEndpoint(CSIV1Supported(s.k8sVersion)), + }, + } + + // Append CSI Provision Creds env var if enabled. + if s.stos.Spec.CSI.EnableProvisionCreds { + envVar = append( + envVar, + corev1.EnvVar{ + Name: csiRequireCredsCreateEnvVar, + Value: "true", + }, + corev1.EnvVar{ + Name: csiRequireCredsDeleteEnvVar, + Value: "true", + }, + getCSICredsEnvVar(csiProvisionCredsUsernameEnvVar, csiProvisionerSecretName, "username"), + getCSICredsEnvVar(csiProvisionCredsPasswordEnvVar, csiProvisionerSecretName, "password"), + ) + } + + // Append CSI Controller Publish env var if enabled. + if s.stos.Spec.CSI.EnableControllerPublishCreds { + envVar = append( + envVar, + corev1.EnvVar{ + Name: csiRequireCredsCtrlPubEnvVar, + Value: "true", + }, + corev1.EnvVar{ + Name: csiRequireCredsCtrlUnpubEnvVar, + Value: "true", + }, + getCSICredsEnvVar(csiControllerPubCredsUsernameEnvVar, csiControllerPublishSecretName, "username"), + getCSICredsEnvVar(csiControllerPubCredsPasswordEnvVar, csiControllerPublishSecretName, "password"), + ) + } + + // Append CSI Node Publish env var if enabled. + if s.stos.Spec.CSI.EnableNodePublishCreds { + envVar = append( + envVar, + corev1.EnvVar{ + Name: csiRequireCredsNodePubEnvVar, + Value: "true", + }, + getCSICredsEnvVar(csiNodePubCredsUsernameEnvVar, csiNodePublishSecretName, "username"), + getCSICredsEnvVar(csiNodePubCredsPasswordEnvVar, csiNodePublishSecretName, "password"), + ) + } + + // Append env vars to the first container, node container. + nodeContainer.Env = append(nodeContainer.Env, envVar...) + + driverReg := corev1.Container{ + Image: s.stos.Spec.GetCSINodeDriverRegistrarImage(CSIV1Supported(s.k8sVersion)), + Name: "csi-driver-registrar", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--v=5", + "--csi-address=$(ADDRESS)", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + { + Name: kubeNodeNameEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + { + Name: "registrar-socket-dir", + MountPath: "/var/lib/csi/sockets/", + }, + { + Name: "registration-dir", + MountPath: "/registration", + }, + }, + } + + // Add extra flags to activate node-register mode if kubelet plugins + // watcher is supported. + if kubeletPluginsWatcherSupported(s.k8sVersion) { + driverReg.Args = append( + driverReg.Args, + fmt.Sprintf("--kubelet-registration-path=%s", s.stos.Spec.GetCSIKubeletRegistrationPath(CSIV1Supported(s.k8sVersion)))) + } + podSpec.Containers = append(podSpec.Containers, driverReg) + + if CSIV1Supported(s.k8sVersion) { + livenessProbe := corev1.Container{ + Image: s.stos.Spec.GetCSILivenessProbeImage(), + Name: "csi-liveness-probe", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--csi-address=$(ADDRESS)", + "--connection-timeout=3s", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + }, + } + podSpec.Containers = append(podSpec.Containers, livenessProbe) + } + } +} + +// addNodeAffinity adds node affinity to the given pod spec from the cluster +// spec NodeSelectorLabel. +func (s *Deployment) addNodeAffinity(podSpec *corev1.PodSpec) { + if len(s.stos.Spec.NodeSelectorTerms) > 0 { + podSpec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: s.stos.Spec.NodeSelectorTerms, + }, + }} + } +} + +// addTolerations adds tolerations to the given pod spec from cluster +// spec Tolerations. +func (s *Deployment) addTolerations(podSpec *corev1.PodSpec) error { + tolerations := s.stos.Spec.Tolerations + for i := range tolerations { + if tolerations[i].Operator == corev1.TolerationOpExists && tolerations[i].Value != "" { + return fmt.Errorf("key(%s): toleration value must be empty when `operator` is 'Exists'", tolerations[i].Key) + } + } + if len(tolerations) > 0 { + podSpec.Tolerations = s.stos.Spec.Tolerations + } + return nil +} diff --git a/pkg/storageos/rbac.go b/pkg/storageos/rbac.go new file mode 100644 index 000000000..7c063e6ac --- /dev/null +++ b/pkg/storageos/rbac.go @@ -0,0 +1,321 @@ +package storageos + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (s *Deployment) createServiceAccount(name string) error { + sa := s.getServiceAccount(name) + return s.createOrUpdateObject(sa) +} + +func (s *Deployment) deleteServiceAccount(name string) error { + return s.deleteObject(s.getServiceAccount(name)) +} + +// getServiceAccount creates a generic service account object with the given +// name and returns it. +func (s *Deployment) getServiceAccount(name string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createServiceAccountForDaemonSet() error { + return s.createServiceAccount("storageos-daemonset-sa") +} + +func (s *Deployment) createServiceAccountForStatefulSet() error { + return s.createServiceAccount("storageos-statefulset-sa") +} + +func (s *Deployment) createRoleForKeyMgmt() error { + role := s.getRole(keyManagementRoleName) + role.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "create", "delete"}, + }, + } + + return s.createOrUpdateObject(role) +} + +func (s *Deployment) deleteRole(name string) error { + return s.deleteObject(s.getRole(keyManagementRoleName)) +} + +// getRole creates a generic role object with the given name and returns it. +func (s *Deployment) getRole(name string) *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "Role", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createClusterRole(name string, rules []rbacv1.PolicyRule) error { + role := s.getClusterRole(name) + role.Rules = rules + return s.createOrUpdateObject(role) +} + +func (s *Deployment) deleteClusterRole(name string) error { + return s.deleteObject(s.getClusterRole(name)) +} + +func (s *Deployment) getClusterRole(name string) *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRole", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createClusterRoleForDriverRegistrar() error { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list", "watch", "create", "update", "patch"}, + }, + { + APIGroups: []string{"apiextensions.k8s.io"}, + Resources: []string{"customresourcedefinitions"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"csi.storage.k8s.io"}, + Resources: []string{"csidrivers"}, + Verbs: []string{"create"}, + }, + } + return s.createClusterRole("driver-registrar-role", rules) +} + +func (s *Deployment) createClusterRoleForProvisioner() error { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumes"}, + Verbs: []string{"list", "watch", "create", "delete"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumeclaims"}, + Verbs: []string{"get", "list", "watch", "update"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"storageclasses"}, + Verbs: []string{"list", "watch", "get"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list", "watch", "create", "update", "patch"}, + }, + } + return s.createClusterRole("csi-provisioner-role", rules) +} + +func (s *Deployment) createClusterRoleForAttacher() error { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumes"}, + Verbs: []string{"get", "list", "watch", "update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"storageclasses"}, + Verbs: []string{"list", "watch", "get"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"volumeattachments"}, + Verbs: []string{"get", "list", "watch", "update"}, + }, + { + APIGroups: []string{"storage.k8s.io"}, + Resources: []string{"csinodeinfos"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list", "watch", "create", "update", "patch"}, + }, + } + return s.createClusterRole("csi-attacher-role", rules) +} + +func (s *Deployment) createRoleBindingForKeyMgmt() error { + roleBinding := s.getRoleBinding(keyManagementBindingName) + roleBinding.Subjects = []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "storageos-daemonset-sa", + Namespace: s.stos.Spec.GetResourceNS(), + }, + } + roleBinding.RoleRef = rbacv1.RoleRef{ + Kind: "Role", + Name: keyManagementRoleName, + APIGroup: "rbac.authorization.k8s.io", + } + return s.createOrUpdateObject(roleBinding) +} + +func (s *Deployment) deleteRoleBinding(name string) error { + return s.deleteObject(s.getRoleBinding(name)) +} + +func (s *Deployment) getRoleBinding(name string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createClusterRoleBinding(name string, subjects []rbacv1.Subject, roleRef rbacv1.RoleRef) error { + roleBinding := s.getClusterRoleBinding(name) + roleBinding.Subjects = subjects + roleBinding.RoleRef = roleRef + return s.createOrUpdateObject(roleBinding) +} + +func (s *Deployment) deleteClusterRoleBinding(name string) error { + return s.deleteObject(s.getClusterRoleBinding(name)) +} + +func (s *Deployment) getClusterRoleBinding(name string) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createClusterRoleBindingForDriverRegistrar() error { + subjects := []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "storageos-daemonset-sa", + Namespace: s.stos.Spec.GetResourceNS(), + }, + } + roleRef := rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "driver-registrar-role", + APIGroup: "rbac.authorization.k8s.io", + } + return s.createClusterRoleBinding("driver-registrar-binding", subjects, roleRef) +} + +func (s *Deployment) createClusterRoleBindingForK8SDriverRegistrar() error { + subjects := []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "storageos-statefulset-sa", + Namespace: s.stos.Spec.GetResourceNS(), + }, + } + roleRef := rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "driver-registrar-role", + APIGroup: "rbac.authorization.k8s.io", + } + return s.createClusterRoleBinding("k8s-driver-registrar-binding", subjects, roleRef) +} + +func (s *Deployment) createClusterRoleBindingForProvisioner() error { + subjects := []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "storageos-statefulset-sa", + Namespace: s.stos.Spec.GetResourceNS(), + }, + } + roleRef := rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "csi-provisioner-role", + APIGroup: "rbac.authorization.k8s.io", + } + return s.createClusterRoleBinding("csi-provisioner-binding", subjects, roleRef) +} + +func (s *Deployment) createClusterRoleBindingForAttacher() error { + subjects := []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "storageos-statefulset-sa", + Namespace: s.stos.Spec.GetResourceNS(), + }, + } + roleRef := rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "csi-attacher-role", + APIGroup: "rbac.authorization.k8s.io", + } + return s.createClusterRoleBinding("csi-attacher-binding", subjects, roleRef) +} diff --git a/pkg/storageos/secret.go b/pkg/storageos/secret.go new file mode 100644 index 000000000..670eb8379 --- /dev/null +++ b/pkg/storageos/secret.go @@ -0,0 +1,225 @@ +package storageos + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func (s *Deployment) deleteSecret(name string) error { + return s.deleteObject(s.getSecret(name)) +} + +func (s *Deployment) getSecret(name string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + }, + } +} + +func (s *Deployment) createInitSecret() error { + username, password, err := s.getAdminCreds() + if err != nil { + return err + } + if err := s.createCredSecret(initSecretName, username, password); err != nil { + return err + } + return nil +} + +func (s *Deployment) createTLSSecret() error { + cert, key, err := s.getTLSData() + if err != nil { + return err + } + + secret := s.getSecret(tlsSecretName) + secret.Type = corev1.SecretType(tlsSecretType) + secret.Data = map[string][]byte{ + tlsCertKey: cert, + tlsKeyKey: key, + } + return s.createOrUpdateObject(secret) +} + +func (s *Deployment) getAdminCreds() ([]byte, []byte, error) { + var username, password []byte + if s.stos.Spec.SecretRefName != "" && s.stos.Spec.SecretRefNamespace != "" { + se := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: s.stos.Spec.SecretRefName, + Namespace: s.stos.Spec.SecretRefNamespace, + }, + } + nsName := types.NamespacedName{ + Name: se.ObjectMeta.GetName(), + Namespace: se.ObjectMeta.GetNamespace(), + } + if err := s.client.Get(context.Background(), nsName, se); err != nil { + return nil, nil, err + } + + username = se.Data[apiUsernameKey] + password = se.Data[apiPasswordKey] + } else { + // Use the default credentials. + username = []byte(defaultUsername) + password = []byte(defaultPassword) + } + + return username, password, nil +} + +func (s *Deployment) getTLSData() ([]byte, []byte, error) { + var cert, key []byte + if s.stos.Spec.SecretRefName != "" && s.stos.Spec.SecretRefNamespace != "" { + se := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: s.stos.Spec.SecretRefName, + Namespace: s.stos.Spec.SecretRefNamespace, + }, + } + nsName := types.NamespacedName{ + Name: se.ObjectMeta.GetName(), + Namespace: se.ObjectMeta.GetNamespace(), + } + if err := s.client.Get(context.Background(), nsName, se); err != nil { + return nil, nil, err + } + + cert = se.Data[tlsCertKey] + key = se.Data[tlsKeyKey] + } else { + cert = []byte("") + key = []byte("") + } + + return cert, key, nil +} + +// createCSISecrets checks which CSI creds are enabled and creates secret for +// those components. +func (s *Deployment) createCSISecrets() error { + // Create Provision Secret. + if s.stos.Spec.CSI.EnableProvisionCreds { + username, password, err := s.getCSICreds(csiProvisionUsernameKey, csiProvisionPasswordKey) + if err != nil { + return err + } + if err := s.createCredSecret(csiProvisionerSecretName, username, password); err != nil { + return err + } + } + + // Create Controller Publish Secret. + if s.stos.Spec.CSI.EnableControllerPublishCreds { + username, password, err := s.getCSICreds(csiControllerPublishUsernameKey, csiControllerPublishPasswordKey) + if err != nil { + return err + } + if err := s.createCredSecret(csiControllerPublishSecretName, username, password); err != nil { + return err + } + } + + // Create Node Publish Secret. + if s.stos.Spec.CSI.EnableNodePublishCreds { + username, password, err := s.getCSICreds(csiNodePublishUsernameKey, csiNodePublishPasswordKey) + if err != nil { + return err + } + if err := s.createCredSecret(csiNodePublishSecretName, username, password); err != nil { + return err + } + } + + return nil +} + +// deleteCSISecrets deletes all the CSI related secrets. +func (s *Deployment) deleteCSISecrets() error { + if err := s.deleteSecret(csiProvisionerSecretName); err != nil { + return err + } + + if err := s.deleteSecret(csiControllerPublishSecretName); err != nil { + return err + } + + if err := s.deleteSecret(csiNodePublishSecretName); err != nil { + return err + } + + return nil +} + +func (s *Deployment) createCredSecret(name string, username, password []byte) error { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + }, + Type: corev1.SecretType(corev1.SecretTypeOpaque), + Data: map[string][]byte{ + "username": username, + "password": password, + }, + } + + return s.createOrUpdateObject(secret) +} + +// getCSICreds - given username and password keys, it fetches the creds from +// storageos-api secret and returns them. +func (s *Deployment) getCSICreds(usernameKey, passwordKey string) (username []byte, password []byte, err error) { + // Get the username and password from storageos-api secret object. + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: s.stos.Spec.SecretRefName, + Namespace: s.stos.Spec.SecretRefNamespace, + }, + } + nsName := types.NamespacedName{ + Name: secret.ObjectMeta.GetName(), + Namespace: secret.ObjectMeta.GetNamespace(), + } + if err := s.client.Get(context.Background(), nsName, secret); err != nil { + return nil, nil, err + } + + username = secret.Data[usernameKey] + password = secret.Data[passwordKey] + + return username, password, err +} diff --git a/pkg/storageos/service.go b/pkg/storageos/service.go new file mode 100644 index 000000000..7effdca55 --- /dev/null +++ b/pkg/storageos/service.go @@ -0,0 +1,97 @@ +package storageos + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func (s *Deployment) createService() error { + svc := s.getService(s.stos.Spec.GetServiceName()) + svc.Spec = corev1.ServiceSpec{ + Type: corev1.ServiceType(s.stos.Spec.GetServiceType()), + Ports: []corev1.ServicePort{ + { + Name: s.stos.Spec.GetServiceName(), + Protocol: "TCP", + Port: int32(s.stos.Spec.GetServiceInternalPort()), + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: int32(s.stos.Spec.GetServiceExternalPort())}, + }, + }, + Selector: map[string]string{ + "app": appName, + "kind": daemonsetKind, + }, + } + + if err := s.client.Create(context.Background(), svc); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create %s: %v", svc.GroupVersionKind().Kind, err) + } + // if err := s.createOrUpdateObject(svc); err != nil { + // return err + // } + + // Patch storageos-api secret with above service IP in apiAddress. + if !s.stos.Spec.CSI.Enable { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: s.stos.Spec.SecretRefName, + Namespace: s.stos.Spec.SecretRefNamespace, + }, + } + nsNameSecret := types.NamespacedName{ + Namespace: secret.ObjectMeta.GetNamespace(), + Name: secret.ObjectMeta.GetName(), + } + if err := s.client.Get(context.Background(), nsNameSecret, secret); err != nil { + return err + } + + nsNameService := types.NamespacedName{ + Namespace: svc.ObjectMeta.GetNamespace(), + Name: svc.ObjectMeta.GetName(), + } + if err := s.client.Get(context.Background(), nsNameService, svc); err != nil { + return err + } + + apiAddress := fmt.Sprintf("tcp://%s:5705", svc.Spec.ClusterIP) + secret.Data[apiAddressKey] = []byte(apiAddress) + + if err := s.client.Update(context.Background(), secret); err != nil { + return err + } + } + + return nil +} + +func (s *Deployment) deleteService(name string) error { + return s.deleteObject(s.getService(name)) +} + +func (s *Deployment) getService(name string) *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": appName, + }, + Annotations: s.stos.Spec.Service.Annotations, + }, + } +} diff --git a/pkg/storageos/statefulset.go b/pkg/storageos/statefulset.go new file mode 100644 index 000000000..201aa1475 --- /dev/null +++ b/pkg/storageos/statefulset.go @@ -0,0 +1,164 @@ +package storageos + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (s *Deployment) createStatefulSet() error { + ls := labelsForStatefulSet(s.stos.Name) + replicas := int32(1) + hostpathDirOrCreate := corev1.HostPathDirectoryOrCreate + + sset := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: statefulsetName, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": "storageos", + }, + }, + Spec: appsv1.StatefulSetSpec{ + ServiceName: "storageos", + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: ls, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: ls, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "storageos-statefulset-sa", + Containers: []corev1.Container{ + { + Image: s.stos.Spec.GetCSIExternalProvisionerImage(CSIV1Supported(s.k8sVersion)), + Name: "csi-external-provisioner", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--v=5", + "--provisioner=storageos", + "--csi-address=$(ADDRESS)", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + }, + }, + { + Image: s.stos.Spec.GetCSIExternalAttacherImage(CSIV1Supported(s.k8sVersion)), + Name: "csi-external-attacher", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--v=5", + "--csi-address=$(ADDRESS)", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "plugin-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), + Type: &hostpathDirOrCreate, + }, + }, + }, + }, + }, + }, + }, + } + + if CSIV1Supported(s.k8sVersion) { + driverReg := corev1.Container{ + Image: s.stos.Spec.GetCSIClusterDriverRegistrarImage(), + Name: "csi-driver-k8s-registrar", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--v=5", + "--csi-address=$(ADDRESS)", + "--pod-info-mount-version=v1", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + { + Name: kubeNodeNameEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + }, + } + + sset.Spec.Template.Spec.Containers = append(sset.Spec.Template.Spec.Containers, driverReg) + } + + podSpec := &sset.Spec.Template.Spec + + s.addNodeAffinity(podSpec) + + if err := s.addTolerations(podSpec); err != nil { + return err + } + + return s.createOrUpdateObject(sset) +} + +func (s *Deployment) deleteStatefulSet(name string) error { + return s.deleteObject(s.getStatefulSet(name)) +} + +func (s *Deployment) getStatefulSet(name string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: s.stos.Spec.GetResourceNS(), + Labels: map[string]string{ + "app": "storageos", + }, + }, + } +} diff --git a/pkg/storageos/storageclass.go b/pkg/storageos/storageclass.go new file mode 100644 index 000000000..551e60076 --- /dev/null +++ b/pkg/storageos/storageclass.go @@ -0,0 +1,81 @@ +package storageos + +import ( + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (s *Deployment) createStorageClass() error { + // Provisioner name for in-tree storage plugin. + provisioner := intreeProvisionerName + + if s.stos.Spec.CSI.Enable { + provisioner = csiProvisionerName + } + + sc := s.getStorageClass("fast") + sc.Provisioner = provisioner + sc.Parameters = map[string]string{ + "pool": "default", + } + + if s.stos.Spec.CSI.Enable { + // Add CSI creds secrets in parameters. + if CSIV1Supported(s.k8sVersion) { + // New CSI secret parameter keys were introduced in CSI v1. + sc.Parameters[csiV1FSType] = defaultFSType + if s.stos.Spec.CSI.EnableProvisionCreds { + sc.Parameters[csiV1ProvisionerSecretNameKey] = csiProvisionerSecretName + sc.Parameters[csiV1ProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + if s.stos.Spec.CSI.EnableControllerPublishCreds { + sc.Parameters[csiV1ControllerPublishSecretNameKey] = csiControllerPublishSecretName + sc.Parameters[csiV1ControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + if s.stos.Spec.CSI.EnableNodePublishCreds { + sc.Parameters[csiV1NodePublishSecretNameKey] = csiNodePublishSecretName + sc.Parameters[csiV1NodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + } else { + sc.Parameters[fsType] = defaultFSType + if s.stos.Spec.CSI.EnableProvisionCreds { + sc.Parameters[csiV0ProvisionerSecretNameKey] = csiProvisionerSecretName + sc.Parameters[csiV0ProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + if s.stos.Spec.CSI.EnableControllerPublishCreds { + sc.Parameters[csiV0ControllerPublishSecretNameKey] = csiControllerPublishSecretName + sc.Parameters[csiV0ControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + if s.stos.Spec.CSI.EnableNodePublishCreds { + sc.Parameters[csiV0NodePublishSecretNameKey] = csiNodePublishSecretName + sc.Parameters[csiV0NodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + } + } else { + sc.Parameters[fsType] = defaultFSType + // Add StorageOS admin secrets name and namespace. + sc.Parameters[secretNamespaceKey] = s.stos.Spec.SecretRefNamespace + sc.Parameters[secretNameKey] = s.stos.Spec.SecretRefName + } + + return s.createOrUpdateObject(sc) +} + +func (s *Deployment) deleteStorageClass(name string) error { + return s.deleteObject(s.getStorageClass(name)) +} + +func (s *Deployment) getStorageClass(name string) *storagev1.StorageClass { + return &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "storage.k8s.io/v1", + Kind: "StorageClass", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "app": appName, + }, + }, + } +} diff --git a/test/e2e/util/cluster.go b/test/e2e/util/cluster.go index 348f3b942..f7dbcd609 100644 --- a/test/e2e/util/cluster.go +++ b/test/e2e/util/cluster.go @@ -92,6 +92,14 @@ func ClusterStatusCheck(t *testing.T, status storageos.StorageOSClusterStatus, n if status.Ready != wantReady { t.Errorf("unexpected Ready:\n\t(GOT) %s\n\t(WNT) %s", status.Ready, wantReady) } + + if len(status.Members.Ready) != nodes { + t.Errorf("unexpected number of ready members:\n\t(GOT) %d\n\t(WNT) %d", len(status.Members.Ready), nodes) + } + + if len(status.Members.Unready) != 0 { + t.Errorf("unexpected number of unready members:\n\t(GOT) %d\n\t(WNT) %d", len(status.Members.Unready), 0) + } } // DeployCluster creates a custom resource and checks if the