Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

provider: send vrcs with fixed replication schedule #2905

Merged
merged 1 commit into from
Nov 22, 2024
Merged

provider: send vrcs with fixed replication schedule #2905

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion controllers/storagecluster/generate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"strings"

ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
"github.com/red-hat-storage/ocs-operator/v4/controllers/util"

cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
)

Expand Down Expand Up @@ -101,7 +103,7 @@ func generateNameForSnapshotClass(initData *ocsv1.StorageCluster, snapshotType S
}

func generateNameForSnapshotClassDriver(snapshotType SnapshotterType) string {
return fmt.Sprintf("%s.%s.csi.ceph.com", storageclassDriverNamePrefix, snapshotType)
return fmt.Sprintf("%s.%s.csi.ceph.com", util.StorageClassDriverNamePrefix, snapshotType)
}

func generateNameForSnapshotClassSecret(instance *ocsv1.StorageCluster, snapshotType SnapshotterType) string {
Expand Down
26 changes: 8 additions & 18 deletions controllers/storagecluster/storageclasses.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,6 @@ import (
const (
storageClassSkippedError = "some StorageClasses were skipped while waiting for pre-requisites to be met"
defaultStorageClassAnnotation = "storageclass.kubernetes.io/is-default-class"

//storage class driver name prefix
storageclassDriverNamePrefix = "openshift-storage"
)

var (
RbdDriverName = storageclassDriverNamePrefix + ".rbd.csi.ceph.com"
CephFSDriverName = storageclassDriverNamePrefix + ".cephfs.csi.ceph.com"
nfsDriverName = storageclassDriverNamePrefix + ".nfs.csi.ceph.com"
obcDriverName = storageclassDriverNamePrefix + ".ceph.rook.io/bucket"
)

// StorageClassConfiguration provides configuration options for a StorageClass.
Expand Down Expand Up @@ -109,7 +99,7 @@ func (r *StorageClusterReconciler) createStorageClasses(sccs []StorageClassConfi
sc := scc.storageClass

switch {
case (strings.Contains(sc.Name, "-ceph-rbd") || (strings.Contains(sc.Provisioner, RbdDriverName)) && !strings.Contains(sc.Name, "-ceph-non-resilient-rbd")) && !scc.isClusterExternal:
case (strings.Contains(sc.Name, "-ceph-rbd") || (strings.Contains(sc.Provisioner, util.RbdDriverName)) && !strings.Contains(sc.Name, "-ceph-non-resilient-rbd")) && !scc.isClusterExternal:
// wait for CephBlockPool to be ready
cephBlockPool := cephv1.CephBlockPool{}
key := types.NamespacedName{Name: sc.Parameters["pool"], Namespace: namespace}
Expand Down Expand Up @@ -171,7 +161,7 @@ func (r *StorageClusterReconciler) createStorageClasses(sccs []StorageClassConfi
skippedSC = append(skippedSC, sc.Name)
continue
}
case (strings.Contains(sc.Name, "-cephfs") || strings.Contains(sc.Provisioner, CephFSDriverName)) && !scc.isClusterExternal:
case (strings.Contains(sc.Name, "-cephfs") || strings.Contains(sc.Provisioner, util.CephFSDriverName)) && !scc.isClusterExternal:
// wait for CephFilesystem to be ready
cephFilesystem := cephv1.CephFilesystem{}
key := types.NamespacedName{Name: sc.Parameters["fsName"], Namespace: namespace}
Expand All @@ -184,7 +174,7 @@ func (r *StorageClusterReconciler) createStorageClasses(sccs []StorageClassConfi
skippedSC = append(skippedSC, sc.Name)
continue
}
case strings.Contains(sc.Name, "-nfs") || strings.Contains(sc.Provisioner, nfsDriverName):
case strings.Contains(sc.Name, "-nfs") || strings.Contains(sc.Provisioner, util.NfsDriverName):
// wait for CephNFS to be ready
cephNFS := cephv1.CephNFS{}
key := types.NamespacedName{Name: sc.Parameters["nfsCluster"], Namespace: namespace}
Expand Down Expand Up @@ -270,7 +260,7 @@ func newCephFilesystemStorageClassConfiguration(initData *ocsv1.StorageCluster)
"description": "Provides RWO and RWX Filesystem volumes",
},
},
Provisioner: CephFSDriverName,
Provisioner: util.CephFSDriverName,
ReclaimPolicy: &persistentVolumeReclaimDelete,
// AllowVolumeExpansion is set to true to enable expansion of OCS backed Volumes
AllowVolumeExpansion: &allowVolumeExpansion,
Expand Down Expand Up @@ -305,7 +295,7 @@ func newCephBlockPoolStorageClassConfiguration(initData *ocsv1.StorageCluster) S
"reclaimspace.csiaddons.openshift.io/schedule": "@weekly",
},
},
Provisioner: RbdDriverName,
Provisioner: util.RbdDriverName,
ReclaimPolicy: &persistentVolumeReclaimDelete,
// AllowVolumeExpansion is set to true to enable expansion of OCS backed Volumes
AllowVolumeExpansion: &allowVolumeExpansion,
Expand Down Expand Up @@ -364,7 +354,7 @@ func newNonResilientCephBlockPoolStorageClassConfiguration(initData *ocsv1.Stora
"reclaimspace.csiaddons.openshift.io/schedule": "@weekly",
},
},
Provisioner: RbdDriverName,
Provisioner: util.RbdDriverName,
ReclaimPolicy: &persistentVolumeReclaimDelete,
VolumeBindingMode: &volumeBindingWaitForFirstConsumer,
// AllowVolumeExpansion is set to true to enable expansion of OCS backed Volumes
Expand Down Expand Up @@ -403,7 +393,7 @@ func newCephNFSStorageClassConfiguration(initData *ocsv1.StorageCluster) Storage
"description": "Provides RWO and RWX Filesystem volumes",
},
},
Provisioner: nfsDriverName,
Provisioner: util.NfsDriverName,
ReclaimPolicy: &persistentVolumeReclaimDelete,
AllowVolumeExpansion: &allowVolumeExpansion,
Parameters: map[string]string{
Expand Down Expand Up @@ -449,7 +439,7 @@ func newCephOBCStorageClassConfiguration(initData *ocsv1.StorageCluster) Storage
"description": "Provides Object Bucket Claims (OBCs)",
},
},
Provisioner: obcDriverName,
Provisioner: util.ObcDriverName,
ReclaimPolicy: &reclaimPolicy,
Parameters: map[string]string{
"objectStoreNamespace": initData.Namespace,
Expand Down
9 changes: 9 additions & 0 deletions controllers/util/csi.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package util

const (
StorageClassDriverNamePrefix = "openshift-storage"
RbdDriverName = StorageClassDriverNamePrefix + ".rbd.csi.ceph.com"
CephFSDriverName = StorageClassDriverNamePrefix + ".cephfs.csi.ceph.com"
NfsDriverName = StorageClassDriverNamePrefix + ".nfs.csi.ceph.com"
ObcDriverName = StorageClassDriverNamePrefix + ".ceph.rook.io/bucket"
)
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ require (
github.com/blang/semver/v4 v4.0.0
github.com/ceph/ceph-csi-operator/api v0.0.0-20241119082218-62dc94e55c32
github.com/ceph/ceph-csi/api v0.0.0-20241119113434-d457840d2183
github.com/csi-addons/kubernetes-csi-addons v0.10.0
github.com/go-logr/logr v1.4.2
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/csi-addons/kubernetes-csi-addons v0.10.0 h1:bBc6nb1oROz4RLhqoLFNeGymk2jIRXcx7LvAup9+3Jg=
github.com/csi-addons/kubernetes-csi-addons v0.10.0/go.mod h1:nqi369YuYMIdysBbHjtYJcWFpcxujPot1HS6tnNWBV4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
Expand Down
9 changes: 9 additions & 0 deletions metrics/vendor/github.com/red-hat-storage/ocs-operator/v4/controllers/util/csi.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

48 changes: 31 additions & 17 deletions services/provider/server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,32 +11,29 @@ import (
"encoding/json"
"encoding/pem"
"fmt"
nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1"
"math"
"net"
"slices"
"strconv"
"strings"
"time"

"k8s.io/utils/ptr"

"github.com/blang/semver/v4"
quotav1 "github.com/openshift/api/quota/v1"
routev1 "github.com/openshift/api/route/v1"
opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1"
pb "github.com/red-hat-storage/ocs-operator/services/provider/api/v4"
controllers "github.com/red-hat-storage/ocs-operator/v4/controllers/storageconsumer"
"github.com/red-hat-storage/ocs-operator/v4/controllers/util"
"github.com/red-hat-storage/ocs-operator/v4/services"
ocsVersion "github.com/red-hat-storage/ocs-operator/v4/version"
rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/blang/semver/v4"
csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1"
replicationv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/replication.storage/v1alpha1"
nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1"
quotav1 "github.com/openshift/api/quota/v1"
routev1 "github.com/openshift/api/route/v1"
opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
Expand All @@ -45,9 +42,12 @@ import (
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
klog "k8s.io/klog/v2"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
)

Expand Down Expand Up @@ -731,6 +731,8 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S
replicationEnabled := len(scp.Items) > 0
replicationID := req.StorageClaimName

clientProfile := util.CalculateMD5Hash(req.StorageClaimName)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this could become problematic when we refactor the responsibility of provider & client side code, better be consistent and let client-op set this.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was already decided that we are going to set clusterID from the provider


var extR []*pb.ExternalResource

storageRequestHash := getStorageRequestHash(req.StorageConsumerUUID, req.StorageClaimName)
Expand Down Expand Up @@ -816,9 +818,15 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S
&pb.ExternalResource{
Name: "ceph-rbd",
Kind: "VolumeReplicationClass",
Data: mustMarshal(map[string]string{
"replication.storage.openshift.io/replication-secret-name": provisionerSecretName,
"mirroringMode": "snapshot",
Data: mustMarshal(&replicationv1alpha1.VolumeReplicationClassSpec{
iamniting marked this conversation as resolved.
Show resolved Hide resolved
Parameters: map[string]string{
"replication.storage.openshift.io/replication-secret-name": provisionerSecretName,
"mirroringMode": "snapshot",
// This is a temporary fix till we get the replication schedule to ocs-operator
"schedulingInterval": "5m",
"clusterID": clientProfile,
},
Provisioner: util.RbdDriverName,
}),
Labels: getExternalResourceLabels("VolumeReplicationClass", mirroringEnabled, false, replicationID, storageID),
Annotations: map[string]string{
Expand All @@ -828,10 +836,16 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S
&pb.ExternalResource{
Name: "ceph-rbd-flatten",
Kind: "VolumeReplicationClass",
Data: mustMarshal(map[string]string{
"replication.storage.openshift.io/replication-secret-name": provisionerSecretName,
"mirroringMode": "snapshot",
"flattenMode": "force",
Data: mustMarshal(&replicationv1alpha1.VolumeReplicationClassSpec{
Parameters: map[string]string{
"replication.storage.openshift.io/replication-secret-name": provisionerSecretName,
"mirroringMode": "snapshot",
"flattenMode": "force",
// This is a temporary fix till we get the replication schedule to ocs-operator
"schedulingInterval": "5m",
"clusterID": clientProfile,
},
Provisioner: util.RbdDriverName,
}),
Labels: getExternalResourceLabels("VolumeReplicationClass", mirroringEnabled, true, replicationID, storageID),
Annotations: map[string]string{},
Expand Down
35 changes: 24 additions & 11 deletions services/provider/server/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,18 @@ import (
"strconv"
"testing"

ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1"
pb "github.com/red-hat-storage/ocs-operator/services/provider/api/v4"
controllers "github.com/red-hat-storage/ocs-operator/v4/controllers/storageconsumer"
"github.com/red-hat-storage/ocs-operator/v4/controllers/util"

csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1"
replicationv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/replication.storage/v1alpha1"
nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1"
quotav1 "github.com/openshift/api/quota/v1"
routev1 "github.com/openshift/api/route/v1"
opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1"
pb "github.com/red-hat-storage/ocs-operator/services/provider/api/v4"
controllers "github.com/red-hat-storage/ocs-operator/v4/controllers/storageconsumer"
rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc/codes"
Expand Down Expand Up @@ -661,9 +664,14 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) {
"ceph-rbd-volumereplicationclass": {
Name: "ceph-rbd",
Kind: "VolumeReplicationClass",
Data: map[string]string{
"replication.storage.openshift.io/replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c",
"mirroringMode": "snapshot",
Data: &replicationv1alpha1.VolumeReplicationClassSpec{
Parameters: map[string]string{
"replication.storage.openshift.io/replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c",
"mirroringMode": "snapshot",
"schedulingInterval": "5m",
"clusterID": "03fa88943ffe9c61edd453f583b37e79",
},
Provisioner: util.RbdDriverName,
},
Labels: map[string]string{
"ramendr.openshift.io/replicationid": "block-pool-claim",
Expand All @@ -676,10 +684,15 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) {
"ceph-rbd-flatten-volumereplicationclass": {
Name: "ceph-rbd-flatten",
Kind: "VolumeReplicationClass",
Data: map[string]string{
"replication.storage.openshift.io/replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c",
"mirroringMode": "snapshot",
"flattenMode": "force",
Data: &replicationv1alpha1.VolumeReplicationClassSpec{
Parameters: map[string]string{
"replication.storage.openshift.io/replication-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c",
"mirroringMode": "snapshot",
"flattenMode": "force",
"schedulingInterval": "5m",
"clusterID": "03fa88943ffe9c61edd453f583b37e79",
},
Provisioner: util.RbdDriverName,
},
Labels: map[string]string{
"replication.storage.openshift.io/flatten-mode": "force",
Expand Down
7 changes: 4 additions & 3 deletions services/ux-backend/handlers/expandstorage/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ import (
"strconv"

ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
storagecluster "github.com/red-hat-storage/ocs-operator/v4/controllers/storagecluster"
"github.com/red-hat-storage/ocs-operator/v4/controllers/util"
"github.com/red-hat-storage/ocs-operator/v4/services/ux-backend/handlers"

cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
Expand Down Expand Up @@ -188,7 +189,7 @@ func createCephBlockPoolStorageClass(w http.ResponseWriter, r *http.Request, cli
"reclaimspace.csiaddons.openshift.io/schedule": "@weekly",
},
},
Provisioner: storagecluster.RbdDriverName,
Provisioner: util.RbdDriverName,
ReclaimPolicy: (*corev1.PersistentVolumeReclaimPolicy)(&reclaimPolicy),
VolumeBindingMode: (*storagev1.VolumeBindingMode)(&volumeBindingMode),
// AllowVolumeExpansion is set to true to enable expansion of OCS backed Volumes
Expand Down Expand Up @@ -260,7 +261,7 @@ func createCephFilesystemStorageClass(w http.ResponseWriter, r *http.Request, cl
"description": "Provides RWO and RWX Filesystem volumes",
},
},
Provisioner: storagecluster.CephFSDriverName,
Provisioner: util.CephFSDriverName,
ReclaimPolicy: (*corev1.PersistentVolumeReclaimPolicy)(&reclaimPolicy),
VolumeBindingMode: (*storagev1.VolumeBindingMode)(&volumeBindingMode),
// AllowVolumeExpansion is set to true to enable expansion of OCS backed Volumes
Expand Down
Loading
Loading