ceph: append additional info in the rbd-mirror bootstrap peer token

We know append additional information to the rbd-mirror bootstrap peer
token. It is useful for disaster recovery scenario where the other
cluster is reading the peer token and needs to know the pool_id as well
as the namespace.

Signed-off-by: Sébastien Han <[email protected]>

pkg/operator/ceph/cluster/rbd/config.go

@@ -50,14 +50,6 @@ type daemonConfig struct {
 	ownerInfo    *k8sutil.OwnerInfo
 }
 
-// PeerToken is the content of the peer token
-type PeerToken struct {
-	ClusterFSID string `json:"fsid"`
-	ClientID    string `json:"client_id"`
-	Key         string `json:"key"`
-	MonHost     string `json:"mon_host"`
-}
-
 func (r *ReconcileCephRBDMirror) generateKeyring(clusterInfo *client.ClusterInfo, daemonConfig *daemonConfig) (string, error) {
 	ctx := context.TODO()
 	user := fullDaemonName(daemonConfig.DaemonID)

pkg/operator/ceph/controller/mirror_peer.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2021 The Rook Authors. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+	http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package controller provides Kubernetes controller/pod/container spec items used for many Ceph daemons
+package controller
+
+// PeerToken is the content of the peer token
+type PeerToken struct {
+	ClusterFSID string `json:"fsid"`
+	ClientID    string `json:"client_id"`
+	Key         string `json:"key"`
+	MonHost     string `json:"mon_host"`
+	// These fields are added by Rook and NOT part of the output of client.CreateRBDMirrorBootstrapPeer()
+	PoolID    int    `json:"pool_id"`
+	Namespace string `json:"namespace"`
+}

pkg/operator/ceph/file/mirror/config.go

@@ -46,14 +46,6 @@ type daemonConfig struct {
 	ownerInfo    *k8sutil.OwnerInfo
 }
 
-// PeerToken is the content of the peer token
-type PeerToken struct {
-	ClusterFSID string `json:"fsid"`
-	ClientID    string `json:"client_id"`
-	Key         string `json:"key"`
-	MonHost     string `json:"mon_host"`
-}
-
 func (r *ReconcileFilesystemMirror) generateKeyring(clusterInfo *client.ClusterInfo, daemonConfig *daemonConfig) (string, error) {
 	access := []string{
 		"mon", "allow r",

pkg/operator/ceph/pool/config.go

@@ -18,6 +18,8 @@ limitations under the License.
 package pool
 
 import (
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
 
 	"github.com/pkg/errors"
@@ -47,8 +49,14 @@ func (r *ReconcileCephBlockPool) createBootstrapPeerSecret(cephBlockPool *cephv1
 		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create rbd-mirror bootstrap peer")
 	}
 
+	// Add additional information to the peer token
+	expandedPeerToken, err := r.expandBootstrapPeerToken(cephBlockPool, namespacedName, boostrapToken)
+	if err != nil {
+		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to add extra information to rbd-mirror bootstrap peer")
+	}
+
 	// Generate and create a Kubernetes Secret with this token
-	s := GenerateBootstrapPeerSecret(cephBlockPool.Name, cephBlockPool.Namespace, boostrapToken)
+	s := GenerateBootstrapPeerSecret(cephBlockPool.Name, cephBlockPool.Namespace, expandedPeerToken)
 
 	// set ownerref to the Secret
 	err = controllerutil.SetControllerReference(cephBlockPool, s, r.scheme)
@@ -93,3 +101,37 @@ func generateStatusInfo(p *cephv1.CephBlockPool) map[string]string {
 	m[RBDMirrorBootstrapPeerSecretName] = buildBoostrapPeerSecretName(p.Name)
 	return m
 }
+
+func (r *ReconcileCephBlockPool) expandBootstrapPeerToken(cephBlockPool *cephv1.CephBlockPool, namespacedName types.NamespacedName, token []byte) ([]byte, error) {
+	// First decode the token, it's base64 encoded
+	decodedToken, err := base64.StdEncoding.DecodeString(string(token))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to decode bootstrap peer token")
+	}
+
+	// Unmarshal the decoded value to a Go type
+	var decodedTokenToGo opcontroller.PeerToken
+	err = json.Unmarshal(decodedToken, &decodedTokenToGo)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal decoded token")
+	}
+
+	// Fetch the pool ID
+	poolDetails, err := cephclient.GetPoolDetails(r.context, r.clusterInfo, cephBlockPool.Name)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get pool %q details", cephBlockPool.Name)
+	}
+
+	// Add extra details to the token
+	decodedTokenToGo.PoolID = poolDetails.Number
+	decodedTokenToGo.Namespace = namespacedName.Namespace
+
+	// Marshal the Go type back to JSON
+	decodedTokenBackToJSON, err := json.Marshal(decodedTokenToGo)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to encode go type back to json")
+	}
+
+	// Return the base64 encoded token
+	return []byte(base64.StdEncoding.EncodeToString(decodedTokenBackToJSON)), nil
+}

pkg/operator/ceph/pool/config_test.go

@@ -18,11 +18,21 @@ limitations under the License.
 package pool
 
 import (
+	"encoding/base64"
+	"reflect"
 	"testing"
 
+	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
+	"github.com/rook/rook/pkg/clusterd"
+	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
+	testop "github.com/rook/rook/pkg/operator/test"
+	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
+	"github.com/tevino/abool"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 func TestGenerateStatusInfo(t *testing.T) {
@@ -38,3 +48,34 @@ func TestGenerateStatusInfo(t *testing.T) {
 	assert.NotEmpty(t, secretName)
 	assert.Equal(t, "pool-peer-token-foo", secretName)
 }
+
+func TestExpandBootstrapPeerToken(t *testing.T) {
+	executor := &exectest.MockExecutor{
+		MockExecuteCommandWithOutputFile: func(command, outfile string, args ...string) (string, error) {
+			if reflect.DeepEqual(args[0:5], []string{"osd", "pool", "get", "pool", "all"}) {
+				return `{"pool_id":13}`, nil
+			}
+
+			return "", errors.Errorf("unknown command args: %s", args[0:5])
+		},
+	}
+	c := &clusterd.Context{
+		Executor:                   executor,
+		Clientset:                  testop.New(t, 1),
+		RookClientset:              rookclient.NewSimpleClientset(),
+		RequestCancelOrchestration: abool.New(),
+	}
+
+	// Create a ReconcileCephBlockPool object with the scheme and fake client.
+	r := &ReconcileCephBlockPool{
+		context:     c,
+		clusterInfo: &cephclient.ClusterInfo{Namespace: "rook-ceph"},
+	}
+
+	newToken, err := r.expandBootstrapPeerToken(&cephv1.CephBlockPool{ObjectMeta: v1.ObjectMeta{Name: "pool", Namespace: "rook-ceph"}}, types.NamespacedName{Namespace: "rook-ceph"}, []byte(`eyJmc2lkIjoiYzZiMDg3ZjItNzgyOS00ZGJiLWJjZmMtNTNkYzM0ZTBiMzVkIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBV1lsWmZVQ1Q2RGhBQVBtVnAwbGtubDA5YVZWS3lyRVV1NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMTExLjEwOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTA6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjEyOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTI6Njc4OV0sW3YyOjE5Mi4xNjguMTExLjExOjMzMDAsdjE6MTkyLjE2OC4xMTEuMTE6Njc4OV0ifQ==`))
+	assert.NoError(t, err)
+	newTokenDecoded, err := base64.StdEncoding.DecodeString(string(newToken))
+	assert.NoError(t, err)
+	assert.Contains(t, string(newTokenDecoded), "pool_id")
+	assert.Contains(t, string(newTokenDecoded), "namespace")
+}