diff --git a/controllers/mirroring/mirroring_controller.go b/controllers/mirroring/mirroring_controller.go index 6790ee07b5..9d167dad78 100644 --- a/controllers/mirroring/mirroring_controller.go +++ b/controllers/mirroring/mirroring_controller.go @@ -355,7 +355,7 @@ func (r *MirroringReconciler) reconcileBlockPoolMirroring( cephBlockPool := blockPoolByName[blockPoolName] mirroringToken := response.BlockPoolsInfo[i].MirroringToken - secretName := fmt.Sprintf("rbd-mirroring-token-%s", util.CalculateMD5Hash(blockPoolName)) + secretName := GetMirroringSecretName(blockPoolName) if mirroringToken != "" { mirroringSecret := &corev1.Secret{} mirroringSecret.Name = secretName @@ -570,3 +570,7 @@ func (r *MirroringReconciler) delete(obj client.Object, opts ...client.DeleteOpt func (r *MirroringReconciler) own(owner *corev1.ConfigMap, obj client.Object) error { return controllerutil.SetControllerReference(owner, obj, r.Scheme) } + +func GetMirroringSecretName(blockPoolName string) string { + return fmt.Sprintf("rbd-mirroring-token-%s", util.CalculateMD5Hash(blockPoolName)) +} diff --git a/services/provider/server/server.go b/services/provider/server/server.go index df3c628769..7dd0690a52 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -1,6 +1,7 @@ package server import ( + "cmp" "context" "crypto" "crypto/md5" @@ -22,6 +23,7 @@ import ( ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" pb "github.com/red-hat-storage/ocs-operator/services/provider/api/v4" + "github.com/red-hat-storage/ocs-operator/v4/controllers/mirroring" controllers "github.com/red-hat-storage/ocs-operator/v4/controllers/storageconsumer" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" "github.com/red-hat-storage/ocs-operator/v4/services" @@ -747,8 +749,6 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S return nil, err } - replicationID := req.StorageClaimName - for _, cephRes := range storageRequest.Status.CephResources { switch cephRes.Kind { case "CephClient": @@ -810,10 +810,81 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S } // SID for RamenDR - storageID := calculateCephRbdStorageId( + storageID := calculateCephRbdStorageID( cephCluster.Status.CephStatus.FSID, strconv.Itoa(blockPool.Status.PoolID), - rns.Name) + rns.Name, + ) + + peerPoolID := blockPool.GetAnnotations()[util.BlockPoolMirroringTargetIDAnnotation] + radosNamespace := cmp.Or(rns.Spec.Mirroring, &rookCephv1.RadosNamespaceMirroring{}).RemoteNamespace + if peerPoolID != "" && radosNamespace != nil { + peerCephfsid, err := getPeerCephFSID( + ctx, + s.client, + mirroring.GetMirroringSecretName(blockPool.Name), + blockPool.Namespace, + ) + if err != nil { + klog.Errorf("failed to get peer Ceph FSIS. %v", err) + } + + peerStorageID := calculateCephRbdStorageID( + peerCephfsid, + peerPoolID, + *radosNamespace, + ) + + storageIDs := []string{storageID, peerStorageID} + slices.Sort(storageIDs) + replicationID := util.CalculateMD5Hash(storageIDs) + + extR = append(extR, + &pb.ExternalResource{ + Name: fmt.Sprintf("rbd-volumereplicationclass-%v", util.FnvHash(volumeReplicationClass5mSchedule)), + Kind: "VolumeReplicationClass", + Labels: map[string]string{ + ramenDRStorageIDLabelKey: storageID, + ramenDRReplicationIDLabelKey: replicationID, + ramenMaintenanceModeLabelKey: "Failover", + }, + Annotations: map[string]string{ + "replication.storage.openshift.io/is-default-class": "true", + }, + Data: mustMarshal(&replicationv1alpha1.VolumeReplicationClassSpec{ + Parameters: map[string]string{ + "replication.storage.openshift.io/replication-secret-name": provisionerSecretName, + "mirroringMode": "snapshot", + // This is a temporary fix till we get the replication schedule to ocs-operator + "schedulingInterval": volumeReplicationClass5mSchedule, + "clusterID": clientProfile, + }, + Provisioner: util.RbdDriverName, + }), + }, + &pb.ExternalResource{ + Name: fmt.Sprintf("rbd-flatten-volumereplicationclass-%v", util.FnvHash(volumeReplicationClass5mSchedule)), + Kind: "VolumeReplicationClass", + Labels: map[string]string{ + ramenDRStorageIDLabelKey: storageID, + ramenDRReplicationIDLabelKey: replicationID, + ramenMaintenanceModeLabelKey: "Failover", + ramenDRFlattenModeLabelKey: "force", + }, + Data: mustMarshal(&replicationv1alpha1.VolumeReplicationClassSpec{ + Parameters: map[string]string{ + "replication.storage.openshift.io/replication-secret-name": provisionerSecretName, + "mirroringMode": "snapshot", + "flattenMode": "force", + // This is a temporary fix till we get the replication schedule to ocs-operator + "schedulingInterval": volumeReplicationClass5mSchedule, + "clusterID": clientProfile, + }, + Provisioner: util.RbdDriverName, + }), + }, + ) + } extR = append(extR, &pb.ExternalResource{ @@ -846,49 +917,6 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S "pool": rns.Spec.BlockPoolName, }), }, - &pb.ExternalResource{ - Name: fmt.Sprintf("rbd-volumereplicationclass-%v", util.FnvHash(volumeReplicationClass5mSchedule)), - Kind: "VolumeReplicationClass", - Labels: map[string]string{ - ramenDRStorageIDLabelKey: storageID, - ramenDRReplicationIDLabelKey: replicationID, - ramenMaintenanceModeLabelKey: "Failover", - }, - Annotations: map[string]string{ - "replication.storage.openshift.io/is-default-class": "true", - }, - Data: mustMarshal(&replicationv1alpha1.VolumeReplicationClassSpec{ - Parameters: map[string]string{ - "replication.storage.openshift.io/replication-secret-name": provisionerSecretName, - "mirroringMode": "snapshot", - // This is a temporary fix till we get the replication schedule to ocs-operator - "schedulingInterval": volumeReplicationClass5mSchedule, - "clusterID": clientProfile, - }, - Provisioner: util.RbdDriverName, - }), - }, - &pb.ExternalResource{ - Name: fmt.Sprintf("rbd-flatten-volumereplicationclass-%v", util.FnvHash(volumeReplicationClass5mSchedule)), - Kind: "VolumeReplicationClass", - Labels: map[string]string{ - ramenDRStorageIDLabelKey: storageID, - ramenDRReplicationIDLabelKey: replicationID, - ramenMaintenanceModeLabelKey: "Failover", - ramenDRFlattenModeLabelKey: "force", - }, - Data: mustMarshal(&replicationv1alpha1.VolumeReplicationClassSpec{ - Parameters: map[string]string{ - "replication.storage.openshift.io/replication-secret-name": provisionerSecretName, - "mirroringMode": "snapshot", - "flattenMode": "force", - // This is a temporary fix till we get the replication schedule to ocs-operator - "schedulingInterval": volumeReplicationClass5mSchedule, - "clusterID": clientProfile, - }, - Provisioner: util.RbdDriverName, - }), - }, &pb.ExternalResource{ Kind: "ClientProfile", Name: "ceph-rbd", @@ -932,7 +960,7 @@ func (s *OCSProviderServer) GetStorageClaimConfig(ctx context.Context, req *pb.S } // SID for RamenDR - storageID := calculateCephFsStorageId( + storageID := calculateCephFsStorageID( cephCluster.Status.CephStatus.FSID, subVolumeGroup.Spec.FilesystemName, subVolumeGroup.Name, @@ -1332,10 +1360,32 @@ func (s *OCSProviderServer) isConsumerMirrorEnabled(ctx context.Context, consume return clientMappingConfig.Data[consumer.Status.Client.ID] != "", nil } -func calculateCephRbdStorageId(cephfsid, poolID, radosnamespacename string) string { +func calculateCephRbdStorageID(cephfsid, poolID, radosnamespacename string) string { return util.CalculateMD5Hash([3]string{cephfsid, poolID, radosnamespacename}) } -func calculateCephFsStorageId(cephfsid, fileSystemName, subVolumeGroupName string) string { +func calculateCephFsStorageID(cephfsid, fileSystemName, subVolumeGroupName string) string { return util.CalculateMD5Hash([3]string{cephfsid, fileSystemName, subVolumeGroupName}) } + +func getPeerCephFSID(ctx context.Context, cl client.Client, secretName, namespace string) (string, error) { + secret := &corev1.Secret{} + secret.Name = secretName + secret.Namespace = namespace + err := cl.Get(ctx, client.ObjectKeyFromObject(secret), secret) + if err != nil { + return "", err + } + decodeString, err := base64.StdEncoding.DecodeString(string(secret.Data["token"])) + if err != nil { + return "", err + } + token := struct { + FSID string `json:"fsid"` + }{} + err = json.Unmarshal(decodeString, &token) + if err != nil { + return "", err + } + return token.FSID, nil +} diff --git a/services/provider/server/server_test.go b/services/provider/server/server_test.go index ac08ccedc9..18094b790e 100644 --- a/services/provider/server/server_test.go +++ b/services/provider/server/server_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "k8s.io/utils/ptr" "reflect" "strconv" "testing" @@ -713,7 +714,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { Name: "rbd-volumereplicationclass-1625360775", Kind: "VolumeReplicationClass", Labels: map[string]string{ - "ramendr.openshift.io/replicationid": "block-pool-claim", + "ramendr.openshift.io/replicationid": "a06c234d29cb35b6fe45fb3d7c8dd8a6", "ramendr.openshift.io/storageid": "854666c7477123fb05f20bf615e69a46", "ramendr.openshift.io/maintenancemodes": "Failover", }, @@ -735,7 +736,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { Kind: "VolumeReplicationClass", Labels: map[string]string{ "replication.storage.openshift.io/flatten-mode": "force", - "ramendr.openshift.io/replicationid": "block-pool-claim", + "ramendr.openshift.io/replicationid": "a06c234d29cb35b6fe45fb3d7c8dd8a6", "ramendr.openshift.io/storageid": "854666c7477123fb05f20bf615e69a46", "ramendr.openshift.io/maintenancemodes": "Failover", }, @@ -1135,7 +1136,8 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { Spec: rookCephv1.CephBlockPoolRadosNamespaceSpec{ BlockPoolName: "cephblockpool", Mirroring: &rookCephv1.RadosNamespaceMirroring{ - Mode: "pool", + Mode: "pool", + RemoteNamespace: ptr.To("peer-remote"), }, }, } @@ -1145,10 +1147,20 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "cephblockpool", Namespace: server.namespace, + Annotations: map[string]string{ + "ocs.openshift.io/mirroring-target-id": "3", + }, }, Spec: rookCephv1.NamedBlockPoolSpec{ PoolSpec: rookCephv1.PoolSpec{ - Mirroring: rookCephv1.MirroringSpec{Enabled: false}, + Mirroring: rookCephv1.MirroringSpec{ + Enabled: false, + Peers: &rookCephv1.MirroringPeerSpec{ + SecretNames: []string{ + "mirroring-token", + }, + }, + }, }, }, Status: &rookCephv1.CephBlockPoolStatus{ @@ -1157,6 +1169,17 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { } assert.NoError(t, client.Create(ctx, cephBlockPool)) + mirroringToken := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mirroring-token", + Namespace: server.namespace, + }, + Data: map[string][]byte{ + "token": []byte("eyJmc2lkIjoiZjk1NGYwNGItM2M1Yi00MWMxLWFkODUtNjEzNzA3ZDBjZDllIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFDYUZJNW5ZRVJrR0JBQWQzLzVQS1V3RFVuTXkxbHBMSEh2ZXc9PSIsIm1vbl9ob3N0IjoidjI6MTAuMC4xMzUuMTY5OjMzMDAvMCx2MjoxMC4wLjE1Ny4xNTY6MzMwMC8wLHYyOjEwLjAuMTcyLjExMjozMzAwLzAiLCJuYW1lc3BhY2UiOiJvcGVuc2hpZnQtc3RvcmFnZSJ9"), + }, + } + assert.NoError(t, client.Create(ctx, mirroringToken)) + // get the storage class request config for block pool req := pb.StorageClaimConfigRequest{ StorageConsumerUUID: string(consumerResource.UID),