Don't merge.

.github/workflows/kind_multicluster_e2e_tests.yaml

@@ -57,22 +57,22 @@ jobs:
     strategy:
       matrix:
         e2e_test:
-          - CreateMultiDatacenterCluster
-          - CreateMixedMultiDataCenterCluster
-          - AddDcToClusterDiffDataplane
-          - RemoveDcFromCluster
-          - CheckStargateApisWithMultiDcCluster
-          - CreateMultiStargateAndDatacenter
-          - CreateMultiReaper
-          - ClusterScoped/MultiDcMultiCluster
+          # - CreateMultiDatacenterCluster
+          # - CreateMixedMultiDataCenterCluster
+          # - AddDcToClusterDiffDataplane
+          # - RemoveDcFromCluster
+          # - CheckStargateApisWithMultiDcCluster
+          # - CreateMultiStargateAndDatacenter
+          # - CreateMultiReaper
+          # - ClusterScoped/MultiDcMultiCluster
           - CreateMultiMedusaJob
-          - MultiDcAuthOnOff
+          # - MultiDcAuthOnOff
           # TODO: these e2e tests started breaking after new client certificates were added. Needs fixing.
           #- MultiDcEncryptionWithStargate
-          - MultiDcEncryptionWithReaper
-          - StopAndRestartDc
-          - CreateMultiDatacenterDseCluster
-          - PerNodeConfig/InitialTokens
+          # - MultiDcEncryptionWithReaper
+          # - StopAndRestartDc
+          # - CreateMultiDatacenterDseCluster
+          # - PerNodeConfig/InitialTokens
       fail-fast: false
     name: ${{ matrix.e2e_test }}
     env:

CHANGELOG/CHANGELOG-1.10.md

@@ -15,6 +15,8 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 
 ## unreleased
 
+* [FEATURE] [#1080](https://github.com/k8ssandra/k8ssandra-operator/issues/1080) When a restore is completed, an annotation is added to the user secrets that will cause cass-operator to refresh them on the Cassandra side.
+
 ## v1.10.3 - 2023-11-15
 
 * [BUGFIX] [#1110](https://github.com/k8ssandra/k8ssandra-operator/issues/1110) Fix cluster name being set to "Test Cluster" when running Cassandra 4.1+

apis/k8ssandra/v1alpha1/constants.go

@@ -59,6 +59,8 @@ const (
 	K8ssandraClusterNamespaceLabel = "k8ssandra.io/cluster-namespace"
 
 	DatacenterLabel = "k8ssandra.io/datacenter"
+	// Forces refresh of secrets which relate to roles and authn in Cassandra.
+	RefreshAnnotation = "k8ssandra.io/refresh"
 )
 
 var (

controllers/medusa/medusarestorejob_controller.go

@@ -176,7 +176,14 @@ func (r *MedusaRestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		return ctrl.Result{RequeueAfter: r.DefaultDelay}, err
 	}
 
-	request.Log.Info("The restore operation is complete")
+	request.Log.Info("The restore operation is complete for DC", "CassandraDatacenter", request.Datacenter.Name)
+
+	if err = medusa.RefreshSecrets(request.Datacenter, ctx, r.Client, logger, r.DefaultDelay); err != nil {
+		request.Log.Error(err, "Failed to refresh Cassandra users in the DB")
+		// Not going to bother applying updates here since we hit an error.
+		return ctrl.Result{RequeueAfter: r.DefaultDelay}, err
+	}
+
 	return ctrl.Result{}, nil
 }
 

controllers/medusa/medusarestorejob_controller_test.go

@@ -292,8 +292,9 @@ func testMedusaRestoreDatacenter(t *testing.T, ctx context.Context, f *framework
 		return !restore.Status.FinishTime.IsZero()
 	}, timeout, interval)
 
-	err = f.DeleteK8ssandraCluster(ctx, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}, timeout, interval)
+	err = f.DeleteK8ssandraCluster(ctx, client.ObjectKey{Namespace: dc.Namespace, Name: kc.Name}, timeout, interval)
 	require.NoError(err, "failed to delete K8ssandraCluster")
+
 }
 
 func testValidationErrorStopsRestore(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) {

pkg/medusa/refresh_secrets.go

@@ -0,0 +1,46 @@
+package medusa
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/go-logr/logr"
+	cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1"
+	k8ssandraapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
+	"github.com/k8ssandra/k8ssandra-operator/pkg/reconciliation"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func RefreshSecrets(dc *cassdcapi.CassandraDatacenter, ctx context.Context, client client.Client, logger logr.Logger, requeueDelay time.Duration) error {
+	logger.Info("Refreshing secrets entered")
+	userSecrets := []string{}
+	for _, user := range dc.Spec.Users {
+		userSecrets = append(userSecrets, user.SecretName)
+	}
+	if dc.Spec.SuperuserSecretName == "" {
+		userSecrets = append(userSecrets, cassdcapi.CleanupForKubernetes(dc.Spec.ClusterName)+"-superuser") //default SU secret
+	} else {
+		userSecrets = append(userSecrets, dc.Spec.SuperuserSecretName)
+	}
+	//  Both Reaper and medusa secrets go into the userSecrets, so they don't need special handling.
+	for _, i := range userSecrets {
+		secret := &corev1.Secret{}
+		err := client.Get(ctx, types.NamespacedName{Name: i, Namespace: dc.Namespace}, secret)
+		if err != nil {
+			logger.Error(err, fmt.Sprintf("Failed to get secret %s", i))
+			return err
+		}
+		if secret.ObjectMeta.Annotations == nil {
+			secret.ObjectMeta.Annotations = make(map[string]string)
+		}
+		secret.ObjectMeta.Annotations[k8ssandraapi.RefreshAnnotation] = time.Now().String()
+		if req := reconciliation.ReconcileObject(ctx, client, requeueDelay, *secret); req.IsError() {
+			return fmt.Errorf("refreshSecret reconcile failed")
+		}
+	}
+	return nil
+
+}

pkg/medusa/refresh_secrets_test.go

@@ -0,0 +1,68 @@
+package medusa
+
+import (
+	"context"
+	"testing"
+
+	"github.com/go-logr/logr"
+	cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1"
+	"github.com/k8ssandra/k8ssandra-operator/pkg/test"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func TestRefreshSecrets_defaultSUSecret(t *testing.T) {
+	fakeClient := test.NewFakeClientWRestMapper()
+	cassDC := test.NewCassandraDatacenter("dc1", "test")
+	cassDC.Spec.Users = []cassdcapi.CassandraUser{
+		{SecretName: "custom-user"},
+	}
+	assert.NoError(t, fakeClient.Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}))
+	assert.NoError(t, fakeClient.Create(context.Background(), &cassDC))
+	secrets := []corev1.Secret{
+		{ObjectMeta: metav1.ObjectMeta{Name: "custom-user", Namespace: "test"}, Data: map[string][]byte{"username": []byte("test")}},
+		{ObjectMeta: metav1.ObjectMeta{Name: "test-cluster-superuser", Namespace: "test"}, Data: map[string][]byte{"username": []byte("test")}},
+	}
+	for _, i := range secrets {
+		assert.NoError(t, fakeClient.Create(context.Background(), &i))
+	}
+	assert.NoError(t, RefreshSecrets(&cassDC, context.Background(), fakeClient, logr.Logger{}, 0))
+	suSecret := &corev1.Secret{}
+	assert.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{Name: "test-cluster-superuser", Namespace: "test"}, suSecret))
+	_, exists := suSecret.ObjectMeta.Annotations["k8ssandra.io/refresh"]
+	assert.True(t, exists)
+	userSecret := &corev1.Secret{}
+	assert.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{Name: "custom-user", Namespace: "test"}, userSecret))
+	_, exists = userSecret.ObjectMeta.Annotations["k8ssandra.io/refresh"]
+	assert.True(t, exists)
+}
+
+func TestRefreshSecrets_customSecrets(t *testing.T) {
+	fakeClient := test.NewFakeClientWRestMapper()
+	cassDC := test.NewCassandraDatacenter("dc1", "test")
+	cassDC.Spec.Users = []cassdcapi.CassandraUser{
+		{SecretName: "custom-user"},
+	}
+	cassDC.Spec.SuperuserSecretName = "cass-custom-superuser"
+	assert.NoError(t, fakeClient.Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}))
+	assert.NoError(t, fakeClient.Create(context.Background(), &cassDC))
+	secrets := []corev1.Secret{
+		{ObjectMeta: metav1.ObjectMeta{Name: "custom-user", Namespace: "test"}},
+		{ObjectMeta: metav1.ObjectMeta{Name: "cass-custom-superuser", Namespace: "test"}},
+	}
+	for _, i := range secrets {
+		assert.NoError(t, fakeClient.Create(context.Background(), &i))
+	}
+	assert.NoError(t, RefreshSecrets(&cassDC, context.Background(), fakeClient, logr.Logger{}, 0))
+	suSecret := &corev1.Secret{}
+	assert.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{Name: "cass-custom-superuser", Namespace: "test"}, suSecret))
+	_, exists := suSecret.ObjectMeta.Annotations["k8ssandra.io/refresh"]
+	assert.True(t, exists)
+	userSecret := &corev1.Secret{}
+	assert.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{Name: "custom-user", Namespace: "test"}, userSecret))
+	_, exists = userSecret.ObjectMeta.Annotations["k8ssandra.io/refresh"]
+	assert.True(t, exists)
+
+}

test/e2e/medusa_test.go

@@ -7,6 +7,7 @@ import (
 
 	cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1"
 	api "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
+	k8ssandraapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
 	medusa "github.com/k8ssandra/k8ssandra-operator/apis/medusa/v1alpha1"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/cassandra"
 	medusapkg "github.com/k8ssandra/k8ssandra-operator/pkg/medusa"
@@ -195,6 +196,31 @@ func verifyRestoreJobFinished(t *testing.T, ctx context.Context, f *framework.E2
 
 		return !restore.Status.FinishTime.IsZero()
 	}, polling.medusaRestoreDone.timeout, polling.medusaRestoreDone.interval, "restore didn't finish within timeout")
+
+	require.Eventually(func() bool {
+		dc := &cassdcapi.CassandraDatacenter{}
+		err := f.Get(ctx, dcKey, dc)
+		if err != nil {
+			t.Log(err)
+			return false
+		}
+		superUserSecret := dc.Spec.SuperuserSecretName
+		if dc.Spec.SuperuserSecretName == "" {
+			superUserSecret = cassdcapi.CleanupForKubernetes(dcKey.Name) + "-superuser"
+		}
+		secret := &corev1.Secret{}
+		err = f.Get(ctx, framework.NewClusterKey(restoreClusterKey.K8sContext, restoreClusterKey.Namespace, superUserSecret), secret)
+		if err != nil {
+			t.Log(err)
+			return false
+		}
+		_, exists := secret.Annotations[k8ssandraapi.RefreshAnnotation]
+		if !exists {
+			t.Logf("%#v", *secret)
+		}
+		return exists
+	}, polling.medusaRestoreDone.timeout, polling.medusaRestoreDone.interval, "superuser secret wasn't updated with refresh annotation")
+
 }
 
 func checkMedusaStandaloneDeploymentExists(t *testing.T, ctx context.Context, dcKey framework.ClusterKey, f *framework.E2eFramework, kc *api.K8ssandraCluster) {