Skip to content

Commit

Permalink
node audience restriction: use csi translator to convert intree inlin…
Browse files Browse the repository at this point in the history 
…e_vol/pv to csi

Signed-off-by: Anish Ramasekar <[email protected]>
  • Loading branch information
@aramase
aramase committed Feb 6, 2025
1 parent d6c50c3 commit 62809dd
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 27 deletions.
43 changes: 34 additions & 9 deletions plugin/pkg/admission/noderestriction/admission.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ import (
storagelisters "k8s.io/client-go/listers/storage/v1"
"k8s.io/component-base/featuregate"
"k8s.io/component-helpers/storage/ephemeral"
csitrans "k8s.io/csi-translation-lib"
"k8s.io/klog/v2"
kubeletapis "k8s.io/kubelet/pkg/apis"
podutil "k8s.io/kubernetes/pkg/api/pod"
authenticationapi "k8s.io/kubernetes/pkg/apis/authentication"
Expand Down Expand Up @@ -80,6 +82,7 @@ type Plugin struct {
csiDriverGetter storagelisters.CSIDriverLister
pvcGetter corev1lister.PersistentVolumeClaimLister
pvGetter corev1lister.PersistentVolumeLister
csiTranslator csitrans.CSITranslator

expansionRecoveryEnabled bool
dynamicResourceAllocationEnabled bool
Expand Down Expand Up @@ -109,6 +112,7 @@ func (p *Plugin) SetExternalKubeInformerFactory(f informers.SharedInformerFactor
p.csiDriverGetter = f.Storage().V1().CSIDrivers().Lister()
p.pvcGetter = f.Core().V1().PersistentVolumeClaims().Lister()
p.pvGetter = f.Core().V1().PersistentVolumes().Lister()
p.csiTranslator = csitrans.New()
}
}

Expand Down Expand Up @@ -189,7 +193,7 @@ func (p *Plugin) Admit(ctx context.Context, a admission.Attributes, o admission.
}

case svcacctResource:
return p.admitServiceAccount(nodeName, a)
return p.admitServiceAccount(ctx, nodeName, a)

case leaseResource:
return p.admitLease(nodeName, a)
Expand Down Expand Up @@ -581,7 +585,7 @@ func (p *Plugin) getForbiddenLabels(modifiedLabels sets.String) sets.String {
return forbiddenLabels
}

func (p *Plugin) admitServiceAccount(nodeName string, a admission.Attributes) error {
func (p *Plugin) admitServiceAccount(ctx context.Context, nodeName string, a admission.Attributes) error {
if a.GetOperation() != admission.Create {
return nil
}
Expand Down Expand Up @@ -620,7 +624,7 @@ func (p *Plugin) admitServiceAccount(nodeName string, a admission.Attributes) er
}

if p.serviceAccountNodeAudienceRestriction {
if err := p.validateNodeServiceAccountAudience(tr, pod); err != nil {
if err := p.validateNodeServiceAccountAudience(ctx, tr, pod); err != nil {
return admission.NewForbidden(a, err)
}
}
Expand All @@ -634,7 +638,7 @@ func (p *Plugin) admitServiceAccount(nodeName string, a admission.Attributes) er
return nil
}

func (p *Plugin) validateNodeServiceAccountAudience(tr *authenticationapi.TokenRequest, pod *v1.Pod) error {
func (p *Plugin) validateNodeServiceAccountAudience(ctx context.Context, tr *authenticationapi.TokenRequest, pod *v1.Pod) error {
// ensure all items in tr.Spec.Audiences are present in a volume mount in the pod
requestedAudience := ""
switch len(tr.Spec.Audiences) {
Expand All @@ -646,7 +650,7 @@ func (p *Plugin) validateNodeServiceAccountAudience(tr *authenticationapi.TokenR
return fmt.Errorf("node may only request 0 or 1 audiences")
}

foundAudiencesInPodSpec, err := p.podReferencesAudience(pod, requestedAudience)
foundAudiencesInPodSpec, err := p.podReferencesAudience(ctx, pod, requestedAudience)
if err != nil {
return fmt.Errorf("error validating audience %q: %w", requestedAudience, err)
}
Expand All @@ -656,7 +660,7 @@ func (p *Plugin) validateNodeServiceAccountAudience(tr *authenticationapi.TokenR
return nil
}

func (p *Plugin) podReferencesAudience(pod *v1.Pod, audience string) (bool, error) {
func (p *Plugin) podReferencesAudience(ctx context.Context, pod *v1.Pod, audience string) (bool, error) {
var errs []error

for _, v := range pod.Spec.Volumes {
Expand All @@ -677,11 +681,20 @@ func (p *Plugin) podReferencesAudience(pod *v1.Pod, audience string) (bool, erro
switch {
case v.Ephemeral != nil && v.Ephemeral.VolumeClaimTemplate != nil:
pvcName := ephemeral.VolumeClaimName(pod, &v)
driverName, err = p.getCSIFromPVC(pod.Namespace, pvcName)
driverName, err = p.getCSIFromPVC(ctx, pod.Namespace, pvcName)
case v.PersistentVolumeClaim != nil:
driverName, err = p.getCSIFromPVC(pod.Namespace, v.PersistentVolumeClaim.ClaimName)
driverName, err = p.getCSIFromPVC(ctx, pod.Namespace, v.PersistentVolumeClaim.ClaimName)
case v.CSI != nil:
driverName = v.CSI.Driver
case p.csiTranslator.IsInlineMigratable(&v):
pv, translateErr := p.csiTranslator.TranslateInTreeInlineVolumeToCSI(klog.FromContext(ctx), &v, pod.Namespace)
if translateErr != nil {
err = translateErr
break
}
if pv != nil && pv.Spec.CSI != nil {
driverName = pv.Spec.CSI.Driver
}
}

if err != nil {
Expand All @@ -705,7 +718,7 @@ func (p *Plugin) podReferencesAudience(pod *v1.Pod, audience string) (bool, erro
}

// getCSIFromPVC returns the CSI driver name from the PVC->PV->CSI->Driver chain
func (p *Plugin) getCSIFromPVC(namespace, claimName string) (string, error) {
func (p *Plugin) getCSIFromPVC(ctx context.Context, namespace, claimName string) (string, error) {
pvc, err := p.pvcGetter.PersistentVolumeClaims(namespace).Get(claimName)
if err != nil {
return "", err
Expand All @@ -717,6 +730,18 @@ func (p *Plugin) getCSIFromPVC(namespace, claimName string) (string, error) {
if pv.Spec.CSI != nil {
return pv.Spec.CSI.Driver, nil
}

if p.csiTranslator.IsPVMigratable(pv) {
// For in-tree PV, we need to convert ("translate") the PV to CSI before checking the driver name.
translatedPV, err := p.csiTranslator.TranslateInTreePVToCSI(klog.FromContext(ctx), pv)
if err != nil {
return "", err
}
if translatedPV != nil && translatedPV.Spec.CSI != nil {
return translatedPV.Spec.CSI.Driver, nil
}
}

return "", nil
}

Expand Down
73 changes: 55 additions & 18 deletions test/integration/auth/node_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -908,6 +908,27 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) {
}, metav1.CreateOptions{})
checkNilError(t, err)

_, err = superuserClient.CoreV1().PersistentVolumeClaims("ns").Create(context.TODO(), &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Name: "mypvc-azurefile"},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany},
Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1")}},
VolumeName: "mypv-azurefile",
},
}, metav1.CreateOptions{})
checkNilError(t, err)

_, err = superuserClient.CoreV1().PersistentVolumes().Create(context.TODO(), &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{Name: "mypv-azurefile"},
Spec: corev1.PersistentVolumeSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany},
Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1")},
ClaimRef: &corev1.ObjectReference{Namespace: "ns", Name: "mypvc-azurefile"},
PersistentVolumeSource: corev1.PersistentVolumeSource{AzureFile: &corev1.AzureFilePersistentVolumeSource{ShareName: "share", SecretName: "secret"}},
},
}, metav1.CreateOptions{})
checkNilError(t, err)

node1Client, _ := clientsetForToken(tokenNode1, clientConfig)
createNode(t, node1Client, "node1")
createDefaultServiceAccount(t, superuserClient)
Expand All @@ -934,12 +955,12 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) {
})

t.Run("pod --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) {
createCSIDriver(t, superuserClient, "csidriver-audience")
createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver")
csiDriverVolumeSource := &corev1.CSIVolumeSource{Driver: "com.example.csi.mydriver"}
pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{CSI: csiDriverVolumeSource}}})
expectAllowed(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience"))
deletePod(t, superuserClient, "pod1")
deleteCSIDriver(t, superuserClient)
deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver")
})

t.Run("pod --> pvc --> pv --> csi --> driver --> tokenrequest with audience forbidden - CSI driver not found", func(t *testing.T) {
Expand All @@ -950,21 +971,21 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) {
})

t.Run("pod --> pvc --> pv --> csi --> driver --> tokenrequest with audience forbidden - pvc not found", func(t *testing.T) {
createCSIDriver(t, superuserClient, "pvcnotfound-audience")
createCSIDriver(t, superuserClient, "pvcnotfound-audience", "com.example.csi.mydriver")
persistentVolumeClaimVolumeSource := &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc1"}
pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: persistentVolumeClaimVolumeSource}}})
expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "pvcnotfound-audience"), `error validating audience "pvcnotfound-audience": persistentvolumeclaim "mypvc1" not found`)
deletePod(t, superuserClient, "pod1")
deleteCSIDriver(t, superuserClient)
deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver")
})

t.Run("pod --> pvc --> pv --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) {
createCSIDriver(t, superuserClient, "pvccsidriver-audience")
createCSIDriver(t, superuserClient, "pvccsidriver-audience", "com.example.csi.mydriver")
persistentVolumeClaimVolumeSource := &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc"}
pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: persistentVolumeClaimVolumeSource}}})
expectAllowed(t, createTokenRequest(node1Client, pod.UID, "pvccsidriver-audience"))
deletePod(t, superuserClient, "pod1")
deleteCSIDriver(t, superuserClient)
deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver")
})

t.Run("pod --> ephemeral --> pvc --> pv --> csi --> driver --> tokenrequest with audience forbidden - CSI driver not found", func(t *testing.T) {
Expand All @@ -980,7 +1001,7 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) {
})

t.Run("pod --> ephemeral --> pvc --> pv --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) {
createCSIDriver(t, superuserClient, "ephemeralcsidriver-audience")
createCSIDriver(t, superuserClient, "ephemeralcsidriver-audience", "com.example.csi.mydriver")
ephemeralVolumeSource := &corev1.EphemeralVolumeSource{VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany},
Expand All @@ -990,28 +1011,28 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) {
pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{Ephemeral: ephemeralVolumeSource}}})
expectAllowed(t, createTokenRequest(node1Client, pod.UID, "ephemeralcsidriver-audience"))
deletePod(t, superuserClient, "pod1")
deleteCSIDriver(t, superuserClient)
deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver")
})

t.Run("csidriver exists but tokenrequest audience not found should be forbidden", func(t *testing.T) {
createCSIDriver(t, superuserClient, "csidriver-audience")
createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver")
pod := createPod(t, superuserClient, nil)
expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience-not-found"), `audience "csidriver-audience-not-found" not found in pod spec volume`)
deletePod(t, superuserClient, "pod1")
deleteCSIDriver(t, superuserClient)
deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver")
})

t.Run("pvc and csidriver exists but tokenrequest audience not found should be forbidden", func(t *testing.T) {
createCSIDriver(t, superuserClient, "csidriver-audience")
createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver")
persistentVolumeClaimVolumeSource := &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc"}
pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: persistentVolumeClaimVolumeSource}}})
expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience-not-found"), `audience "csidriver-audience-not-found" not found in pod spec volume`)
deletePod(t, superuserClient, "pod1")
deleteCSIDriver(t, superuserClient)
deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver")
})

t.Run("ephemeral volume source with audience not found should be forbidden", func(t *testing.T) {
createCSIDriver(t, superuserClient, "csidriver-audience")
createCSIDriver(t, superuserClient, "csidriver-audience", "com.example.csi.mydriver")
ephemeralVolumeSource := &corev1.EphemeralVolumeSource{VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany},
Expand All @@ -1021,7 +1042,23 @@ func TestNodeRestrictionServiceAccountAudience(t *testing.T) {
pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{Ephemeral: ephemeralVolumeSource}}})
expectedForbiddenMessage(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience-not-found"), `audience "csidriver-audience-not-found" not found in pod spec volume`)
deletePod(t, superuserClient, "pod1")
deleteCSIDriver(t, superuserClient)
deleteCSIDriver(t, superuserClient, "com.example.csi.mydriver")
})

t.Run("intree pv to csi migration, pod --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) {
createCSIDriver(t, superuserClient, "csidriver-audience", "file.csi.azure.com")
pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "mypvc-azurefile"}}}})
expectAllowed(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience"))
deletePod(t, superuserClient, "pod1")
deleteCSIDriver(t, superuserClient, "file.csi.azure.com")
})

t.Run("intree inline volume to csi migration, pod --> csi --> driver --> tokenrequest with audience works", func(t *testing.T) {
createCSIDriver(t, superuserClient, "csidriver-audience", "file.csi.azure.com")
pod := createPod(t, superuserClient, []corev1.Volume{{Name: "foo", VolumeSource: corev1.VolumeSource{AzureFile: &corev1.AzureFileVolumeSource{ShareName: "default", SecretName: "mypvsecret"}}}})
expectAllowed(t, createTokenRequest(node1Client, pod.UID, "csidriver-audience"))
deletePod(t, superuserClient, "pod1")
deleteCSIDriver(t, superuserClient, "file.csi.azure.com")
})

t.Run("token request with multiple audiences should be forbidden", func(t *testing.T) {
Expand Down Expand Up @@ -1090,22 +1127,22 @@ func createTokenRequest(client clientset.Interface, uid types.UID, audiences ...
}
}

func createCSIDriver(t *testing.T, client clientset.Interface, audience string) {
func createCSIDriver(t *testing.T, client clientset.Interface, audience, driverName string) {
t.Helper()

_, err := client.StorageV1().CSIDrivers().Create(context.TODO(), &storagev1.CSIDriver{
ObjectMeta: metav1.ObjectMeta{Name: "com.example.csi.mydriver"},
ObjectMeta: metav1.ObjectMeta{Name: driverName},
Spec: storagev1.CSIDriverSpec{
TokenRequests: []storagev1.TokenRequest{{Audience: audience}},
},
}, metav1.CreateOptions{})
checkNilError(t, err)
}

func deleteCSIDriver(t *testing.T, client clientset.Interface) {
func deleteCSIDriver(t *testing.T, client clientset.Interface, driverName string) {
t.Helper()

checkNilError(t, client.StorageV1().CSIDrivers().Delete(context.TODO(), "com.example.csi.mydriver", metav1.DeleteOptions{GracePeriodSeconds: ptr.To[int64](0)}))
checkNilError(t, client.StorageV1().CSIDrivers().Delete(context.TODO(), driverName, metav1.DeleteOptions{GracePeriodSeconds: ptr.To[int64](0)}))
}

func createDefaultServiceAccount(t *testing.T, client clientset.Interface) {
Expand Down

0 comments on commit 62809dd

Please sign in to comment.