Merge pull request kubernetes#29272 from mksalawa/pending

Automatic merge from submit-queue

Modify predicate() interface to return all failed predicates

As stated in the comments below, this is the first step of showing the user all predicates that failed for a given node when scheduling of a given pod failed on every node.
ref kubernetes#20064

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.kubernetes.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.kubernetes.io/reviews/kubernetes/kubernetes/29272)
<!-- Reviewable:end -->

Author: Kubernetes Submit Queue

pkg/controller/daemon/daemoncontroller.go

@@ -693,7 +693,7 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *api.Node, ds *exte
 		if pod.Status.Phase == api.PodSucceeded || pod.Status.Phase == api.PodFailed {
 			continue
 		}
-		// ignore pods that belong to the daemonset when taking into account wheter
+		// ignore pods that belong to the daemonset when taking into account whether
 		// a daemonset should bind to a node.
 		if pds := dsc.getPodDaemonSet(pod); pds != nil && ds.Name == pds.Name {
 			continue
@@ -703,18 +703,12 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *api.Node, ds *exte
 
 	nodeInfo := schedulercache.NewNodeInfo(pods...)
 	nodeInfo.SetNode(node)
-	fit, err := predicates.GeneralPredicates(newPod, nil, nodeInfo)
+	fit, reasons, err := predicates.GeneralPredicates(newPod, nil, nodeInfo)
 	if err != nil {
-		if re, ok := err.(*predicates.PredicateFailureError); ok {
-			message := re.Error()
-			glog.V(2).Infof("Predicate failed on Pod: %s, for reason: %v", newPod.Name, message)
-		}
-		if re, ok := err.(*predicates.InsufficientResourceError); ok {
-			message := re.Error()
-			glog.V(2).Infof("Predicate failed on Pod: %s, for reason: %v", newPod.Name, message)
-		}
-		message := fmt.Sprintf("GeneralPredicates failed due to %v.", err)
-		glog.Warningf("Predicate failed on Pod %s - %s", newPod.Name, message)
+		glog.Warningf("GeneralPredicates failed on pod %s due to unexpected error: %v", newPod.Name, err)
+	}
+	for _, r := range reasons {
+		glog.V(2).Infof("GeneralPredicates failed on pod %s for reason: %v", newPod.Name, r.GetReason())
 	}
 	return fit
 }

pkg/kubelet/kubelet.go

@@ -2059,23 +2059,40 @@ func (kl *Kubelet) canAdmitPod(pods []*api.Pod, pod *api.Pod) (bool, string, str
 	}
 	nodeInfo := schedulercache.NewNodeInfo(pods...)
 	nodeInfo.SetNode(node)
-	fit, err := predicates.GeneralPredicates(pod, nil, nodeInfo)
+	fit, reasons, err := predicates.GeneralPredicates(pod, nil, nodeInfo)
+	if err != nil {
+		message := fmt.Sprintf("GeneralPredicates failed due to %v, which is unexpected.", err)
+		glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+		return fit, "UnexpectedError", message
+	}
 	if !fit {
-		if re, ok := err.(*predicates.PredicateFailureError); ok {
-			reason := re.PredicateName
-			message := re.Error()
+		var reason string
+		var message string
+		if len(reasons) == 0 {
+			message = fmt.Sprint("GeneralPredicates failed due to unknown reason, which is unexpected.")
+			glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+			return fit, "UnknownReason", message
+		}
+		// If there are failed predicates, we only return the first one as a reason.
+		r := reasons[0]
+		switch re := r.(type) {
+		case *predicates.PredicateFailureError:
+			reason = re.PredicateName
+			message = re.Error()
 			glog.V(2).Infof("Predicate failed on Pod: %v, for reason: %v", format.Pod(pod), message)
-			return fit, reason, message
-		}
-		if re, ok := err.(*predicates.InsufficientResourceError); ok {
-			reason := fmt.Sprintf("OutOf%s", re.ResourceName)
+		case *predicates.InsufficientResourceError:
+			reason = fmt.Sprintf("OutOf%s", re.ResourceName)
 			message := re.Error()
 			glog.V(2).Infof("Predicate failed on Pod: %v, for reason: %v", format.Pod(pod), message)
-			return fit, reason, message
+		case *predicates.FailureReason:
+			reason = re.GetReason()
+			message = fmt.Sprintf("Failure: %s", re.GetReason())
+			glog.V(2).Infof("Predicate failed on Pod: %v, for reason: %v", format.Pod(pod), message)
+		default:
+			reason = "UnexpectedPredicateFailureType"
+			message := fmt.Sprintf("GeneralPredicates failed due to %v, which is unexpected.", r)
+			glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
 		}
-		reason := "UnexpectedPredicateFailureType"
-		message := fmt.Sprintf("GeneralPredicates failed due to %v, which is unexpected.", err)
-		glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
 		return fit, reason, message
 	}
 	// TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and

plugin/pkg/scheduler/algorithm/predicates/error.go

@@ -16,13 +16,9 @@ limitations under the License.
 
 package predicates
 
-import "fmt"
-
-const (
-	podCountResourceName  string = "PodCount"
-	cpuResourceName       string = "CPU"
-	memoryResourceName    string = "Memory"
-	nvidiaGpuResourceName string = "NvidiaGpu"
+import (
+	"fmt"
+	"k8s.io/kubernetes/pkg/api"
 )
 
 var (
@@ -49,13 +45,13 @@ var (
 // hit and caused the unfitting failure.
 type InsufficientResourceError struct {
 	// resourceName is the name of the resource that is insufficient
-	ResourceName string
+	ResourceName api.ResourceName
 	requested    int64
 	used         int64
 	capacity     int64
 }
 
-func newInsufficientResourceError(resourceName string, requested, used, capacity int64) *InsufficientResourceError {
+func NewInsufficientResourceError(resourceName api.ResourceName, requested, used, capacity int64) *InsufficientResourceError {
 	return &InsufficientResourceError{
 		ResourceName: resourceName,
 		requested:    requested,
@@ -69,14 +65,34 @@ func (e *InsufficientResourceError) Error() string {
 		e.ResourceName, e.requested, e.used, e.capacity)
 }
 
+func (e *InsufficientResourceError) GetReason() string {
+	return fmt.Sprintf("Insufficient %v", e.ResourceName)
+}
+
 type PredicateFailureError struct {
 	PredicateName string
 }
 
 func newPredicateFailureError(predicateName string) *PredicateFailureError {
-	return &PredicateFailureError{predicateName}
+	return &PredicateFailureError{PredicateName: predicateName}
 }
 
 func (e *PredicateFailureError) Error() string {
 	return fmt.Sprintf("Predicate %s failed", e.PredicateName)
 }
+
+func (e *PredicateFailureError) GetReason() string {
+	return e.PredicateName
+}
+
+type FailureReason struct {
+	reason string
+}
+
+func NewFailureReason(msg string) *FailureReason {
+	return &FailureReason{reason: msg}
+}
+
+func (e *FailureReason) GetReason() string {
+	return e.reason
+}