BB-171210: fetch Pod from API server when not present in cache

k8s/k8s.go

@@ -90,7 +90,7 @@ func (k8s *Client) ListNamespaces() []string {
 func (k8s *Client) PodByIP(IP string) (*v1.Pod, error) {
 	pods, err := k8s.podIndexer.ByIndex(podIPIndexName, IP)
 	if err != nil {
-		return nil, err
+		return getPodFromK8s(k8s, IP)
 	}
 
 	if len(pods) == 0 {
@@ -116,6 +116,21 @@ func (k8s *Client) PodByIP(IP string) (*v1.Pod, error) {
 	return pod, nil
 }
 
+func getPodFromK8s(k8s *Client, IP string) (*v1.Pod, error) {
+	runningPodList, err := k8s.CoreV1().Pods("").List(metav1.ListOptions{
+		FieldSelector: selector.OneTermEqualSelector("status.podIP", IP).String(),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("getPodFromK8s: error retriving the pod with IP %s from the k8s api", IP)
+	}
+
+	for _, pod := range runningPodList.Items {
+		return &pod, nil
+	}
+
+	return nil, fmt.Errorf("getPodFromK8s: error received empty pods respone from the k8s api %s", IP)
+}
+
 // resolveDuplicatedIP queries the k8s api server trying to make a decision based on NON cached data
 // If the indexed pods all have HostNetwork = true the function return nil and the error message.
 // If we retrive a running pod that doesn't have HostNetwork = true and it is in Running state will return that.

server/server.go

@@ -36,15 +36,15 @@ const (
 	// Choosing the larger value for max elasped time will have the impact on downstream API latency
 	// The default EC2 metadata timeout is 1 second, hence choosing the value less than 1 second
 	// The downstream API will by default retries 3 times
-	defaultMaxElapsedTime = 500 * time.Millisecond
+	defaultMaxElapsedTime = 9000 * time.Millisecond
 
 	defaultIAMRoleSessionTTL = 15 * time.Minute
 
 	// The max interval specifies the operation to be completed with in this time period
 	// The shared informer has to populate the cache with in this interval based on the Pod events
 	// If the Pod event is not received, the operation will error out and
 	// the exponential backoff will retry untill the max elasped time
-	defaultMaxInterval = 100 * time.Millisecond
+	defaultMaxInterval = 200 * time.Millisecond
 
 	defaultMetadataAddress            = "169.254.169.254"
 	defaultNamespaceKey               = "iam.amazonaws.com/allowed-roles"
@@ -337,11 +337,11 @@ func (s *Server) roleHandler(logger *log.Entry, w http.ResponseWriter, r *http.R
 		return
 	}
 
-	externalID, err := s.getExternalIDMapping(remoteIP)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusNotFound)
-		return
-	}
+	// externalID, err := s.getExternalIDMapping(remoteIP)
+	// if err != nil {
+	// 	http.Error(w, err.Error(), http.StatusNotFound)
+	// 	return
+	// }
 
 	roleLogger := logger.WithFields(log.Fields{
 		"pod.iam.role": roleMapping.Role,
@@ -358,7 +358,7 @@ func (s *Server) roleHandler(logger *log.Entry, w http.ResponseWriter, r *http.R
 		return
 	}
 
-	credentials, err := s.iam.AssumeRole(wantedRoleARN, externalID, remoteIP, s.IAMRoleSessionTTL)
+	credentials, err := s.iam.AssumeRole(wantedRoleARN, "", remoteIP, s.IAMRoleSessionTTL)
 	if err != nil {
 		roleLogger.Errorf("Error assuming role %+v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)