Allow restart to continue if observability index is yellow (#582)

Fixes #581

opensearch-operator/opensearch-gateway/responses/ClusterHealthResponse.go

@@ -1,10 +1,22 @@
 package responses
 
 type ClusterHealthResponse struct {
-	Status             string  `json:"status,omitempty"`
-	ActiveShards       int     `json:"active_shards,omitempty"`
-	RelocatingShards   int     `json:"relocating_shards,omitempty"`
-	InitializingShards int     `json:"initializing_shards,omitempty"`
-	UnassignedShards   int     `json:"unassigned_shards,omitempty"`
-	PercentActive      float32 `json:"active_shards_percent_as_number,omitempty"`
+	Status             string                 `json:"status,omitempty"`
+	ActiveShards       int                    `json:"active_shards,omitempty"`
+	RelocatingShards   int                    `json:"relocating_shards,omitempty"`
+	InitializingShards int                    `json:"initializing_shards,omitempty"`
+	UnassignedShards   int                    `json:"unassigned_shards,omitempty"`
+	PercentActive      float32                `json:"active_shards_percent_as_number,omitempty"`
+	Indices            map[string]IndexHealth `json:"indices,omitempty"`
+}
+
+type IndexHealth struct {
+	Status              string `json:"status"`
+	NumberOfShards      int    `json:"number_of_shards"`
+	NumberOfReplicas    int    `json:"number_of_replicas"`
+	ActivePrimaryShards int    `json:"active_primary_shards"`
+	ActiveShards        int    `json:"active_shards"`
+	RelocatingShards    int    `json:"relocating_shards"`
+	InitializingShards  int    `json:"initializing_shards"`
+	UnassignedShards    int    `json:"unassigned_shards"`
 }

opensearch-operator/opensearch-gateway/services/os_client.go

@@ -118,10 +118,12 @@ func MainPage(client *opensearch.Client) (responses.MainResponse, error) {
 	return response, err
 }
 
-func (client *OsClusterClient) GetHealth() (responses.CatHealthResponse, error) {
-	req := opensearchapi.ClusterHealthRequest{}
+func (client *OsClusterClient) GetHealth() (responses.ClusterHealthResponse, error) {
+	req := opensearchapi.ClusterHealthRequest{
+		Level: "indices",
+	}
 	catNodesRes, err := req.Do(context.Background(), client.client)
-	var response responses.CatHealthResponse
+	var response responses.ClusterHealthResponse
 	if err == nil {
 		defer catNodesRes.Body.Close()
 		err = json.NewDecoder(catNodesRes.Body).Decode(&response)

opensearch-operator/opensearch-gateway/services/os_data_service.go

@@ -157,6 +157,10 @@ func CheckClusterStatusForRestart(service *OsClusterClient, drainNodes bool) (bo
 		return true, "", nil
 	}
 
+	if continueRestartWithYellowHealth(health) {
+		return true, "", nil
+	}
+
 	if drainNodes {
 		return false, "cluster is not green and drain nodes is enabled", nil
 	}
@@ -250,3 +254,23 @@ func GetExistingSystemIndices(service *OsClusterClient) ([]string, error) {
 
 	return existing, nil
 }
+
+// continueRestartWithYellowHealth allows upgrades and rolling restarts to continue when the cluster is yellow
+// if the yellow status is caused by the .opensearch-observability index.  This is a new index that is created
+// on upgrade and will be yellow until at least 2 data nodes are upgraded.
+func continueRestartWithYellowHealth(health responses.ClusterHealthResponse) bool {
+	if health.Status != "yellow" {
+		return false
+	}
+
+	if health.RelocatingShards > 0 || health.InitializingShards > 0 || health.UnassignedShards > 1 {
+		return false
+	}
+
+	observabilityIndex, ok := health.Indices[".opensearch-observability"]
+	if !ok {
+		return false
+	}
+
+	return observabilityIndex.Status == "yellow"
+}