Dont fail on the unresolved hostname (#3024)

If we have an unresolvable hostname from property store data node configs, we will fail helix cluster manager and kill all the frontend and server instance when booting up the process. Let's don't do this. We should create an alert for those hosts.

Author: justinlin-linkedin

ambry-clustermap/src/main/java/com/github/ambry/clustermap/HelixClusterManager.java

@@ -124,6 +124,7 @@ public class HelixClusterManager implements ClusterMap {
   private final AtomicLong currentXid;
   final HelixClusterManagerMetrics helixClusterManagerMetrics;
   private HelixAggregatedViewClusterInfo helixAggregatedViewClusterInfo = null;
+  private final AtomicLong dataNodeInitializationFailureCount = new AtomicLong(0);
 
   // The map from resource name to resource config, This is only used in FULL AUTO. This map is not going to be updated
   // if the ResourceConfig is updated, but we are only using default replica capacity from the ResourceConfig. So if you
@@ -230,7 +231,7 @@ public HelixClusterManager(ClusterMapConfig clusterMapConfig, String instanceNam
           initializationFailureMap.values().stream().filter(Objects::nonNull).count());
       helixClusterManagerMetrics.initializeXidMetric(currentXid);
       helixClusterManagerMetrics.initializeDatacenterMetrics();
-      helixClusterManagerMetrics.initializeDataNodeMetrics();
+      helixClusterManagerMetrics.initializeDataNodeMetrics(dataNodeInitializationFailureCount);
       helixClusterManagerMetrics.initializeDiskMetrics();
       helixClusterManagerMetrics.initializePartitionMetrics();
       helixClusterManagerMetrics.initializeCapacityMetrics();
@@ -2077,10 +2078,18 @@ private void updateReplicaStateAndOverrideIfNeeded(AmbryReplica replica, Collect
     private List<ReplicaId> createNewInstance(DataNodeConfig dataNodeConfig, String dcName) throws Exception {
       String instanceName = dataNodeConfig.getInstanceName();
       logger.info("Adding node {} and its disks and replicas in {}", instanceName, dcName);
-      AmbryDataNode datanode =
-          new AmbryServerDataNode(dataNodeConfig.getDatacenterName(), clusterMapConfig, dataNodeConfig.getHostName(),
-              dataNodeConfig.getPort(), dataNodeConfig.getRackId(), dataNodeConfig.getSslPort(),
-              dataNodeConfig.getHttp2Port(), DEFAULT_XID, helixClusterManagerQueryHelper);
+      AmbryDataNode datanode = null;
+      try {
+        datanode =
+            new AmbryServerDataNode(dataNodeConfig.getDatacenterName(), clusterMapConfig, dataNodeConfig.getHostName(),
+                dataNodeConfig.getPort(), dataNodeConfig.getRackId(), dataNodeConfig.getSslPort(),
+                dataNodeConfig.getHttp2Port(), DEFAULT_XID, helixClusterManagerQueryHelper);
+      } catch (Exception e) {
+        logger.error("Fail to create an AmbryServerDataNode for {} in datacenter {}, skip adding this node.",
+            dataNodeConfig.getHostName(), dataNodeConfig.getDatacenterName(), e);
+        dataNodeInitializationFailureCount.incrementAndGet();
+        return Collections.emptyList();
+      }
       // for new instance, we first set it to unavailable and rely on its participation to update its liveness
       if (!instanceName.equals(selfInstanceName)) {
         datanode.setState(HardwareState.UNAVAILABLE);

ambry-clustermap/src/main/java/com/github/ambry/clustermap/HelixClusterManagerMetrics.java

@@ -140,7 +140,7 @@ void initializeDatacenterMetrics() {
   /**
    * Initialize datanode related metrics.
    */
-  void initializeDataNodeMetrics() {
+  void initializeDataNodeMetrics(AtomicLong dataNodeInitializationFailureCount) {
     Gauge<Long> dataNodeCount = clusterMapCallback::getDatanodeCount;
     registry.gauge(MetricRegistry.name(HelixClusterManager.class, "dataNodeCount"), () -> dataNodeCount);
 
@@ -152,6 +152,10 @@ void initializeDataNodeMetrics() {
       Gauge<Long> dataNodeState = () -> datanode.getState() == HardwareState.AVAILABLE ? 1L : 0L;
       registry.gauge(MetricRegistry.name(HelixClusterManager.class, metricName), () -> dataNodeState);
     }
+
+    Gauge<Long> dataNodeInitializationFailureCountGauge = dataNodeInitializationFailureCount::get;
+    registry.gauge(MetricRegistry.name(HelixClusterManager.class, "dataNodeInitializationFailureCount"),
+        () -> dataNodeInitializationFailureCountGauge);
   }
 
   /**