[PowerFlex] Add & Remove PowerFlex/ScaleIO MDMs while preparing & unpreparing the storage SDC connections

core/src/main/java/com/cloud/agent/api/UnprepareStorageClientCommand.java

@@ -19,18 +19,22 @@
 
 package com.cloud.agent.api;
 
+import java.util.Map;
+
 import com.cloud.storage.Storage.StoragePoolType;
 
 public class UnprepareStorageClientCommand extends Command {
     private StoragePoolType poolType;
     private String poolUuid;
+    private Map<String, String> details;
 
     public UnprepareStorageClientCommand() {
     }
 
-    public UnprepareStorageClientCommand(StoragePoolType poolType, String poolUuid) {
+    public UnprepareStorageClientCommand(StoragePoolType poolType, String poolUuid, Map<String, String> details) {
         this.poolType = poolType;
         this.poolUuid = poolUuid;
+        this.details = details;
     }
 
     @Override
@@ -45,4 +49,8 @@
     public String getPoolUuid() {
         return poolUuid;
     }
+
+    public Map<String, String> getDetails() {
+        return details;
+    }
 }

engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java

@@ -33,6 +33,7 @@
 import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
 import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClientConnectionPool;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
@@ -300,7 +301,11 @@
                 srcSnapshotDestVolumeMap.put(srcSnapshotVolumeId, destVolumeId);
             }
 
-            String systemId = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID).getValue();
+            String systemId = null;
+            StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
+            if (systemIdDetail != null) {
+                systemId = systemIdDetail.getValue();
+            }
             if (systemId == null) {
                 throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool for reverting VM snapshot: " + vmSnapshot.getName());
             }
@@ -379,7 +384,11 @@
         try {
             List<VolumeObjectTO> volumeTOs = vmSnapshotHelper.getVolumeTOList(vmSnapshot.getVmId());
             Long storagePoolId = vmSnapshotHelper.getStoragePoolForVM(userVm.getId());
-            String systemId = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID).getValue();
+            String systemId = null;
+            StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
+            if (systemIdDetail != null) {
+                systemId = systemIdDetail.getValue();
+            }
             if (systemId == null) {
                 throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool for deleting VM snapshot: " + vmSnapshot.getName());
             }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUnprepareStorageClientCommandWrapper.java

@@ -34,7 +34,7 @@ public class LibvirtUnprepareStorageClientCommandWrapper extends CommandWrapper<
     @Override
     public Answer execute(UnprepareStorageClientCommand cmd, LibvirtComputingResource libvirtComputingResource) {
         final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
-        Pair<Boolean, String> unprepareStorageClientResult = storagePoolMgr.unprepareStorageClient(cmd.getPoolType(), cmd.getPoolUuid());
+        Pair<Boolean, String> unprepareStorageClientResult = storagePoolMgr.unprepareStorageClient(cmd.getPoolType(), cmd.getPoolUuid(), cmd.getDetails());
         if (!unprepareStorageClientResult.first()) {
             String msg = unprepareStorageClientResult.second();
             logger.debug("Couldn't unprepare storage client, due to: " + msg);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java

@@ -482,8 +482,8 @@
         return adaptor.prepareStorageClient(type, uuid, details);
     }
 
-    public Pair<Boolean, String> unprepareStorageClient(StoragePoolType type, String uuid) {
+    public Pair<Boolean, String> unprepareStorageClient(StoragePoolType type, String uuid, Map<String, String> details) {
         StorageAdaptor adaptor = getStorageAdaptor(type);
-        return adaptor.unprepareStorageClient(type, uuid);
+        return adaptor.unprepareStorageClient(type, uuid, details);
     }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java

@@ -584,14 +584,24 @@
             if (!ScaleIOUtil.startSDCService()) {
                 return new Ternary<>(false, null, "Couldn't start SDC service on host");
             }
-        } else if (!ScaleIOUtil.restartSDCService()) {
-            return new Ternary<>(false, null, "Couldn't restart SDC service on host");
+        }
+
+        if (details != null && details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) {
+            // Assuming SDC service is started, add mdms
+            String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS);
+            String[] mdmAddresses = mdms.split(",");
+            if (mdmAddresses.length > 0) {
+                ScaleIOUtil.addMdms(Arrays.asList(mdmAddresses));
+                if (!ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+                    return new Ternary<>(false, null, "Failed to add MDMs");
+                }
+            }
         }
 
         return new Ternary<>( true, getSDCDetails(details), "Prepared client successfully");
     }
 
-    public Pair<Boolean, String> unprepareStorageClient(Storage.StoragePoolType type, String uuid) {
+    public Pair<Boolean, String> unprepareStorageClient(Storage.StoragePoolType type, String uuid, Map<String, String> details) {
         if (!ScaleIOUtil.isSDCServiceInstalled()) {
             logger.debug("SDC service not installed on host, no need to unprepare the SDC client");
             return new Pair<>(true, "SDC service not installed on host, no need to unprepare the SDC client");
@@ -602,8 +612,19 @@
             return new Pair<>(true, "SDC service not enabled on host, no need to unprepare the SDC client");
         }
 
-        if (!ScaleIOUtil.stopSDCService()) {
-            return new Pair<>(false, "Couldn't stop SDC service on host");
+        if (details != null && details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) {
+            String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS);
+            String[] mdmAddresses = mdms.split(",");
+            if (mdmAddresses.length > 0) {
+                if (!ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+                    return new Pair<>(true, "MDM not added, no need to unprepare the SDC client");
+                }
+
+                ScaleIOUtil.removeMdms(Arrays.asList(mdmAddresses));
+                if (ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+                    return new Pair<>(false, "Failed to remove MDMs, unable to unprepare the SDC client");
+                }
+            }
         }
 
         return new Pair<>(true, "Unprepared SDC client successfully");

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java

@@ -133,9 +133,10 @@
      * Unprepares the storage client.
      * @param type type of the storage pool
      * @param uuid uuid of the storage pool
+     * @param details any details of the storage pool that are required for client unpreparation
      * @return status, & message in case failed
      */
-    default Pair<Boolean, String> unprepareStorageClient(StoragePoolType type, String uuid) {
+    default Pair<Boolean, String> unprepareStorageClient(StoragePoolType type, String uuid, Map<String, String> details) {
         return new Pair<>(true, "");
     }
 }

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUnprepareStorageClientCommandWrapperTest.java

@@ -14,6 +14,10 @@
 
 package com.cloud.hypervisor.kvm.resource.wrapper;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -45,10 +49,13 @@ public void testUnprepareStorageClientSuccess() {
         UnprepareStorageClientCommand cmd = Mockito.mock(UnprepareStorageClientCommand.class);
         Mockito.when(cmd.getPoolType()).thenReturn(Storage.StoragePoolType.PowerFlex);
         Mockito.when(cmd.getPoolUuid()).thenReturn(poolUuid);
+        Map<String, String> details = new HashMap<>();
+        details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2");
+        Mockito.when(cmd.getDetails()).thenReturn(details);
 
         KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class);
         Mockito.when(libvirtComputingResourceMock.getStoragePoolMgr()).thenReturn(storagePoolMgr);
-        Mockito.when(storagePoolMgr.unprepareStorageClient(cmd.getPoolType(), cmd.getPoolUuid())).thenReturn(new Pair<>(true, ""));
+        Mockito.when(storagePoolMgr.unprepareStorageClient(cmd.getPoolType(), cmd.getPoolUuid(), cmd.getDetails())).thenReturn(new Pair<>(true, ""));
 
         UnprepareStorageClientAnswer result = (UnprepareStorageClientAnswer) libvirtUnprepareStorageClientCommandWrapperSpy.execute(cmd, libvirtComputingResourceMock);
 
@@ -60,10 +67,13 @@ public void testUnprepareStorageClientFailure() {
         UnprepareStorageClientCommand cmd = Mockito.mock(UnprepareStorageClientCommand.class);
         Mockito.when(cmd.getPoolType()).thenReturn(Storage.StoragePoolType.PowerFlex);
         Mockito.when(cmd.getPoolUuid()).thenReturn(poolUuid);
+        Map<String, String> details = new HashMap<>();
+        details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2");
+        Mockito.when(cmd.getDetails()).thenReturn(details);
 
         KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class);
         Mockito.when(libvirtComputingResourceMock.getStoragePoolMgr()).thenReturn(storagePoolMgr);
-        Mockito.when(storagePoolMgr.unprepareStorageClient(cmd.getPoolType(), cmd.getPoolUuid())).thenReturn(new Pair<>(false, "Unprepare storage client failed"));
+        Mockito.when(storagePoolMgr.unprepareStorageClient(cmd.getPoolType(), cmd.getPoolUuid(), cmd.getDetails())).thenReturn(new Pair<>(false, "Unprepare storage client failed"));
 
         UnprepareStorageClientAnswer result = (UnprepareStorageClientAnswer) libvirtUnprepareStorageClientCommandWrapperSpy.execute(cmd, libvirtComputingResourceMock);
 

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java

@@ -93,34 +93,6 @@ public void testPrepareStorageClient_SDCServiceNotEnabled() {
         Assert.assertEquals("SDC service not enabled on host", result.third());
     }
 
-    @Test
-    public void testPrepareStorageClient_SDCServiceNotRestarted() {
-        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
-        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
-        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-active scini"))).thenReturn(0);
-        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(1);
-
-        Ternary<Boolean, Map<String, String>, String> result = scaleIOStorageAdaptor.prepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid, new HashMap<>());
-
-        Assert.assertFalse(result.first());
-        Assert.assertNull(result.second());
-        Assert.assertEquals("Couldn't restart SDC service on host", result.third());
-    }
-
-    @Test
-    public void testPrepareStorageClient_SDCServiceRestarted() {
-        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
-        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
-        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-active scini"))).thenReturn(0);
-        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(0);
-
-        Ternary<Boolean, Map<String, String>, String> result = scaleIOStorageAdaptor.prepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid, new HashMap<>());
-
-        Assert.assertTrue(result.first());
-        Assert.assertNotNull(result.second());
-        Assert.assertTrue(result.second().isEmpty());
-    }
-
     @Test
     public void testPrepareStorageClient_SDCServiceNotStarted() {
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
@@ -181,45 +153,53 @@ public void testPrepareStorageClient_SDCServiceStartedReturnSDCGuid() {
 
     @Test
     public void testUnprepareStorageClient_SDCServiceNotInstalled() {
+        Map<String, String> details = new HashMap<>();
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(4);
 
-        Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid);
+        Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid, details);
 
         Assert.assertTrue(result.first());
         Assert.assertEquals("SDC service not installed on host, no need to unprepare the SDC client", result.second());
     }
 
     @Test
     public void testUnprepareStorageClient_SDCServiceNotEnabled() {
+        Map<String, String> details = new HashMap<>();
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(1);
 
-        Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid);
+        Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid, details);
 
         Assert.assertTrue(result.first());
         Assert.assertEquals("SDC service not enabled on host, no need to unprepare the SDC client", result.second());
     }
 
     @Test
-    public void testUnprepareStorageClient_SDCServiceNotStopped() {
+    public void testUnprepareStorageClient_MDMNotAdded() {
+        Map<String, String> details = new HashMap<>();
+        details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2");
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
-        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl stop scini"))).thenReturn(1);
+        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-3.3.3.3 [1]-4.4.4.4");
 
-        Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid);
+        Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid, details);
 
-        Assert.assertFalse(result.first());
-        Assert.assertEquals("Couldn't stop SDC service on host", result.second());
+        Assert.assertTrue(result.first());
+        Assert.assertEquals("MDM not added, no need to unprepare the SDC client", result.second());
     }
 
     @Test
-    public void testUnprepareStorageClient_SDCServiceStopped() {
+    public void testUnprepareStorageClient_RemoveMDMFailed() {
+        Map<String, String> details = new HashMap<>();
+        details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2");
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
-        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl stop scini"))).thenReturn(0);
+        when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(0);
+        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
 
-        Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid);
+        Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid, details);
 
-        Assert.assertTrue(result.first());
+        Assert.assertFalse(result.first());
+        Assert.assertEquals("Failed to remove MDMs, unable to unprepare the SDC client", result.second());
     }
 }

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/api/StorageConfiguration.java

@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.storage.datastore.api;
+
+public class StorageConfiguration {
+    String systemId;
+    Long mdmPort;
+    String[] mdmAddresses;
+
+    public String getSystemId() {
+        return systemId;
+    }
+
+    public void setSystemId(String systemId) {
+        this.systemId = systemId;
+    }
+
+    public Long getMdmPort() {
+        return mdmPort;
+    }
+
+    public void setMdmPort(Long mdmPort) {
+        this.mdmPort = mdmPort;
+    }
+
+    public String[] getMdmAddresses() {
+        return mdmAddresses;
+    }
+
+    public void setMdmAddresses(String[] mdmAddresses) {
+        this.mdmAddresses = mdmAddresses;
+    }
+}

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/api/StoragePool.java

@@ -17,12 +17,15 @@
 
 package org.apache.cloudstack.storage.datastore.api;
 
+import java.util.List;
+
 public class StoragePool {
     String id;
     String name;
     String mediaType;
     String protectionDomainId;
     String systemId;
+    List<String> mdmAddresses;
     StoragePoolStatistics statistics;
 
     public String getId() {
@@ -65,6 +68,14 @@
         this.systemId = systemId;
     }
 
+    public List<String> getMdmAddresses() {
+        return mdmAddresses;
+    }
+
+    public void setMdmAddresses(List<String> mdmAddresses) {
+        this.mdmAddresses = mdmAddresses;
+    }
+
     public StoragePoolStatistics getStatistics() {
         return statistics;
     }

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java

@@ -38,6 +38,7 @@ public interface ScaleIOGatewayClient {
     String GATEWAY_API_PASSWORD = "powerflex.gw.password";
     String STORAGE_POOL_NAME = "powerflex.storagepool.name";
     String STORAGE_POOL_SYSTEM_ID = "powerflex.storagepool.system.id";
+    String STORAGE_POOL_MDMS = "powerflex.storagepool.mdms";
     String SDC_ID = "powerflex.sdc.id";
     String SDC_GUID = "powerflex.sdc.guid";
 
@@ -91,4 +92,6 @@ Volume createVolume(final String name, final String storagePoolId,
     boolean haveConnectedSdcs();
     boolean isSdcConnected(String sdcId);
     boolean isSdcConnectedByIP(String ipAddress);
+
+    List<String> getMdmAddresses();
 }