Subsequent VM Creation is failing for ISCSI Storage Pool on Oracle Linux

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java

@@ -47,6 +47,12 @@ public class IscsiAdmStorageAdaptor implements StorageAdaptor {
 
     private static final Map<String, KVMStoragePool> MapStorageUuidToStoragePool = new HashMap<>();
 
+    /** iscsiadm's ISCSI_ERR_NO_OBJS_FOUND: returned by "-m session" when no session is established. */
+    private static final int ISCSI_ERR_NO_OBJS_FOUND = 21;
+
+    /** iscsiadm's ISCSI_ERR_SESS_EXISTS: returned by "--login" when the session is already logged in (e.g. Ubuntu). */
+    private static final int ISCSI_SESSION_EXISTS_CODE = 15;
+
     @Override
     public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map<String, String> details, boolean isPrimaryStorage) {
         IscsiAdmStoragePool storagePool = new IscsiAdmStoragePool(uuid, host, port, storagePoolType, this);
@@ -90,12 +96,16 @@ public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, KVMStoragePool pool
 
     @Override
     public boolean connectPhysicalDisk(String volumeUuid, KVMStoragePool pool, Map<String, String> details, boolean isVMMigrate) {
+        final String host = pool.getSourceHost();
+        final int port = pool.getSourcePort();
+        final String iqn = getIqn(volumeUuid);
+
         // ex. sudo iscsiadm -m node -T iqn.2012-03.com.test:volume1 -p 192.168.233.10:3260 -o new
         Script iScsiAdmCmd = new Script(true, "iscsiadm", 0, logger);
 
         iScsiAdmCmd.add("-m", "node");
-        iScsiAdmCmd.add("-T", getIqn(volumeUuid));
-        iScsiAdmCmd.add("-p", pool.getSourceHost() + ":" + pool.getSourcePort());
+        iScsiAdmCmd.add("-T", iqn);
+        iScsiAdmCmd.add("-p", host + ":" + port);
         iScsiAdmCmd.add("-o", "new");
 
         String result = iScsiAdmCmd.execute();
@@ -122,25 +132,29 @@ public boolean connectPhysicalDisk(String volumeUuid, KVMStoragePool pool, Map<S
             }
         }
 
-        final String host = pool.getSourceHost();
-        final int port = pool.getSourcePort();
-        final String iqn = getIqn(volumeUuid);
+        final boolean sessionAlreadyActive = isIscsiSessionActive(iqn, host, port);
+        logger.debug("iSCSI session active check for target {} at {}:{}: {}", iqn, host, port, sessionAlreadyActive);
 
-        // Always try to login; treat benign outcomes as success (idempotent)
+        // Always try to login; treat benign outcomes as success (idempotent).
+        // Ubuntu returns ISCSI_ERR_SESS_EXISTS (15) when already logged in; Oracle often returns 0 with no output.
         iScsiAdmCmd = new Script(true, "iscsiadm", 0, logger);
         iScsiAdmCmd.add("-m", "node");
         iScsiAdmCmd.add("-T", iqn);
         iScsiAdmCmd.add("-p", host + ":" + port);
         iScsiAdmCmd.add("--login");
 
         result = iScsiAdmCmd.execute();
+        final boolean sessionPreExisted = (iScsiAdmCmd.getExitValue() == ISCSI_SESSION_EXISTS_CODE) || sessionAlreadyActive;
 
-        if (!handleLoginResult(result, volumeUuid)) {
+        if (!handleLoginResult(result, sessionPreExisted, volumeUuid)) {
             return false;
         }
 
-        // If the session already existed, a newly mapped LUN won't be visible until a rescan.
-        if (result != null) {
+        // Newly mapped LUNs on an existing session are invisible until rescan.
+        // Prefer the pre-login session check (required on Oracle, where re-login exits 0).
+        // Also treat ISCSI_ERR_SESS_EXISTS as a hint (Ubuntu).
+        if (sessionPreExisted) {
+            logger.debug("iSCSI session for target {} at {}:{} pre-existed, performing rescan", iqn, host, port);
             rescanIscsiSessions(iqn, host, port);
         }
 
@@ -156,6 +170,11 @@ public boolean connectPhysicalDisk(String volumeUuid, KVMStoragePool pool, Map<S
         // isn't solved here, but made highly unlikely to be a problem).
         waitForDiskToBecomeAvailable(volumeUuid, pool);
 
+        if (getPhysicalDisk(volumeUuid, pool).getSize() <= 0) {
+            logger.warn("iSCSI device not ready for target {} at {}:{} after wait", volumeUuid, host, port);
+            return false;
+        }
+
         return true;
     }
 
@@ -179,22 +198,56 @@ boolean handleNodeCreateResult(String result, String volumeUuid) {
 
     /**
      * Checks the result of an iscsiadm login command.
-     * Returns true if the login succeeded or session already exists, false on failure.
+     * Returns true if the session already existed (pre-check and/or ISCSI_ERR_SESS_EXISTS)
+     * or login succeeded with no error output, false on failure.
+     *
+     * sessionPreExisted must be checked first: on Ubuntu, re-login exits 15 with a non-null
+     * error message that would otherwise be treated as failure.
      */
-    boolean handleLoginResult(String result, String volumeUuid) {
-        if (result == null) {
-            logger.debug("Successfully logged in to iSCSI target {}", volumeUuid);
+    boolean handleLoginResult(String result, boolean sessionPreExisted, String volumeUuid) {
+        if (sessionPreExisted) {
+            logger.debug("iSCSI session already active for target {}", volumeUuid);
             return true;
         }
-        String msg = result.toLowerCase();
-        if (msg.contains("already present") || msg.contains("already logged in") || msg.contains("session exists")) {
-            logger.debug("iSCSI session already exists for target {}, proceeding", volumeUuid);
+        if (result == null) {
+            logger.debug("Successfully logged in to iSCSI target {}", volumeUuid);
             return true;
         }
         logger.debug("Failed to log in to iSCSI target {}: {}", volumeUuid, result);
         return false;
     }
 
+    /**
+     * Checks whether a session to the given target and portal is already established.
+     *
+     * "iscsiadm -m session" exits with ISCSI_ERR_NO_OBJS_FOUND when no session exists, which is a
+     * normal outcome here, so that exit value is ignored and the output is taken from the parser.
+     */
+    private boolean isIscsiSessionActive(String iqn, String host, int port) {
+        Script sessionCmd = new Script(true, "iscsiadm", 0, logger);
+        sessionCmd.add("-m", "session");
+
+        OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
+
+        sessionCmd.executeIgnoreExitValue(parser, ISCSI_ERR_NO_OBJS_FOUND);
+
+        String sessions = parser.getLines();
+
+        if (StringUtils.isBlank(sessions)) {
+            return false;
+        }
+
+        String portal = host + ":" + port;
+
+        for (String line : sessions.split("\n")) {
+            if (line.contains(iqn) && line.contains(portal)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     private void rescanIscsiSessions(String iqn, String host, int port) {
         Script rescanCmd = new Script(true, "iscsiadm", 0, logger);
         rescanCmd.add("-m", "node");
@@ -290,8 +343,17 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) {
 
     private long getDeviceSize(String deviceByPath) {
         try {
-            if (!Files.exists(Paths.get(deviceByPath))) {
-                logger.debug("Device by-path does not exist yet: " + deviceByPath);
+            Path devicePath = Paths.get(deviceByPath);
+            if (!Files.exists(devicePath)) {
+                logger.debug("Device by-path does not exist yet: {}", deviceByPath);
+                return 0L;
+            }
+            if (Files.isRegularFile(devicePath)) {
+                logger.warn("Removing corrupt regular file at iSCSI by-path {} (expected block device symlink)", deviceByPath);
+                return 0L;
+            }
+            if (!Files.isSymbolicLink(devicePath)) {
+                logger.warn("Path {} exists but is not an iSCSI block device symlink", deviceByPath);
                 return 0L;
             }
         } catch (Exception ex) {