HBASE-30220: A replica cluster can have read-only mode disabled even when another active cluster already exists

hbase-client/src/main/java/org/apache/hadoop/hbase/ReadOnlyTransitionException.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Thrown when a replica cluster attempts to disable read-only mode while another active cluster
+ * already exists on the same storage location.
+ */
+@InterfaceAudience.Public
+public class ReadOnlyTransitionException extends DoNotRetryIOException {
+
+  private static final long serialVersionUID = 1L;
+
+  public ReadOnlyTransitionException() {
+    super();
+  }
+
+  /**
+   * @param message the message for this exception
+   */
+  public ReadOnlyTransitionException(String message) {
+    super(message);
+  }
+
+  /**
+   * @param message   the message for this exception
+   * @param throwable the {@link Throwable} to use for this exception
+   */
+  public ReadOnlyTransitionException(String message, Throwable throwable) {
+    super(message, throwable);
+  }
+
+  /**
+   * @param throwable the {@link Throwable} to use for this exception
+   */
+  public ReadOnlyTransitionException(Throwable throwable) {
+    super(throwable);
+  }
+}

hbase-server/src/main/java/org/apache/hadoop/hbase/HBaseServerBase.java

@@ -73,6 +73,7 @@
 import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
 import org.apache.hadoop.hbase.util.Addressing;
 import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.ConfigurationUtil;
 import org.apache.hadoop.hbase.util.DNS;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
@@ -102,15 +103,19 @@ public abstract class HBaseServerBase<R extends HBaseRpcServicesBase<?>> extends
   protected final AtomicBoolean abortRequested = new AtomicBoolean(false);
 
   // Set when a report to the master comes back with a message asking us to
-  // shutdown. Also set by call to stop when debugging or running unit tests
+  // shut down. Also set by call to stop when debugging or running unit tests
   // of HRegionServer in isolation.
   protected volatile boolean stopped = false;
 
+  // Flag set when a read-only to read-write transition is blocked because another active cluster
+  // exists
+  protected volatile boolean readOnlyTransitionBlocked = false;
+
   // Only for testing
   private boolean isShutdownHookInstalled = false;
 
   /**
-   * This servers startcode.
+   * This server's startcode.
    */
   protected final long startcode;
 
@@ -639,11 +644,30 @@ public void updateConfiguration() throws IOException {
     LOG.info("Reloading the configuration from disk.");
     // Reload the configuration from disk.
     preUpdateConfiguration();
+    this.readOnlyTransitionBlocked = false;
     conf.reloadConfiguration();
     configurationManager.notifyAllObservers(conf);
+    this.checkForBlockedReadOnlyTransition();
     postUpdateConfiguration();
   }
 
+  protected Configuration blockReadOnlyTransition(Configuration updatedConf) {
+    LOG.error(
+      "Cannot disable read-only mode. The {} file contains a different cluster ID, which means "
+        + "that cluster is already the active cluster. Reverting {} to true",
+      HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME, HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY);
+    this.readOnlyTransitionBlocked = true;
+    return ConfigurationUtil.getReadOnlyEnabledConfigurationCopy(updatedConf);
+  }
+
+  protected void checkForBlockedReadOnlyTransition() throws ReadOnlyTransitionException {
+    if (this.readOnlyTransitionBlocked) {
+      throw new ReadOnlyTransitionException(
+        "Cannot disable read-only mode because another active cluster already exists on this "
+          + "storage location. The read-only coprocessors have not been removed.");
+    }
+  }
+
   @Override
   public KeyManagementService getKeyManagementService() {
     return this;

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -4520,11 +4520,31 @@ public void onConfigurationChange(Configuration updatedConf) {
 
     boolean originalIsReadOnlyEnabled = CoprocessorConfigurationUtil
       .areReadOnlyCoprocessorsLoaded(this.conf, CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
+    boolean newReadOnlyEnabled = ConfigurationUtil.isReadOnlyModeEnabledInConf(updatedConf);
 
-    // updatedConf and this.conf reference the same Configuration object in an actual HBase
-    // deployment. However, in unit test cases they reference different Configuration objects, so
-    // this.conf needs to be updated.
-    CoprocessorConfigurationUtil.maybeUpdateCoprocessors(updatedConf, this.conf,
+    // The updatedConf is potentially a shared Configuration object, so we do not want to directly
+    // revert its read-only value if another active cluster already exists. For now, we reference
+    // updatedConf and create a copy for modification below if necessary.
+    Configuration confForCoprocessors = updatedConf;
+
+    if (originalIsReadOnlyEnabled && !newReadOnlyEnabled) {
+      // Changing this cluster from a replica to an active cluster. There should not be another
+      // active cluster already.
+      MasterFileSystem mfs = this.getMasterFileSystem();
+      if (
+        AbstractReadOnlyController.isAnotherClusterActive(mfs.getFileSystem(), mfs.getRootDir(),
+          mfs.getActiveClusterSuffix())
+      ) {
+        // Revert read-only mode here
+        confForCoprocessors = this.blockReadOnlyTransition(updatedConf);
+      }
+    }
+
+    // In a real HBase deployment, confForCoprocessors may reference the same object as this.conf.
+    // This is assuming confForCoprocessors still references updatedConf, as mentioned in a previous
+    // comment. For unit tests, this Configuration object is not shared, so we need to make sure to
+    // update the coprocessors specifically for this.conf.
+    CoprocessorConfigurationUtil.maybeUpdateCoprocessors(confForCoprocessors, this.conf,
       originalIsReadOnlyEnabled, this.cpHost, CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
       this.maintenanceMode, this.toString(), this::initializeCoprocessorHost);
 

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java

@@ -78,12 +78,14 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.ActiveClusterSuffix;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellBuilderType;
 import org.apache.hadoop.hbase.CellComparator;
 import org.apache.hadoop.hbase.CellComparatorImpl;
 import org.apache.hadoop.hbase.CellScanner;
 import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.ClusterId;
 import org.apache.hadoop.hbase.CompareOperator;
 import org.apache.hadoop.hbase.CompoundConfiguration;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
@@ -172,6 +174,7 @@
 import org.apache.hadoop.hbase.replication.ReplicationUtils;
 import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver;
 import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.access.AbstractReadOnlyController;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
 import org.apache.hadoop.hbase.trace.TraceUtil;
@@ -8997,20 +9000,54 @@ public void onConfigurationChange(Configuration updatedConf) {
 
     boolean originalIsReadOnlyEnabled = CoprocessorConfigurationUtil
       .areReadOnlyCoprocessorsLoaded(this.conf, CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
+    boolean newReadOnlyEnabled = ConfigurationUtil.isReadOnlyModeEnabledInConf(updatedConf);
+
+    // The updatedConf is potentially a shared Configuration object, so we do not want to directly
+    // revert its read-only value if another active cluster already exists. For now, we reference
+    // updatedConf and create a copy for modification below if necessary.
+    Configuration confForCoprocessors = updatedConf;
+
+    if (originalIsReadOnlyEnabled && !newReadOnlyEnabled) {
+      // Changing this cluster from a replica to an active cluster. There should not be another
+      // active cluster already.
+      try {
+        FileSystem regionFs = getFilesystem();
+        Path rootDir = CommonFSUtils.getRootDir(this.conf);
+        ClusterId clusterId = FSUtils.getClusterIdFile(regionFs, rootDir, new ClusterId.Parser());
+        if (clusterId != null) {
+          ActiveClusterSuffix localSuffix = ActiveClusterSuffix.fromConfig(updatedConf, clusterId);
+          if (AbstractReadOnlyController.isAnotherClusterActive(regionFs, rootDir, localSuffix)) {
+            LOG.error(
+              "Cannot disable read-only mode for region {}. Another cluster is already "
+                + "the active cluster on this storage location. Reverting {} to true.",
+              this, HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY);
+            // Revert read-only mode here
+            confForCoprocessors =
+              ConfigurationUtil.getReadOnlyEnabledConfigurationCopy(updatedConf);
+            newReadOnlyEnabled = true;
+          }
+        }
+      } catch (IOException e) {
+        LOG.error("Failed to check active cluster status for region {}. "
+          + "Blocking read-only mode transition to prevent potential data corruption.", this, e);
+        // Revert read-only mode here
+        confForCoprocessors = ConfigurationUtil.getReadOnlyEnabledConfigurationCopy(updatedConf);
+        newReadOnlyEnabled = true;
+      }
+    }
 
     // HRegion's this.conf is a special Configuration type called CompoundConfiguration. This means
-    // we don't want to use the updatedConf provided in onConfigurationChange() for creating a new
+    // we don't want to use the confForCoprocessors Configuration for creating a new
     // RegionCoprocessorHost. Instead, we update this.conf and use that for decorating the region
     // config and updating this.coprocessorHost.
-    CoprocessorConfigurationUtil.maybeUpdateCoprocessors(updatedConf, this.conf,
+    CoprocessorConfigurationUtil.maybeUpdateCoprocessors(confForCoprocessors, this.conf,
       originalIsReadOnlyEnabled, this.coprocessorHost, CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
       false, this.toString(), conf -> {
         decorateRegionConfiguration(conf);
         this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
       });
 
-    boolean newReadOnlyEnabled = ConfigurationUtil.isReadOnlyModeEnabledInConf(updatedConf);
-
+    // Changing this cluster from a replica to an active cluster
     if (originalIsReadOnlyEnabled && !newReadOnlyEnabled) {
       LOG.info("Cluster Read Only mode disabled");
       for (HStore store : stores.values()) {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -74,9 +74,11 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.ActiveClusterSuffix;
 import org.apache.hadoop.hbase.CacheEvictionStats;
 import org.apache.hadoop.hbase.CallQueueTooBigException;
 import org.apache.hadoop.hbase.ClockOutOfSyncException;
+import org.apache.hadoop.hbase.ClusterId;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.ExecutorStatusChore;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -155,9 +157,11 @@
 import org.apache.hadoop.hbase.security.Superusers;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.security.access.AbstractReadOnlyController;
 import org.apache.hadoop.hbase.trace.TraceUtil;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.CompressionTest;
+import org.apache.hadoop.hbase.util.ConfigurationUtil;
 import org.apache.hadoop.hbase.util.CoprocessorConfigurationUtil;
 import org.apache.hadoop.hbase.util.DNS;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -315,7 +319,6 @@ public class HRegionServer extends HBaseServerBase<RSRpcServices>
   private LeaseManager leaseManager;
 
   private volatile boolean dataFsOk;
-  private volatile boolean isGlobalReadOnlyEnabled;
 
   static final String ABORT_TIMEOUT = "hbase.regionserver.abort.timeout";
   // Default abort timeout is 1200 seconds for safe
@@ -3505,11 +3508,32 @@ public void onConfigurationChange(Configuration updatedConf) {
 
     boolean originalIsReadOnlyEnabled = CoprocessorConfigurationUtil
       .areReadOnlyCoprocessorsLoaded(this.conf, CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY);
-
-    // updatedConf and this.conf reference the same Configuration object in an actual HBase
-    // deployment. However, in unit test cases they reference different Configuration objects, so
-    // this.conf needs to be updated.
-    CoprocessorConfigurationUtil.maybeUpdateCoprocessors(updatedConf, this.conf,
+    boolean newReadOnlyEnabled = ConfigurationUtil.isReadOnlyModeEnabledInConf(updatedConf);
+
+    // The updatedConf is potentially a shared Configuration object, so we do not want to directly
+    // revert its read-only value if another active cluster already exists. For now, we reference
+    // updatedConf and create a copy for modification below if necessary.
+    Configuration confForCoprocessors = updatedConf;
+
+    if (originalIsReadOnlyEnabled && !newReadOnlyEnabled) {
+      // Changing this cluster from a replica to an active cluster. There should not be another
+      // active cluster already.
+      ActiveClusterSuffix localSuffix =
+        ActiveClusterSuffix.fromConfig(this.conf, new ClusterId(getClusterId()));
+      if (
+        AbstractReadOnlyController.isAnotherClusterActive(getFileSystem(), getDataRootDir(),
+          localSuffix)
+      ) {
+        // Revert read-only mode here
+        confForCoprocessors = this.blockReadOnlyTransition(updatedConf);
+      }
+    }
+
+    // In a real HBase deployment, confForCoprocessors may reference the same object as this.conf.
+    // This is assuming confForCoprocessors still references updatedConf, as mentioned in a previous
+    // comment. For unit tests, this Configuration object is not shared, so we need to make sure to
+    // update the coprocessors specifically for this.conf.
+    CoprocessorConfigurationUtil.maybeUpdateCoprocessors(confForCoprocessors, this.conf,
       originalIsReadOnlyEnabled, this.rsHost, CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
       false, this.toString(), conf -> this.rsHost = new RegionServerCoprocessorHost(this, conf));
   }

hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AbstractReadOnlyController.java

@@ -68,6 +68,33 @@ public void start(CoprocessorEnvironment env) throws IOException {
   public void stop(CoprocessorEnvironment env) {
   }
 
+  /**
+   * Checks whether another cluster is currently active on this storage location by reading the
+   * {@value HConstants#ACTIVE_CLUSTER_SUFFIX_FILE_NAME} file on the filesystem.
+   * @param fs                 the filesystem to read from
+   * @param rootDir            the HBase root directory
+   * @param localClusterSuffix the local cluster's ActiveClusterSuffix identity
+   * @return true if the active cluster file exists and belongs to a different cluster; false if the
+   *         file does not exist or belongs to this cluster
+   */
+  public static boolean isAnotherClusterActive(FileSystem fs, Path rootDir,
+    ActiveClusterSuffix localClusterSuffix) {
+    try {
+      ActiveClusterSuffix fileData =
+        FSUtils.getClusterIdFile(fs, rootDir, new ActiveClusterSuffix.Parser());
+      if (fileData == null) {
+        return false;
+      }
+      return !localClusterSuffix.equals(fileData);
+    } catch (IOException e) {
+      LOG.error(
+        "Failed to read active cluster suffix file from {} at {}. "
+          + "Assuming another cluster is active to prevent potential data corruption.",
+        HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME, rootDir, e);
+      return true;
+    }
+  }
+
   public static void manageActiveClusterIdFile(boolean readOnlyEnabled, MasterFileSystem mfs) {
     FileSystem fs = mfs.getFileSystem();
     Path rootDir = mfs.getRootDir();
@@ -110,8 +137,23 @@ public static void manageActiveClusterIdFile(boolean readOnlyEnabled, MasterFile
           FSUtils.setClusterIdFile(fs, rootDir, HConstants.ACTIVE_CLUSTER_SUFFIX_FILE_NAME,
             mfs.getActiveClusterSuffix(), wait);
         } else {
-          LOG.debug("Active cluster file already exists at: {}. No need to create it again.",
-            activeClusterFile);
+          try (FSDataInputStream in = fs.open(activeClusterFile)) {
+            ActiveClusterSuffix existingData = ActiveClusterSuffix.parseFrom(in.readAllBytes());
+            ActiveClusterSuffix localData = mfs.getActiveClusterSuffix();
+            if (localData.equals(existingData)) {
+              LOG.debug("Active cluster file already exists at {} and belongs to this cluster. "
+                + "No need to create it again.", activeClusterFile);
+            } else {
+              LOG.error(
+                "Active cluster file at {} belongs to a different cluster. "
+                  + "ID from active cluster file: {}, ID of this cluster: {}. "
+                  + "Another cluster is already the active cluster.",
+                activeClusterFile, existingData, localData);
+            }
+          } catch (DeserializationException e) {
+            LOG.error("Failed to deserialize ActiveClusterSuffix from file {}", activeClusterFile,
+              e);
+          }
         }
       }
     } catch (IOException e) {

hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigurationUtil.java

@@ -116,8 +116,22 @@ public static List<Map.Entry<String, String>> getKeyValues(Configuration conf, S
     return rtn;
   }
 
+  /**
+   * Returns true if the provided Configuration object has
+   * {@link HConstants#HBASE_GLOBAL_READONLY_ENABLED_KEY} set to true; false otherwise.
+   */
   public static boolean isReadOnlyModeEnabledInConf(Configuration conf) {
     return conf.getBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY,
       HConstants.HBASE_GLOBAL_READONLY_ENABLED_DEFAULT);
   }
+
+  /**
+   * Returns a copied version of the provided Configuration object that has
+   * {@link HConstants#HBASE_GLOBAL_READONLY_ENABLED_KEY} set to true.
+   */
+  public static Configuration getReadOnlyEnabledConfigurationCopy(Configuration conf) {
+    Configuration readOnlyConf = new Configuration(conf);
+    readOnlyConf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, true);
+    return readOnlyConf;
+  }
 }