Repository: rook/rook
Pull request #10772 (2022-08-23 19:29:22)

docs: update upgrade docs for v1.10
Signed-off-by: Blaine Gardner <blaine.gardner@redhat.com>

<!-- Please take a look at our [Contributing](https://rook.io/docs/rook/latest/Contributing/development-flow/)
documentation before submitting a Pull Request!
Thank you for contributing to Rook! -->

**Description of your changes:**

**Which issue is resolved by this Pull Request:**
Resolves #

**Checklist:**

- [ ] **Commit Message Formatting**: Commit titles and messages follow guidelines in the [developer guide](https://rook.io/docs/rook/latest/Contributing/development-flow/#commit-structure)).
- [ ] **Skip Tests for Docs**: If this is only a documentation change, add the label `skip-ci` on the PR.
- [ ] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/latest/Contributing/development-flow/#submitting-a-pull-request)
- [ ] [Pending release notes](https://github.com/rook/rook/blob/master/PendingReleaseNotes.md) updated with breaking and/or notable changes for the next minor release.
- [ ] Documentation has been updated, if necessary.
- [ ] Unit tests have been added, if necessary.
- [ ] Integration tests have been added, if necessary.



Patch:
diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md
index 579d7d9e7..7a9769e34 100644
--- a/Documentation/Helm-Charts/operator-chart.md
+++ b/Documentation/Helm-Charts/operator-chart.md
@@ -73,8 +73,6 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.rbdFSGroupPolicy`              | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted                                                       | `ReadWriteOnceWithFSType`                                 |
 | `csi.cephFSFSGroupPolicy`           | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted                                                    | `ReadWriteOnceWithFSType`                                 |
 | `csi.nfsFSGroupPolicy`              | Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted                                                       | `ReadWriteOnceWithFSType`                                 |
-| `csi.enableMetadata`                | Set volume metadata on the CephFS subvolumes and RBD images.                                                                                     | `false`                                                   |
-| `csi.clusterName`                   | Cluster name identifier to set as metadata on the CephFS subvolumes and RBD images.                                                              | `<none>`                                                  |
 | `csi.logLevel`                      | Set logging level for cephCSI containers maintained by the cephCSI. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.                      | `0`                                                       |
 | `csi.sidecarLogLevel`                      | Set logging level for cephCSI sidecar containers. Supported values from 0 to 5. 0 for general useful logs (the default), 5 for trace level verbosity.                      | `0`                                                       |
 | `csi.grpcTimeoutInSeconds`          | Set GRPC timeout for csi containers.                                                                                                             | `150`                                                     |
diff --git a/Documentation/Upgrade/ceph-upgrade.md b/Documentation/Upgrade/ceph-upgrade.md
index 25ba10de4..75d165496 100644
--- a/Documentation/Upgrade/ceph-upgrade.md
+++ b/Documentation/Upgrade/ceph-upgrade.md
@@ -39,6 +39,10 @@ before upgrading to Rook v1.10.
 We recommend updating to v16.2.7 or newer. If you require updating **to v16.2.0-v16.2.6**,
 please see the [v1.8 upgrade guide for a special upgrade consideration](https://rook.github.io/docs/rook/v1.8/ceph-upgrade.html#disable-bluestore_fsck_quick_fix_on_mount).
 
+!!! warning
+    Ceph v17.2.2 has a blocking issue when running with Rook. If you are running Ceph v17, we
+    recommend using v17.2.3 or newer.
+
 ### Quincy Consideration
 
 In Ceph Quincy (v17), the `device_health_metrics` pool was renamed to `.mgr`. Ceph will perform this
diff --git a/Documentation/Upgrade/rook-upgrade.md b/Documentation/Upgrade/rook-upgrade.md
index df0975815..ce7fb83b3 100644
--- a/Documentation/Upgrade/rook-upgrade.md
+++ b/Documentation/Upgrade/rook-upgrade.md
@@ -14,7 +14,7 @@ We welcome feedback and opening issues!
 
 ## Supported Versions
 
-This guide is for upgrading from **Rook v1.8.x to Rook v1.9.x**.
+This guide is for upgrading from **Rook v1.9.x to Rook v1.10.x**.
 
 Please refer to the upgrade guides from previous releases for supported upgrade paths.
 Rook upgrades are only supported between official releases.
@@ -22,6 +22,7 @@ Rook upgrades are only supported between official releases.
 For a guide to upgrade previous versions of Rook, please refer to the version of documentation for
 those releases.
 
+* [Upgrade 1.8 to 1.9](https://rook.io/docs/rook/v1.9/Upgrade/rook-upgrade/)
 * [Upgrade 1.7 to 1.8](https://rook.io/docs/rook/v1.8/ceph-upgrade.html)
 * [Upgrade 1.6 to 1.7](https://rook.io/docs/rook/v1.7/ceph-upgrade.html)
 * [Upgrade 1.5 to 1.6](https://rook.io/docs/rook/v1.6/ceph-upgrade.html)
@@ -43,27 +44,14 @@ those releases.
     official releases. Builds from the master branch can have functionality changed or removed at any
     time without compatibility support and without prior notice.
 
-## Breaking changes in v1.9
+## Breaking changes in v1.10
 
-* Helm charts now define default resource requests and limits for Rook Pods. If you use Helm,
-  ensure you have defined an override for these in your `values.yaml` if you don't wish to use the
-  recommended defaults. Setting resource requests and limits could mean that Kubernetes will not
-  allow Pods to be scheduled in some cases. If sufficient resources are not available, you can
-  reduce or remove the requests and limits.
+* Support for Ceph Octopus (15.2.x) was removed. If you are running v15 you must upgrade to Ceph
+  Pacific (v16) or Quincy (v17) before upgrading to Rook v1.10
 
-* MDS liveness and startup probes are now configured by the CephFilesystem resource instead of
-  CephCluster. Upgrade instructions are [below](#mds-liveness-and-startup-probes).
+* The minimum supported version of Ceph-CSI is v3.6.0. You must update to at least this version of
+  Ceph-CSI before or at the same time you update the Rook operator image to v1.10
 
-* Rook no longer deploys Prometheus rules from the operator. If you have been relying on Rook to
-  deploy prometheus rules in the past, please follow the upgrade instructions [below](#prometheus).
-
-* Due to a number of Ceph issues and changes, Rook officially only supports Ceph
-  v16.2.7 or higher for CephNFS. If you are using an earlier version, upgrade your Ceph version
-  following the advice given in Rook's
-  [v1.8 NFS docs](https://rook.github.io/docs/rook/v1.8/ceph-nfs-crd.html).
-
-* If you use Helm and your operator is in a different namespace from the cluster and object storage class is enabled
-  you have to delete the object storage class before upgrading. [#10153](https://github.com/rook/rook/issues/10153)
 
 ## Considerations
 
@@ -79,13 +67,11 @@ With this upgrade guide, there are a few notes to consider:
 
 Unless otherwise noted due to extenuating requirements, upgrades from one patch release of Rook to
 another are as simple as updating the common resources and the image of the Rook operator. For
-example, when Rook v1.9.3 is released, the process of updating from v1.9.0 is as simple as running
+example, when Rook v1.10.1 is released, the process of updating from v1.10.0 is as simple as running
 the following:
 
-First get the latest common resources manifests that contain the latest changes for Rook v1.9.
-
 ```console
-git clone --single-branch --depth=1 --branch v1.9.3 https://github.com/rook/rook.git
+git clone --single-branch --depth=1 --branch v1.10.1 https://github.com/rook/rook.git
 cd rook/deploy/examples
 ```
 
@@ -93,11 +79,11 @@ If you have deployed the Rook Operator or the Ceph cluster into a different name
 `rook-ceph`, see the [Update common resources and CRDs](#1-update-common-resources-and-crds)
 section for instructions on how to change the default namespaces in `common.yaml`.
 
-Then apply the latest changes from v1.9 and update the Rook Operator image.
+Then apply the latest changes from v1.10 and update the Rook Operator image.
 
 ```console
 kubectl apply -f common.yaml -f crds.yaml
-kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.9.3
+kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.10.1
 ```
 
 As exemplified above, it is a good practice to update Rook common resources from the example
@@ -129,9 +115,9 @@ In order to successfully upgrade a Rook cluster, the following prerequisites mus
 
 ## Rook Operator Upgrade
 
-In the examples given in this guide, we will be upgrading a live Rook cluster running `v1.8.9` to
-the version `v1.9.3`. This upgrade should work from any official patch release of Rook v1.8 to any
-official patch release of v1.9.
+In the examples given in this guide, we will be upgrading a live Rook cluster running `v1.9.10` to
+the version `v1.10.0`. This upgrade should work from any official patch release of Rook v1.9 to any
+official patch release of v1.10.
 
 Let's get started!
 
@@ -158,7 +144,7 @@ by the Operator. Also update the Custom Resource Definitions (CRDs).
 Get the latest common resources manifests that contain the latest changes.
 
 ```console
-git clone --single-branch --depth=1 --branch v1.9.3 https://github.com/rook/rook.git
+git clone --single-branch --depth=1 --branch master https://github.com/rook/rook.git
 cd rook/deploy/examples
 ```
 
@@ -188,31 +174,16 @@ step to upgrade the Prometheus RBAC resources as well.
 kubectl apply -f deploy/examples/monitoring/rbac.yaml
 ```
 
-Rook no longer deploys Prometheus rules from the operator.
-
-If you use the Helm chart `monitoring.enabled` value to deploy Prometheus rules, you may now
-additionally use `monitoring.createPrometheusRules` to instruct Helm to deploy the rules. You may
-alternately deploy the rules manually if you wish.
-
-To see the latest information about manually deploying rules, see the
-[Prometheus monitoring docs](../Storage-Configuration/Monitoring/ceph-monitoring.md#prometheus-alets).
-
-#### **MDS liveness and startup probes**
-
-If you configure MDS probes in the CephCluster resource, copy them to the
-[CephFilesystem `metadataServer` settings](../CRDs/Shared-Filesystem/ceph-filesystem-crd.md#metadata-server-settings) at this
-point. Do not remove them from the CephCluster until after the Rook upgrade is fully complete.
-
 ### **2. Update the Rook Operator**
 
 !!! hint
     If you are upgrading via the Helm chart, the operator is automatically updated.
 
-The largest portion of the upgrade is triggered when the operator's image is updated to `v1.9.x`.
+The largest portion of the upgrade is triggered when the operator's image is updated to `v1.10.x`.
 When the operator is updated, it will proceed to update all of the Ceph daemons.
 
 ```console
-kubectl -n $ROOK_OPERATOR_NAMESPACE set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.9.3
+kubectl -n $ROOK_OPERATOR_NAMESPACE set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:master
 ```
 
 ### **3. Update Ceph CSI**
@@ -220,8 +191,11 @@ kubectl -n $ROOK_OPERATOR_NAMESPACE set image deploy/rook-ceph-operator rook-cep
 !!! hint
     If have not customized the CSI image versions, this is automatically updated.
 
-If you have specified custom CSI images, we recommended you
-update to use the latest Ceph-CSI drivers. See the [CSI Custom Images](../Storage-Configuration/Ceph-CSI/custom-images.md) documentation.
+!!! important
+    The minimum supported version of Ceph-CSI is v3.6.0.
+
+If you have specified custom CSI images, we recommended you update to the latest Ceph-CSI drivers.
+See the [CSI Custom Images](../Storage-Configuration/Ceph-CSI/custom-images.md) documentation.
 
 !!! note
     If using snapshots, refer to the [Upgrade Snapshot API guide](../Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md#upgrade-snapshot-api).
@@ -239,18 +213,18 @@ watch --exec kubectl -n $ROOK_CLUSTER_NAMESPACE get deployments -l rook_cluster=
 ```
 
 As an example, this cluster is midway through updating the OSDs. When all deployments report `1/1/1`
-availability and `rook-version=v1.9.3`, the Ceph cluster's core components are fully updated.
+availability and `rook-version=v1.10.0`, the Ceph cluster's core components are fully updated.
 
 ```console
 Every 2.0s: kubectl -n rook-ceph get deployment -o j...
 
-rook-ceph-mgr-a         req/upd/avl: 1/1/1      rook-version=v1.9.3
-rook-ceph-mon-a         req/upd/avl: 1/1/1      rook-version=v1.9.3
-rook-ceph-mon-b         req/upd/avl: 1/1/1      rook-version=v1.9.3
-rook-ceph-mon-c         req/upd/avl: 1/1/1      rook-version=v1.9.3
-rook-ceph-osd-0         req/upd/avl: 1//        rook-version=v1.9.3
-rook-ceph-osd-1         req/upd/avl: 1/1/1      rook-version=v1.8.9
-rook-ceph-osd-2         req/upd/avl: 1/1/1      rook-version=v1.8.9
+rook-ceph-mgr-a         req/upd/avl: 1/1/1      rook-version=v1.10.0
+rook-ceph-mon-a         req/upd/avl: 1/1/1      rook-version=v1.10.0
+rook-ceph-mon-b         req/upd/avl: 1/1/1      rook-version=v1.10.0
+rook-ceph-mon-c         req/upd/avl: 1/1/1      rook-version=v1.10.0
+rook-ceph-osd-0         req/upd/avl: 1//        rook-version=v1.10.0
+rook-ceph-osd-1         req/upd/avl: 1/1/1      rook-version=v1.9.10
+rook-ceph-osd-2         req/upd/avl: 1/1/1      rook-version=v1.9.10
 ```
 
 An easy check to see if the upgrade is totally finished is to check that there is only one
@@ -259,14 +233,14 @@ An easy check to see if the upgrade is totally finished is to check that there i
 ```console
 # kubectl -n $ROOK_CLUSTER_NAMESPACE get deployment -l rook_cluster=$ROOK_CLUSTER_NAMESPACE -o jsonpath='{range .items[*]}{"rook-version="}{.metadata.labels.rook-version}{"\n"}{end}' | sort | uniq
 This cluster is not yet finished:
-  rook-version=v1.8.9
-  rook-version=v1.9.3
+  rook-version=v1.9.10
+  rook-version=v1.10.0
 This cluster is finished:
-  rook-version=v1.9.3
+  rook-version=v1.10.0
 ```
 
 ### **5. Verify the updated cluster**
 
-At this point, your Rook operator should be running version `rook/ceph:v1.9.3`.
+At this point, your Rook operator should be running version `rook/ceph:v1.10.0`.
 
 Verify the Ceph cluster's health using the [health verification doc](health-verification.md).
diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml
index 2289b7062..2864d73f8 100644
--- a/deploy/charts/rook-ceph/templates/configmap.yaml
+++ b/deploy/charts/rook-ceph/templates/configmap.yaml
@@ -17,7 +17,6 @@ data:
   CSI_ENABLE_ENCRYPTION: {{ .Values.csi.enableCSIEncryption | quote }}
   CSI_ENABLE_OMAP_GENERATOR: {{ .Values.csi.enableOMAPGenerator | quote }}
   CSI_ENABLE_HOST_NETWORK: {{ .Values.csi.enableCSIHostNetwork | quote }}
-  CSI_ENABLE_METADATA: {{ .Values.csi.enableMetadata | quote }}
 {{- if .Values.csi.pluginPriorityClassName }}
   CSI_PLUGIN_PRIORITY_CLASSNAME: {{ .Values.csi.pluginPriorityClassName | quote }}
 {{- end }}
@@ -178,9 +177,6 @@ data:
 {{- if .Values.csi.sidecarLogLevel }}
   CSI_SIDECAR_LOG_LEVEL: {{ .Values.csi.sidecarLogLevel | quote }}
 {{- end }}
-{{- if .Values.csi.clusterName }}
-  CSI_CLUSTER_NAME: {{ .Values.csi.clusterName | quote }}
-{{- end }}
 {{- if .Values.csi.grpcTimeoutInSeconds }}
   CSI_GRPC_TIMEOUT_SECONDS: {{ .Values.csi.grpcTimeoutInSeconds | quote }}
 {{- end }}
diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml
index 9b21fc54b..77e28f0d7 100644
--- a/deploy/charts/rook-ceph/values.yaml
+++ b/deploy/charts/rook-ceph/values.yaml
@@ -85,7 +85,7 @@ csi:
 
   # (Optional) policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted.
   # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
-  cephFSFSGroupPolicy: "File"
+  cephFSFSGroupPolicy: "ReadWriteOnceWithFSType"
 
   # (Optional) policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted.
   # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
@@ -98,17 +98,9 @@ csi:
   # sidecar with CSI provisioner pod, to enable set it to true.
   enableOMAPGenerator: false
 
-  # (Optional) set to true to enable adding volume metadata on the CephFS subvolumes and RBD images.
-  # Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images.
-  # Hence enable metadata is false by default.
-  enableMetadata: false
-
   # Set replicas for csi provisioner deployment.
   provisionerReplicas: 2
 
-  # (Optional) cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful
-  # in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster.
-  # clusterName: "my-prod-cluster"
   # Set logging level for cephCSI containers maintained by the cephCSI.
   # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
   #logLevel: 0
diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml
index b4ddb7bdd..2f8678c2e 100644
--- a/deploy/examples/operator-openshift.yaml
+++ b/deploy/examples/operator-openshift.yaml
@@ -122,13 +122,6 @@ data:
   # Set to true to enable Ceph CSI pvc encryption support.
   CSI_ENABLE_ENCRYPTION: "false"
 
-  # set to true to enable adding volume metadata on the CephFS subvolume and RBD images.
-  # CSI_ENABLE_METADATA: "true"
-
-  # cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases
-  # like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster.
-  # CSI_CLUSTER_NAME: "my-prod-cluster"
-
   # Set to true to enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary
   # in some network configurations where the SDN does not provide access to an external cluster or
   # there is significant drop in read/write performance.
@@ -169,7 +162,7 @@ data:
 
   # (Optional) policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted.
   # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
-  CSI_CEPHFS_FSGROUPPOLICY: "File"
+  CSI_CEPHFS_FSGROUPPOLICY: "ReadWriteOnceWithFSType"
 
   # (Optional) policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted.
   # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml
index c3ab1654d..e1a6a67ab 100644
--- a/deploy/examples/operator.yaml
+++ b/deploy/examples/operator.yaml
@@ -42,15 +42,6 @@ data:
   # there is significant drop in read/write performance.
   # CSI_ENABLE_HOST_NETWORK: "true"
 
-  # Set to true to enable adding volume metadata on the CephFS subvolume and RBD images.
-  # Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images.
-  # Hence enable metadata is false by default.
-  # CSI_ENABLE_METADATA: "true"
-
-  # cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases
-  # like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster.
-  # CSI_CLUSTER_NAME: "my-prod-cluster"
-
   # Set logging level for cephCSI containers maintained by the cephCSI.
   # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
   # CSI_LOG_LEVEL: "0"
@@ -86,7 +77,7 @@ data:
 
   # (Optional) policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted.
   # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
-  CSI_CEPHFS_FSGROUPPOLICY: "File"
+  CSI_CEPHFS_FSGROUPPOLICY: "ReadWriteOnceWithFSType"
 
   # (Optional) policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted.
   # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
@@ -101,7 +92,7 @@ data:
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.
-  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.7.0"
+  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.6.2"
   # ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1"
   # ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v1.4.0"
   # ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v3.1.0"
diff --git a/pkg/operator/ceph/cluster/mon/health.go b/pkg/operator/ceph/cluster/mon/health.go
index 4393cc54d..d9573189c 100644
--- a/pkg/operator/ceph/cluster/mon/health.go
+++ b/pkg/operator/ceph/cluster/mon/health.go
@@ -320,7 +320,7 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
 			logger.Warningf("cannot reduce mon quorum size from 2 to 1")
 		} else {
 			logger.Infof("removing an extra mon. currently %d are in quorum and only %d are desired", len(quorumStatus.MonMap.Mons), desiredMonCount)
-			return c.removeMon(c.determineExtraMonToRemove())
+			return c.removeMon(quorumStatus.MonMap.Mons[0].Name)
 		}
 	}
 
@@ -341,80 +341,6 @@ func (c *Cluster) checkHealth(ctx context.Context) error {
 	return nil
 }
 
-// determineExtraMonToRemove assumes all mons are in quorum and that there are more mons
-// that required for desired state. One mon will be picked for removal in this priority:
-// 1. If a stretch cluster, remove the extra mon according to the stretch topology
-// 2. If more than one mon on a node, remove one of them
-// 3. If no criteria require for 1 or 2, pick an arbitrary mon
-func (c *Cluster) determineExtraMonToRemove() string {
-	mons := c.clusterInfoToMonConfig()
-	if c.spec.IsStretchCluster() {
-		stretchMonToRemove := c.findExtraMonToRemoveFromStretchCluster(mons)
-		if stretchMonToRemove != "" {
-			return stretchMonToRemove
-		}
-		logger.Infof("did not find an extra mon to remove from the stretch cluster")
-		return ""
-	}
-
-	nodesWithMons := map[string]string{}
-	arbitraryMon := ""
-	for _, mon := range mons {
-		if mon.NodeName == "" {
-			logger.Debugf("mon %q is not scheduled to a specific host", mon.DaemonName)
-			continue
-		}
-		// Check if there are multiple mons on the node
-		if existingMon, ok := nodesWithMons[mon.NodeName]; ok {
-			logger.Infof("found mons %q and %q on node %s, removing mon %q", existingMon, mon.DaemonName, mon.NodeName, mon.DaemonName)
-			return mon.DaemonName
-		}
-		nodesWithMons[mon.NodeName] = mon.DaemonName
-
-		// assign the current mon as the fallback mon
-		arbitraryMon = mon.DaemonName
-	}
-
-	logger.Infof("removing arbitrary extra mon %q", arbitraryMon)
-	return arbitraryMon
-}
-
-func (c *Cluster) findExtraMonToRemoveFromStretchCluster(mons []*monConfig) string {
-	// Build the count of current mons per zone
-	zoneCount := map[string]int{}
-	monInZones := map[string]string{}
-	for _, m := range mons {
-		if m.Zone == "" {
-			logger.Warningf("zone not found on mon %q", m.DaemonName)
-			continue
-		}
-		zoneCount[m.Zone]++
-		// We just need the name of one of the mons in the zone in case there are extra
-		monInZones[m.Zone] = m.DaemonName
-	}
-
-	// Find a zone that has too many mons
-	for _, zone := range c.spec.Mon.StretchCluster.Zones {
-		count, ok := zoneCount[zone.Name]
-		if !ok {
-			// The zone isn't currently assigned to any mon, so skip it
-			continue
-		}
-		if zone.Arbiter {
-			if count > 1 {
-				logger.Infof("removing extra mon %q in arbiter zone %q", monInZones[zone.Name], zone.Name)
-				return monInZones[zone.Name]
-			}
-		} else {
-			if count > 2 {
-				logger.Infof("removing extra mon %q in zone %q", monInZones[zone.Name], zone.Name)
-				return monInZones[zone.Name]
-			}
-		}
-	}
-	return ""
-}
-
 // failMon compares the monCount against desiredMonCount
 // Returns whether the failover request was attempted. If false,
 // the operator should check for other mons to failover.
@@ -533,52 +459,38 @@ func (c *Cluster) updateMonDeploymentReplica(name string, enabled bool) error {
 func (c *Cluster) failoverMon(name string) error {
 	logger.Infof("Failing over monitor %q", name)
 
-	// remove the failed mon from a local list of the existing mons for finding a stretch zone
-	existingMons := c.clusterInfoToMonConfigWithExclude(name)
-
-	zone, err := c.findAvailableZoneIfStretched(existingMons)
-	if err != nil {
-		return errors.Wrap(err, "failed to find available stretch zone")
-	}
-
-	// Start a new monitor
-	m := c.newMonConfig(c.maxMonID+1, zone)
-	logger.Infof("starting new mon: %+v", m)
-
 	// Scale down the failed mon to allow a new one to start
 	if err := c.updateMonDeploymentReplica(name, false); err != nil {
 		// attempt to continue with the failover even if the bad mon could not be stopped
 		logger.Warningf("failed to stop mon %q for failover. %v", name, err)
 	}
-
-	// If the mon failover is not successful, revert the failover
 	newMonSucceeded := false
-	newMonMightBeInQuorum := false
 	defer func() {
 		if newMonSucceeded {
 			// do nothing if the new mon was started successfully, the deployment will anyway be deleted
 			return
 		}
-		logger.Warningf("failover of mon %q unsuccessful, cleaning up replacement mon %q", name, m.DaemonName)
 		if err := c.updateMonDeploymentReplica(name, true); err != nil {
 			// attempt to continue even if the bad mon could not be restarted
 			logger.Warningf("failed to restart failed mon %q after new mon wouldn't start. %v", name, err)
 		}
-		if err := c.removeMonWithOptionalQuorum(m.DaemonName, newMonMightBeInQuorum); err != nil {
-			logger.Infof("failed to remove mon %q from quorum. %v", m.DaemonName, err)
-		}
-
-		// Make sure the maxMonID is reverted to its previous value
-		// The maxMonId is committed to a configmap immediately after the mon deployment
-		// is started, even though c.maxMonID is not incremented until the mon failover is successful
-		logger.Infof("reverting maxMonId to %d", c.maxMonID)
-		if err := c.commitMaxMonIDRequireIncrementing(c.maxMonID, false); err != nil {
-			logger.Errorf("failed to revert maxMonId after starting mon %q", m.DaemonName)
-		}
 	}()
 
-	// Assign the pod to a node
+	// remove the failed mon from a local list of the existing mons for finding a stretch zone
+	existingMons := c.clusterInfoToMonConfig(name)
+
+	zone, err := c.findAvailableZoneIfStretched(existingMons)
+	if err != nil {
+		return errors.Wrap(err, "failed to find available stretch zone")
+	}
+
+	// Start a new monitor
+	m := c.newMonConfig(c.maxMonID+1, zone)
+	logger.Infof("starting new mon: %+v", m)
+
 	mConf := []*monConfig{m}
+
+	// Assign the pod to a node
 	if err := c.assignMons(mConf); err != nil {
 		return errors.Wrap(err, "failed to place new mon on a node")
 	}
@@ -600,7 +512,6 @@ func (c *Cluster) failoverMon(name string) error {
 	c.ClusterInfo.Monitors[m.DaemonName] = cephclient.NewMonInfo(m.DaemonName, m.PublicIP, m.Port)
 
 	// Start the deployment
-	newMonMightBeInQuorum = true
 	if err := c.startDeployments(mConf, true); err != nil {
 		return errors.Wrapf(err, "failed to start new mon %s", m.DaemonName)
 	}
@@ -622,15 +533,6 @@ func (c *Cluster) failoverMon(name string) error {
 
 // make a best effort to remove the mon and all its resources
 func (c *Cluster) removeMon(daemonName string) error {
-	return c.removeMonWithOptionalQuorum(daemonName, true)
-}
-
-// make a best effort to remove the mon and all its resources
-func (c *Cluster) removeMonWithOptionalQuorum(daemonName string, shouldRemoveFromQuorum bool) error {
-	if daemonName == "" {
-		logger.Info("did not identify a mon to remove")
-		return nil
-	}
 	logger.Infof("ensuring removal of unhealthy monitor %s", daemonName)
 
 	resourceName := resourceName(daemonName)
@@ -648,10 +550,8 @@ func (c *Cluster) removeMonWithOptionalQuorum(daemonName string, shouldRemoveFro
 	}
 
 	// Remove the bad monitor from quorum
-	if shouldRemoveFromQuorum {
-		if err := c.removeMonitorFromQuorum(daemonName); err != nil {
-			logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err)
-		}
+	if err := c.removeMonitorFromQuorum(daemonName); err != nil {
+		logger.Errorf("failed to remove mon %q from quorum. %v", daemonName, err)
 	}
 	delete(c.ClusterInfo.Monitors, daemonName)
 
diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go
index 0ad758ce8..7d7ade30d 100644
--- a/pkg/operator/ceph/cluster/mon/mon.go
+++ b/pkg/operator/ceph/cluster/mon/mon.go
@@ -127,8 +127,6 @@ type monConfig struct {
 	Port int32
 	// The zone used for a stretch cluster
 	Zone string
-	// The node where the mon is assigned
-	NodeName string
 	// DataPathMap is the mapping relationship between mon data stored on the host and mon data
 	// stored in containers.
 	DataPathMap *config.DataPathMap
@@ -517,7 +515,7 @@ func (c *Cluster) initClusterInfo(cephVersion cephver.CephVersion, clusterName s
 func (c *Cluster) initMonConfig(size int) (int, []*monConfig, error) {
 
 	// initialize the mon pod info for mons that have been previously created
-	mons := c.clusterInfoToMonConfig()
+	mons := c.clusterInfoToMonConfig("")
 
 	// initialize mon info if we don't have enough mons (at first startup)
 	existingCount := len(c.ClusterInfo.Monitors)
@@ -533,11 +531,7 @@ func (c *Cluster) initMonConfig(size int) (int, []*monConfig, error) {
 	return existingCount, mons, nil
 }
 
-func (c *Cluster) clusterInfoToMonConfig() []*monConfig {
-	return c.clusterInfoToMonConfigWithExclude("")
-}
-
-func (c *Cluster) clusterInfoToMonConfigWithExclude(excludedMon string) []*monConfig {
+func (c *Cluster) clusterInfoToMonConfig(excludedMon string) []*monConfig {
 	mons := []*monConfig{}
 	for _, monitor := range c.ClusterInfo.Monitors {
 		if monitor.Name == excludedMon {
@@ -545,11 +539,9 @@ func (c *Cluster) clusterInfoToMonConfigWithExclude(excludedMon string) []*monCo
 			continue
 		}
 		var zone string
-		var nodeName string
 		schedule := c.mapping.Schedule[monitor.Name]
 		if schedule != nil {
 			zone = schedule.Zone
-			nodeName = schedule.Name
 		}
 		mons = append(mons, &monConfig{
 			ResourceName: resourceName(monitor.Name),
@@ -557,7 +549,6 @@ func (c *Cluster) clusterInfoToMonConfigWithExclude(excludedMon string) []*monCo
 			Port:         cephutil.GetPortFromEndpoint(monitor.Endpoint),
 			PublicIP:     cephutil.GetIPFromEndpoint(monitor.Endpoint),
 			Zone:         zone,
-			NodeName:     nodeName,
 			DataPathMap:  config.NewStatefulDaemonDataPathMap(c.spec.DataDirHostPath, dataDirRelativeHostPath(monitor.Name), config.MonType, monitor.Name, c.Namespace),
 		})
 	}
@@ -1134,10 +1125,6 @@ func (c *Cluster) commitMaxMonID(monName string) error {
 		return errors.Wrapf(err, "invalid mon name %q", monName)
 	}
 
-	return c.commitMaxMonIDRequireIncrementing(committedMonID, true)
-}
-
-func (c *Cluster) commitMaxMonIDRequireIncrementing(desiredMaxMonID int, requireIncrementing bool) error {
 	configmap, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(c.ClusterInfo.Context, EndpointConfigMapName, metav1.GetOptions{})
 	if err != nil {
 		return errors.Wrap(err, "failed to find existing mon endpoint config map")
@@ -1149,13 +1136,13 @@ func (c *Cluster) commitMaxMonIDRequireIncrementing(desiredMaxMonID int, require
 		return errors.Wrap(err, "failed to read existing maxMonId")
 	}
 
-	if requireIncrementing && existingMax >= desiredMaxMonID {
-		logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", desiredMaxMonID, existingMax)
+	if existingMax >= committedMonID {
+		logger.Infof("no need to commit maxMonID %d since it is not greater than existing maxMonID %d", committedMonID, existingMax)
 		return nil
 	}
 
-	logger.Infof("updating maxMonID from %d to %d", existingMax, desiredMaxMonID)
-	configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(desiredMaxMonID)
+	logger.Infof("updating maxMonID from %d to %d after committing mon %q", existingMax, committedMonID, monName)
+	configmap.Data[controller.MaxMonIDKey] = strconv.Itoa(committedMonID)
 
 	if _, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(c.ClusterInfo.Context, configmap, metav1.UpdateOptions{}); err != nil {
 		return errors.Wrap(err, "failed to update mon endpoint config map for the maxMonID")
@@ -1340,7 +1327,7 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterInfo *cephclient.Cl
 
 	// wait for monitors to establish quorum
 	retryCount := 0
-	retryMax := 60
+	retryMax := 30
 	for {
 		// Return immediately if the context has been canceled
 		if clusterInfo.Context.Err() != nil {
diff --git a/pkg/operator/ceph/csi/betav1csidriver.go b/pkg/operator/ceph/csi/betav1csidriver.go
index cf7b487b8..efbf4c77f 100644
--- a/pkg/operator/ceph/csi/betav1csidriver.go
+++ b/pkg/operator/ceph/csi/betav1csidriver.go
@@ -52,11 +52,6 @@ func (d beta1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kuber
 		},
 	}
 	if fsGroupPolicy != "" {
-		if fsGroupPolicy == "File" {
-			// fsGroupPolicy: File was added in Kubernetes 1.19, considering beta1CsiDriver
-			// life exist before 1.19, we are falling back to `ReadWriteOnceWithFStype`.
-			fsGroupPolicy = "ReadWriteOnceWithFSType"
-		}
 		policy := betav1k8scsi.FSGroupPolicy(fsGroupPolicy)
 		csiDriver.Spec.FSGroupPolicy = &policy
 	}
diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go
index 87cffea51..14b110c10 100644
--- a/pkg/operator/ceph/csi/csi.go
+++ b/pkg/operator/ceph/csi/csi.go
@@ -180,10 +180,6 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error {
 		CSIParam.EnableCSIEncryption = true
 	}
 
-	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_METADATA", "false"), "true") {
-		CSIParam.CSIEnableMetadata = true
-	}
-
 	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) {
 		CSIParam.CephFSPluginUpdateStrategy = onDelete
 	} else {
@@ -265,7 +261,6 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error {
 	CSIParam.CSINFSPodLabels = k8sutil.ParseStringToLabels(csiNFSPodLabels)
 	csiRBDPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RBD_POD_LABELS", "")
 	CSIParam.CSIRBDPodLabels = k8sutil.ParseStringToLabels(csiRBDPodLabels)
-	CSIParam.CSIClusterName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_CLUSTER_NAME", "")
 
 	return nil
 }
diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
index 6077d6aff..ae0074816 100644
--- a/pkg/operator/ceph/csi/spec.go
+++ b/pkg/operator/ceph/csi/spec.go
@@ -57,9 +57,7 @@ type Param struct {
 	ProvisionerPriorityClassName   string
 	VolumeReplicationImage         string
 	CSIAddonsImage                 string
-	CSIClusterName                 string
 	GRPCTimeout                    time.Duration
-	CSIEnableMetadata              bool
 	EnablePluginSelinuxHostMount   bool
 	EnableCSIHostNetwork           bool
 	EnableOMAPGenerator            bool
diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
index d865a1e73..a4c023e6e 100644
--- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
+++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
@@ -114,12 +114,6 @@ spec:
             - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}"
             - "--metricspath=/metrics"
             - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
-            {{ if .CSIEnableMetadata }}
-            - "--setmetadata={{ .CSIEnableMetadata }}"
-            {{ end }}
-            {{ if .CSIClusterName }}
-            - "--clustername={{ .CSIClusterName }}"
-            {{ end }}
           env:
             - name: POD_IP
               valueFrom:
diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
index 243f5a7cb..5deaca9c9 100644
--- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
+++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
@@ -199,12 +199,6 @@ spec:
             {{ if .EnableCSIAddonsSideCar }}
             - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
             {{ end }}
-            {{ if .CSIEnableMetadata }}
-            - "--setmetadata={{ .CSIEnableMetadata }}"
-            {{ end }}
-            {{ if .CSIClusterName }}
-            - "--clustername={{ .CSIClusterName }}"
-            {{ end }}
           env:
             - name: POD_IP
               valueFrom:
diff --git a/pkg/operator/ceph/csi/version.go b/pkg/operator/ceph/csi/version.go
index fbea487a4..7546c1556 100644
--- a/pkg/operator/ceph/csi/version.go
+++ b/pkg/operator/ceph/csi/version.go
@@ -26,7 +26,7 @@ import (
 
 var (
 	//minimum supported version is 3.7.0
-	minimum = CephCSIVersion{3, 6, 0}
+	minimum = CephCSIVersion{3, 7, 0}
 	//supportedCSIVersions are versions that rook supports
 	releasev370 = CephCSIVersion{3, 7, 0}
 


Test patch:
diff --git a/pkg/operator/ceph/cluster/mon/health_test.go b/pkg/operator/ceph/cluster/mon/health_test.go
index 25131b628..2d6c70b26 100644
--- a/pkg/operator/ceph/cluster/mon/health_test.go
+++ b/pkg/operator/ceph/cluster/mon/health_test.go
@@ -150,62 +150,6 @@ func TestCheckHealth(t *testing.T) {
 	}
 }
 
-func TestRemoveExtraMon(t *testing.T) {
-	endpoint := "1.2.3.4:6789"
-	c := &Cluster{mapping: &opcontroller.Mapping{}}
-	c.ClusterInfo = &cephclient.ClusterInfo{Monitors: map[string]*cephclient.MonInfo{
-		"a": {Name: "a", Endpoint: endpoint},
-		"b": {Name: "b", Endpoint: endpoint},
-		"c": {Name: "c", Endpoint: endpoint},
-		"d": {Name: "d", Endpoint: endpoint},
-	}}
-	c.mapping.Schedule = map[string]*opcontroller.MonScheduleInfo{
-		"a": {Name: "node1"},
-		"b": {Name: "node2"},
-		"c": {Name: "node3"},
-		"d": {Name: "node1"},
-	}
-	// Remove mon an extra mon on the same node
-	removedMon := c.determineExtraMonToRemove()
-	if removedMon != "a" && removedMon != "d" {
-		assert.Fail(t, fmt.Sprintf("removed mon %q instead of a or d", removedMon))
-	}
-
-	// Remove an arbitrary mon that are all on different nodes
-	c.mapping.Schedule["d"].Name = "node4"
-	removedMon = c.determineExtraMonToRemove()
-	assert.NotEqual(t, "", removedMon)
-
-	// Don't remove any extra mon from a proper stretch cluster
-	c.spec.Mon.StretchCluster = &cephv1.StretchClusterSpec{Zones: []cephv1.StretchClusterZoneSpec{
-		{Name: "x", Arbiter: true},
-		{Name: "y"},
-		{Name: "z"},
-	}}
-	c.ClusterInfo.Monitors["e"] = &cephclient.MonInfo{Name: "e", Endpoint: endpoint}
-	c.mapping.Schedule["a"].Zone = "x"
-	c.mapping.Schedule["b"].Zone = "y"
-	c.mapping.Schedule["c"].Zone = "y"
-	c.mapping.Schedule["d"].Zone = "z"
-	c.mapping.Schedule["e"] = &opcontroller.MonScheduleInfo{Name: "node5", Zone: "z"}
-	removedMon = c.determineExtraMonToRemove()
-	assert.Equal(t, "", removedMon)
-
-	// Remove an extra mon from the arbiter zone
-	c.mapping.Schedule["d"].Zone = "x"
-	removedMon = c.determineExtraMonToRemove()
-	if removedMon != "a" && removedMon != "d" {
-		assert.Fail(t, "removed mon %q instead of a or d from the arbiter zone", removedMon)
-	}
-
-	// Remove an extra mon from a non-arbiter zone
-	c.mapping.Schedule["d"].Zone = "y"
-	removedMon = c.determineExtraMonToRemove()
-	if removedMon != "b" && removedMon != "c" && removedMon != "d" {
-		assert.Fail(t, fmt.Sprintf("removed mon %q instead of b, c, or d from the non-arbiter zone", removedMon))
-	}
-}
-
 func TestSkipMonFailover(t *testing.T) {
 	c := New(context.TODO(), &clusterd.Context{}, "ns", cephv1.ClusterSpec{}, nil)
 	c.ClusterInfo = clienttest.CreateTestClusterInfo(1)
diff --git a/pkg/operator/ceph/csi/version_test.go b/pkg/operator/ceph/csi/version_test.go
index 84e8c6ed3..b155829d8 100644
--- a/pkg/operator/ceph/csi/version_test.go
+++ b/pkg/operator/ceph/csi/version_test.go
@@ -23,7 +23,7 @@ import (
 )
 
 var (
-	testMinVersion         = CephCSIVersion{3, 6, 0}
+	testMinVersion         = CephCSIVersion{3, 7, 0}
 	testReleaseV350        = CephCSIVersion{3, 5, 0}
 	testReleaseV360        = CephCSIVersion{3, 6, 0}
 	testReleaseV361        = CephCSIVersion{3, 6, 1}
@@ -64,7 +64,7 @@ func TestSupported(t *testing.T) {
 	assert.Equal(t, false, ret)
 
 	ret = testReleaseV360.Supported()
-	assert.Equal(t, true, ret)
+	assert.Equal(t, false, ret)
 
 	ret = testReleaseV370.Supported()
 	assert.Equal(t, true, ret)