Repository: Azure/ARO-RP
Pull request #1300 (2021-02-08 17:16:58)

disable PersistentPrometheus
Disable persistent Prometheus 


Patch:
diff --git a/docs/prepare-a-shared-rp-development-environment.md b/docs/prepare-a-shared-rp-development-environment.md
index 9fa121746..7caa9d33f 100644
--- a/docs/prepare-a-shared-rp-development-environment.md
+++ b/docs/prepare-a-shared-rp-development-environment.md
@@ -167,19 +167,6 @@ locations.
    * `Contributor`  on your subscription.
    * `User Access Administrator` on your subscription.
 
-   You must also manually grant this application the `Microsoft.Graph/Application.ReadWrite.OwnedBy` permission, which requires admin access, in order for AAD applications to be created/deleted on a per-cluster basis.
-
-   * Go into the Azure Portal
-   * Go to Azure Active Directory
-   * Navigate to the `aro-v4-tooling-shared` app page
-   * Click 'API permissions' in the left side pane
-   * Click 'Microsoft Graph'
-   * Click 'Add a permission'.
-   * Select 'Application permissions'
-   * Search for 'Application' and select `Application.ReadWrite.OwnedBy`
-   * Click 'Add permissions'
-   * This request will need to be approved by a tenant administrator. If you are one, you can click the `Grant admin consent for <name>` button to the right of the `Add a permission` button on the app page
-
 1. Set up the RP role definitions and subscription role assignments in your
    Azure subscription. This mimics the RBAC that ARM sets up.  With at least
    `User Access Administrator` permissions on your subscription, do:
@@ -314,14 +301,10 @@ locations.
    ADMIN_OBJECT_ID='$ADMIN_OBJECT_ID'
    PARENT_DOMAIN_NAME='$PARENT_DOMAIN_NAME'
    PARENT_DOMAIN_RESOURCEGROUP='$PARENT_DOMAIN_RESOURCEGROUP'
-   export DOMAIN_NAME="\$LOCATION.\$PARENT_DOMAIN_NAME"
-   export AZURE_ENVIRONMENT='AzurePublicCloud'
+   export DOMAIN_NAME="\$RESOURCEGROUP.\$PARENT_DOMAIN_NAME"
    EOF
    ```
 
-After creating this file, run `make secrets-update` to upload it to your
-storage account so other people on your team can access it via `make secrets`.
-
 ## Deploy shared RP development environment (once per location)
 
 Look at the [helper file](../hack/devtools/deploy-shared-env.sh) to understand
@@ -371,10 +354,9 @@ each of the bash functions below.
    import_certs_secrets
    ```
 
-   Note: in production, three additional keys/certificates (rp-mdm, rp-mdsd, and
-   cluster-mdsd) are also required in the $KEYVAULT_PREFIX-svc key vault.  These
-   are client certificates for RP metric and log forwarding (respectively) to
-   Geneva.
+   Note: in production, two additional keys/certificates (rp-mdm and rp-mdsd)
+   are also required in the $KEYVAULT_PREFIX-svc key vault.  These are client
+   certificates for RP metric and log forwarding (respectively) to Geneva.
 
    If you need them in development:
 
@@ -389,17 +371,8 @@ each of the bash functions below.
         --vault-name "$KEYVAULT_PREFIX-svc" \
         --name rp-mdsd \
         --file secrets/rp-logging-int.pem
-   az keyvault certificate import \
-        --vault-name "$KEYVAULT_PREFIX-svc" \
-        --name cluster-mdsd \
-        --file secrets/cluster-logging-int.pem
    ```
 
-   Note: in development, if you don't have valid certs for these, you can just
-   upload `localhost.pem` as a placeholder for each of these. This will avoid an
-   error stemming from them not existing, but it will result in logging pods
-   crash looping in any clusters you make.
-
 1. In pre-production (int, e2e) certain certificates are provisioned via keyvault
 integration. These should be rotated and generated in the keyvault itself:
 
diff --git a/pkg/deploy/upgrade.go b/pkg/deploy/upgrade.go
index 26ddf0c02..7104eba9e 100644
--- a/pkg/deploy/upgrade.go
+++ b/pkg/deploy/upgrade.go
@@ -30,6 +30,12 @@ func (d *deployer) Upgrade(ctx context.Context) error {
 		return err
 	}
 
+	// maybe starting the new RP and stopping all the old ones in the same
+	// minute is causing some of our alert noisiness.  Let's try a sleep and see
+	// if that helps.
+	d.log.Print("sleeping 5 minutes")
+	time.Sleep(5 * time.Minute)
+
 	err = d.removeOldScalesets(ctx)
 	if err != nil {
 		return err
diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go
index 40f4912ad..ab459b146 100644
--- a/pkg/frontend/frontend.go
+++ b/pkg/frontend/frontend.go
@@ -66,7 +66,8 @@ type frontend struct {
 
 	bucketAllocator bucket.Allocator
 
-	ready atomic.Value
+	startTime time.Time
+	ready     atomic.Value
 
 	now func() time.Time
 }
@@ -106,6 +107,8 @@ func NewFrontend(ctx context.Context,
 
 		bucketAllocator: &bucket.Random{},
 
+		startTime: time.Now(),
+
 		now: time.Now,
 	}
 
diff --git a/pkg/frontend/ready_get.go b/pkg/frontend/ready_get.go
index 6cf347a28..6f3239bbb 100644
--- a/pkg/frontend/ready_get.go
+++ b/pkg/frontend/ready_get.go
@@ -5,13 +5,22 @@ package frontend
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/util/deployment"
 )
 
 // checkReady checks the ready status of the frontend to make it consistent
-// across the /healthz/ready endpoint and emitted metrics.
+// across the /healthz/ready endpoint and emitted metrics.   We wait for 2
+// minutes before indicating health.  This ensures that there will be a gap in
+// our health metric if we crash or restart.
 func (f *frontend) checkReady() bool {
+	if f.env.DeploymentMode() != deployment.Development &&
+		time.Since(f.startTime) < 2*time.Minute {
+		return false
+	}
+
 	return f.ready.Load().(bool) &&
 		f.env.ArmClientAuthorizer().IsReady() &&
 		f.env.AdminClientAuthorizer().IsReady()
diff --git a/pkg/monitor/monitor.go b/pkg/monitor/monitor.go
index b4d6db361..0270fd732 100644
--- a/pkg/monitor/monitor.go
+++ b/pkg/monitor/monitor.go
@@ -41,6 +41,7 @@ type monitor struct {
 
 	lastBucketlist atomic.Value //time.Time
 	lastChangefeed atomic.Value //time.Time
+	startTime      time.Time
 }
 
 type Runnable interface {
@@ -63,6 +64,8 @@ func NewMonitor(log *logrus.Entry, dialer proxy.Dialer, dbMonitors database.Moni
 
 		bucketCount: bucket.Buckets,
 		buckets:     map[int]struct{}{},
+
+		startTime: time.Now(),
 	}
 }
 
@@ -121,5 +124,6 @@ func (mon *monitor) checkReady() bool {
 		return false
 	}
 	return (time.Since(lastBucketTime) < time.Minute) && // did we list buckets successfully recently?
-		(time.Since(lastChangefeedTime) < time.Minute) // did we process the change feed recently?
+		(time.Since(lastChangefeedTime) < time.Minute) && // did we process the change feed recently?
+		(time.Since(mon.startTime) > 2*time.Minute) // are we running for at least 2 minutes?
 }
diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go
index 023dcfb31..859f404a3 100644
--- a/pkg/operator/deploy/deploy.go
+++ b/pkg/operator/deploy/deploy.go
@@ -181,7 +181,7 @@ func (o *operator) resources() ([]runtime.Object, error) {
 					},
 				},
 				Features: arov1alpha1.FeaturesSpec{
-					PersistentPrometheus: true,
+					PersistentPrometheus: false,
 				},
 			},
 		},


Test patch:
diff --git a/pkg/frontend/security_test.go b/pkg/frontend/security_test.go
index 798be3d47..36c779139 100644
--- a/pkg/frontend/security_test.go
+++ b/pkg/frontend/security_test.go
@@ -10,6 +10,7 @@ import (
 	"crypto/x509"
 	"net/http"
 	"testing"
+	"time"
 
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/golang/mock/gomock"
@@ -79,6 +80,7 @@ func TestSecurity(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	f.(*frontend).startTime = time.Time{} // enable /healthz to return 200
 
 	go f.Run(ctx, nil, nil)
 
diff --git a/test/e2e/operator.go b/test/e2e/operator.go
index 218334c95..49483a462 100644
--- a/test/e2e/operator.go
+++ b/test/e2e/operator.go
@@ -183,7 +183,7 @@ var _ = Describe("ARO Operator - Routefix Daemonset", func() {
 })
 
 var _ = Describe("ARO Operator - Cluster Monitoring ConfigMap", func() {
-	Specify("cluster monitoring configmap should have persistent volume config", func() {
+	Specify("cluster monitoring configmap should not have persistent volume config", func() {
 		var cm *v1.ConfigMap
 		var err error
 		configMapExists := func() (bool, error) {
@@ -206,8 +206,8 @@ var _ = Describe("ARO Operator - Cluster Monitoring ConfigMap", func() {
 			log.Warn(err)
 		}
 
-		Expect(configData.PrometheusK8s.Retention).To(Equal("15d"))
-		Expect(configData.PrometheusK8s.VolumeClaimTemplate.Spec.Resources.Requests.Storage).To(Equal("100Gi"))
+		Expect(configData.PrometheusK8s.Retention).To(Equal(""))
+		Expect(configData.PrometheusK8s.VolumeClaimTemplate.Spec.Resources.Requests.Storage).To(Equal(""))
 	})
 
 	Specify("cluster monitoring configmap should be restored if deleted", func() {