Change:
Which jobs are flaking: () () Which test(s) are flaking: Reason for failure: Timeout: Anything else we need to know: This test flaking about 10% of the time in this job. /sig storage /kind flake /priority important-soon cc
/remove-kind failing-test
/assign
I noticed one issue with the test is that it's not filtering metrics by plugin name. So it will just count the last plugin's metric. I'm not sure if the metrics collected are in any sorted order or not. If not, then when you have a suite that runs multiple plugins, then you might be pulling from the wrong metric.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/remove-lifecycle rotten
still very prevalent in sig-release dashboards
I have pushed which should hopefully fix this.
Also PR against 1.17 branch -

Discussion:
framework.Skipf("Environment does not support getting controller-manager metrics - skipping") } <ins> ginkgo.By("Getting plugin name") defaultClass, err := c.StorageV1().StorageClasses().Get(defaultScName, metav1.GetOptions{}) framework.ExpectNoError(err, "Error getting default storageclass: %v", err) pluginName := defaultClass.Provisioner </ins> controllerMetrics, err := metricsGrabber.GrabFromControllerManager() framework.ExpectNoError(err, "Error getting c-m metrics : %v", err) <del> storageOpMetrics := getControllerStorageMetrics(controllerMetrics) </del> <ins> storageOpMetrics := getControllerStorageMetrics(controllerMetrics, pluginName) </ins> pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc) framework.ExpectNoError(err)

framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name) framework.ExpectNoError(e2epod.DeletePodWithWait(c, pod)) <del> updatedStorageMetrics := waitForDetachAndGrabMetrics(storageOpMetrics, metricsGrabber) </del> <ins> updatedStorageMetrics := waitForDetachAndGrabMetrics(storageOpMetrics, metricsGrabber, pluginName) </ins> framework.ExpectNotEqual(len(updatedStorageMetrics.latencyMetrics), 0, "Error fetching c-m updated storage metrics") framework.ExpectNotEqual(len(updatedStorageMetrics.statusMetrics), 0, "Error fetching c-m updated storage metrics")

framework.Skipf("Environment does not support getting controller-manager metrics - skipping") } <ins> ginkgo.By("Geting default storageclass") defaultClass, err := c.StorageV1().StorageClasses().Get(defaultScName, metav1.GetOptions{}) framework.ExpectNoError(err, "Error getting default storageclass: %v", err) pluginName := defaultClass.Provisioner </ins> controllerMetrics, err := metricsGrabber.GrabFromControllerManager() framework.ExpectNoError(err, "Error getting c-m metrics : %v", err) <del> storageOpMetrics := getControllerStorageMetrics(controllerMetrics) ginkgo.By("Creating an invalid storageclass") defaultClass, err := c.StorageV1().StorageClasses().Get(defaultScName, metav1.GetOptions{}) framework.ExpectNoError(err, "Error getting default storageclass: %v", err) </del> <ins> storageOpMetrics := getControllerStorageMetrics(controllerMetrics, pluginName) </ins> invalidSc = &storagev1.StorageClass{ ObjectMeta: metav1.ObjectMeta{

ginkgo.By("Checking failure metrics") updatedControllerMetrics, err := metricsGrabber.GrabFromControllerManager() framework.ExpectNoError(err, "failed to get controller manager metrics") <del> updatedStorageMetrics := getControllerStorageMetrics(updatedControllerMetrics) </del> <ins> updatedStorageMetrics := getControllerStorageMetrics(updatedControllerMetrics, pluginName) </ins> framework.ExpectNotEqual(len(updatedStorageMetrics.statusMetrics), 0, "Error fetching c-m updated storage metrics") verifyMetricCount(storageOpMetrics, updatedStorageMetrics, "volume_provision", true)

} } <del> func waitForDetachAndGrabMetrics(oldMetrics *storageControllerMetrics, metricsGrabber *metrics.Grabber) *storageControllerMetrics { </del> <ins> func waitForDetachAndGrabMetrics(oldMetrics *storageControllerMetrics, metricsGrabber *metrics.Grabber, pluginName string) *storageControllerMetrics { </ins> backoff := wait.Backoff{ Duration: 10 * time.Second, Factor:   1.2,

return false, err } <del> updatedStorageMetrics = getControllerStorageMetrics(updatedMetrics) </del> <ins> updatedStorageMetrics = getControllerStorageMetrics(updatedMetrics, pluginName) </ins> newDetachCount, ok := updatedStorageMetrics.latencyMetrics["volume_detach"] // if detach metrics are not yet there, we need to retry

if oldDetachCount >= newDetachCount { return false, nil } return true, nil }

gomega.Expect(newStatusCount).To(gomega.BeNumerically(">", oldStatusCount), "New status count %d should be more than old count %d for action %s", newStatusCount, oldStatusCount, metricName) } <del> func getControllerStorageMetrics(ms metrics.ControllerManagerMetrics) *storageControllerMetrics { </del> <ins> func getControllerStorageMetrics(ms metrics.ControllerManagerMetrics, pluginName string) *storageControllerMetrics { </ins> result := newStorageControllerMetrics() for method, samples := range ms {

for _, sample := range samples { count := int64(sample.Value) operation := string(sample.Metric["operation_name"]) <ins> metricPluginName := string(sample.Metric["volume_plugin"]) if len(pluginName) > 0 && pluginName != metricPluginName { continue } </ins> result.latencyMetrics[operation] = count } case "storage_operation_status_count":

operation := string(sample.Metric["operation_name"]) status := string(sample.Metric["status"]) statusCounts := result.statusMetrics[operation] <ins> metricPluginName := string(sample.Metric["volume_plugin"]) if len(pluginName) > 0 && pluginName != metricPluginName { continue } </ins> switch status { case "success": statusCounts.successCount = count