Change:
[It] [sig-storage] CSI Mock selinux on mount metrics SELinuxMount metrics [LinuxOnly][Feature:SELinux][Feature:SELinuxMountReadWriteOncePod][Serial] warning is not bumped on two Pods with the same context on RWO volume [sig-storage] [It] [sig-storage] CSI Mock selinux on mount metrics SELinuxMount metrics [LinuxOnly][Feature:SELinux][Feature:SELinuxMountReadWriteOncePod][Serial] warning is bumped on two Pods with a different context on RWO volume [sig-storage] Sample failure: October 25, 2023 It fails since The PR a label to some SELinux metric and the tests do not count with that. No response /sig storage
/assign /triage accepted

Discussion:
"context" "fmt" "sort" <ins> "strings" </ins> "sync/atomic" "time"

// [Serial]: the tests read global kube-controller-manager metrics, so no other test changes them in parallel. f.Context("SELinuxMount metrics [LinuxOnly]", feature.SELinux, feature.SELinuxMountReadWriteOncePod, f.WithSerial(), func() { <del> // All SELinux metrics. Unless explicitly mentioned in test.expectIncreases, these metrics must not grow during // a test. allMetrics := sets.NewString( "volume_manager_selinux_container_errors_total", "volume_manager_selinux_container_warnings_total", "volume_manager_selinux_pod_context_mismatch_errors_total", "volume_manager_selinux_pod_context_mismatch_warnings_total", "volume_manager_selinux_volume_context_mismatch_errors_total", "volume_manager_selinux_volume_context_mismatch_warnings_total", "volume_manager_selinux_volumes_admitted_total", ) </del> // Make sure all options are set so system specific defaults are not used. seLinuxOpts1 := v1.SELinuxOptions{ User:  "system_u",

for _, t := range tests { t := t ginkgo.It(t.name, func(ctx context.Context) { <ins> // Some metrics use CSI driver name as a label, which is "csi-mock-" + the namespace name. volumePluginLabel := "{volume_plugin="kubernetes.io/csi/csi-mock-" + f.Namespace.Name + ""}" // All SELinux metrics. Unless explicitly mentioned in test.expectIncreases, these metrics must not grow during // a test. allMetrics := sets.NewString( "volume_manager_selinux_container_errors_total", "volume_manager_selinux_container_warnings_total", "volume_manager_selinux_pod_context_mismatch_errors_total", "volume_manager_selinux_pod_context_mismatch_warnings_total", "volume_manager_selinux_volume_context_mismatch_errors_total"+volumePluginLabel, "volume_manager_selinux_volume_context_mismatch_warnings_total"+volumePluginLabel, "volume_manager_selinux_volumes_admitted_total"+volumePluginLabel, ) </ins> if framework.NodeOSDistroIs("windows") { e2eskipper.Skipf("SELinuxMount is only applied on linux nodes -- skipping") }

framework.ExpectNoError(err) metrics := map[string]float64{} <del> for method, samples := range response { if metricNames.Has(method) { if len(samples) == 0 { return nil, fmt.Errorf("metric %s has no samples", method) } lastSample := samples[len(samples)-1] metrics[method] = float64(lastSample.Value) </del> <ins> for _, samples := range response { if len(samples) == 0 { continue </ins> } <del> } // Ensure all metrics were provided for name := range metricNames { if _, found := metrics[name]; !found { return nil, fmt.Errorf("metric %s not found", name) </del> <ins> // Find the *last* sample that has the label we are interested in. for i := len(samples) - 1; i >= 0; i-- { metricNameWithLabels := samples[i].Metric.String() if metricNames.Has(metricNameWithLabels) { metrics[metricNameWithLabels] = float64(samples[i].Value) break } </ins> } }

noIncreaseMetrics = sets.NewString() // Always evaluate all SELinux metrics to check that the other metrics are not unexpectedly increased. for name := range allMetricNames { <del> if expectedIncreaseNames.Has(name) { </del> <ins> expectIncrease := false // allMetricNames can include "{volume_plugin="XXX"}", while expectedIncreaseNames does not. // Compare them properly. Value of volume_plugin="XXX" was already checked in grabMetrics(), // we can ignore it here. for expectedIncreaseMetricName := range expectedIncreaseNames { if strings.HasPrefix(name, expectedIncreaseMetricName) { expectIncrease = true break } } if expectIncrease { </ins> if metrics[name] <= initialValues[name] { noIncreaseMetrics.Insert(name) } } else { <ins> // Expect the metric to be stable </ins> if initialValues[name] != metrics[name] { return false, fmt.Errorf("metric %s unexpectedly increased to %v", name, metrics[name]) }