Code change:
<!-- Please only use this template for submitting enhancement requests --What would you like to be : We should allow collecting both default scheduler metrics (Prometheus) and custom metrics easily. At a high level, these collectors should produce a json formatted result which will then be consumed by perf-dash to present in the dashboard. Prometheus metrics We should provide a metrics collector that collects and summarizes default scheduler metrics defined in . To begin with, we can manually pick a list of interesting metrics. Longer term, we could allow configuring the metric list in test definition. already had a PoC . As a reference, is how clusterloader collects scheduler latency metrics. Custom metrics The most important metric is scheduling throughput. is how clusterloader calculates the throughput. We can start with just calculating the average throughput without the percentiles. We may want other custom metrics that are either common to all tests or specific to certain tests. We should allow these custom collectors to be easily and ideally configurable. Why is this needed: Give more insights into scheduler performance. Part of /sig scheduling /assign
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Related discussion:
text: } return scheduled, nil } <ins> // DataItem is the data point. type DataItem struct { // Data is a map from bucket to real data point (e.g. "Perc90" -> 23.5). Notice // that all data items with the same label combination should have the same buckets. Data map[string]float64 `json:"data"` // Unit is the data unit. Notice that all data items with the same label combination // should have the same unit. Unit string `json:"unit"` // Labels is the labels of the data item. Labels map[string]string `json:"labels,omitempty"` } // DataItems is the data point set. It is the struct that perf dashboard expects. type DataItems struct { Version   string     `json:"version"` DataItems []DataItem `json:"dataItems"` } func dataItems2JSONFile(dataItems DataItems, namePrefix string) error { b, err := json.Marshal(dataItems) if err != nil { return err } destFile := fmt.Sprintf("%v_%v.json", namePrefix, time.Now().Format(dateFormat)) if *dataItemsDir != "" { destFile = path.Join(*dataItemsDir, destFile) } return ioutil.WriteFile(destFile, b, 0644) } // prometheusCollector collects metrics from legacyregistry.DefaultGatherer.Gather() endpoint. // Currently only Histrogram metrics are supported. type prometheusCollector struct { metric string cache  *dto.MetricFamily } func newPrometheusCollector(metric string) *prometheusCollector { return &prometheusCollector{ metric: metric, } } func (pc *prometheusCollector) collect() *DataItem { var metricFamily *dto.MetricFamily m, err := legacyregistry.DefaultGatherer.Gather() if err != nil { klog.Error(err) return nil } for _, mFamily := range m { if mFamily.Name != nil && *mFamily.Name == pc.metric { metricFamily = mFamily break } } if metricFamily == nil { klog.Infof("Metric %q not found", pc.metric) return nil } if metricFamily.GetMetric() == nil { klog.Infof("Metric %q is empty", pc.metric) return nil } if len(metricFamily.GetMetric()) == 0 { klog.Infof("Metric %q is empty", pc.metric) return nil } // Histograms are stored under the first index (based on observation). // Given there's only one histogram registered per each metric name, accessaing // the first index is sufficient. dataItem := pc.promHist2Summary(metricFamily.GetMetric()[0].GetHistogram()) if dataItem.Data == nil { return nil } // clear the metrics so that next test always starts with empty prometheus // metrics (since the metrics are shared among all tests run inside the same binary) clearPromHistogram(metricFamily.GetMetric()[0].GetHistogram()) return dataItem } // Bucket of a histogram type bucket struct { upperBound float64 count      float64 } func bucketQuantile(q float64, buckets []bucket) float64 { if q < 0 { return math.Inf(-1) } if q > 1 { return math.Inf(+1) } if len(buckets) < 2 { return math.NaN() } rank := q * buckets[len(buckets)-1].count b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank }) if b == 0 { return buckets[0].upperBound * (rank / buckets[0].count) } // linear approximation of b-th bucket brank := rank - buckets[b-1].count bSize := buckets[b].upperBound - buckets[b-1].upperBound bCount := buckets[b].count - buckets[b-1].count return buckets[b-1].upperBound + bSize*(brank/bCount) } func (pc *prometheusCollector) promHist2Summary(hist *dto.Histogram) *DataItem { buckets := []bucket{} if hist.SampleCount == nil || *hist.SampleCount == 0 { return &DataItem{} } if hist.SampleSum == nil || *hist.SampleSum == 0 { return &DataItem{} } for _, bckt := range hist.Bucket { if bckt == nil { return &DataItem{} } if bckt.UpperBound == nil || *bckt.UpperBound < 0 { return &DataItem{} } buckets = append(buckets, bucket{ count:      float64(*bckt.CumulativeCount), upperBound: *bckt.UpperBound, }) } // bucketQuantile expects the upper bound of the last bucket to be +inf buckets[len(buckets)-1].upperBound = math.Inf(+1) q50 := bucketQuantile(0.50, buckets) q90 := bucketQuantile(0.90, buckets) q99 := bucketQuantile(0.95, buckets) msFactor := float64(time.Second) / float64(time.Millisecond) return &DataItem{ Labels: map[string]string{ "Metric": pc.metric, }, Data: map[string]float64{ "Perc50":  q50 * msFactor, "Perc90":  q90 * msFactor, "Perc99":  q99 * msFactor, "Average": (*hist.SampleSum / float64(*hist.SampleCount)) * msFactor, }, Unit: "ms", } } func clearPromHistogram(hist *dto.Histogram) { if hist.SampleCount != nil { *hist.SampleCount = 0 } if hist.SampleSum != nil { *hist.SampleSum = 0 } for _, b := range hist.Bucket { if b.CumulativeCount != nil { *b.CumulativeCount = 0 } if b.UpperBound != nil { *b.UpperBound = 0 } } } type throughputCollector struct { podInformer           coreinformers.PodInformer schedulingThroughputs []float64 } func newThroughputCollector(podInformer coreinformers.PodInformer) *throughputCollector { return &throughputCollector{ podInformer: podInformer, } } func (tc *throughputCollector) run(stopCh chan struct{}) { podsScheduled, err := getScheduledPods(tc.podInformer) if err != nil { klog.Fatalf("%v", err) } lastScheduledCount := len(podsScheduled) for { select { case <-stopCh: return case <-time.After(throughputSampleFrequency): podsScheduled, err := getScheduledPods(tc.podInformer) if err != nil { klog.Fatalf("%v", err) } scheduled := len(podsScheduled) samplingRatioSeconds := float64(throughputSampleFrequency) / float64(time.Second) throughput := float64(scheduled-lastScheduledCount) / samplingRatioSeconds tc.schedulingThroughputs = append(tc.schedulingThroughputs, throughput) lastScheduledCount = scheduled klog.Infof("%d pods scheduled", lastScheduledCount) } } } func (tc *throughputCollector) collect() *DataItem { throughputSummary := &DataItem{} if length := len(tc.schedulingThroughputs); length > 0 { sort.Float64s(tc.schedulingThroughputs) sum := 0.0 for i := range tc.schedulingThroughputs { sum += tc.schedulingThroughputs[i] } throughputSummary.Labels = map[string]string{ "Metric": "SchedulingThroughput", } throughputSummary.Data = map[string]float64{ "Average": sum / float64(length), "Perc50":  tc.schedulingThroughputs[int(math.Ceil(float64(length*50)/100))-1], "Perc90":  tc.schedulingThroughputs[int(math.Ceil(float64(length*90)/100))-1], "Perc99":  tc.schedulingThroughputs[int(math.Ceil(float64(length*99)/100))-1], } throughputSummary.Unit = "pods/s" } return throughputSummary } </ins>