Repository: prometheus/prometheus
File: cmd/promtool/unittest.go (Go)

Code before:

		sort.Slice(expSamples, func(i, j int) bool {
			return labels.Compare(expSamples[i].Labels, expSamples[j].Labels) <= 0
		})
		sort.Slice(gotSamples, func(i, j int) bool {
			return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
		})
		if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal)) {
			errs = append(errs, fmt.Errorf("    expr: %q, time: %s,\n        exp: %v\n        got: %v", testCase.Expr,
				testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
		}
	}

	if len(errs) > 0 {
		return errs
	}
	return nil
}

// seriesLoadingString returns the input series in PromQL notation.
func (tg *testGroup) seriesLoadingString() string {
	result := fmt.Sprintf("load %v\n", shortDuration(tg.Interval))
	for _, is := range tg.InputSeries {
		result += fmt.Sprintf("  %v %v\n", is.Series, is.Values)
	}
	return result
}

func shortDuration(d model.Duration) string {
	s := d.String()
	if strings.HasSuffix(s, "m0s") {
		s = s[:len(s)-2]
	}
	if strings.HasSuffix(s, "h0m") {
		s = s[:len(s)-2]
	}
	return s
}

// orderedGroups returns a slice of `*rules.Group` from `groupsMap` which follows the order
// mentioned by `groupOrderMap`. NOTE: This is partial ordering.
func orderedGroups(groupsMap map[string]*rules.Group, groupOrderMap map[string]int) []*rules.Group {
	groups := make([]*rules.Group, 0, len(groupsMap))
	for _, g := range groupsMap {
		groups = append(groups, g)
	}
	sort.Slice(groups, func(i, j int) bool {
		return groupOrderMap[groups[i].Name()] < groupOrderMap[groups[j].Name()]
	})
	return groups
}

Review comment (bug):
The one issue here is that the engine is started with max samples == 10000, so I get an error when running with a lot of data:
```
$ ./promtool test rules --debug cmd/promtool/testdata/unittest.yml 
DEBUG: Starting test unnamed#0
DEBUG: Dump of all data (input_series and rules) at 5m1s:
{__name__="ALERTS", alertname="AlwaysFiring", alertstate="firing"} =>
1 @[60000]
1 @[120000]
1 @[180000]
1 @[240000]
1 @[300000]
{__name__="ALERTS_FOR_STATE", alertname="AlwaysFiring"} =>
0 @[60000]
0 @[120000]
0 @[180000]
0 @[240000]
0 @[300000]
{__name__="fixed_data"} =>
1 @[60000]
1 @[120000]
1 @[180000]
1 @[240000]
1 @[300000]
{__name__="test_full"} =>
0 @[60000]
{__name__="test_histogram_increase"} =>
{count:3, sum:4.3, (0.5,1]:3} @[60000]
{count:4, sum:5.6, (0.5,1]:4} @[120000]
{__name__="test_histogram_repeat"} =>
{count:2, sum:3, (0.5,1]:2} @[60000]
{count:2, sum:3, (0.5,1]:2} @[120000]
{__name__="test_increase"} =>
2 @[60000]
3 @[120000]
{__name__="test_repeat"} =>
1 @[60000]
1 @[120000]
DEBUG: Test unnamed#0 finished, took 21.667828ms
DEBUG: Starting test unnamed#1
DEBUG: Dump of all data (input_series and rules) at 1s:

DEBUG: Test unnamed#1 finished, took 18.396051ms
DEBUG: Starting test unnamed#2
DEBUG: Dump of all data (input_series and rules) at 1h0m1s:
<cut out by krajorama>
DEBUG: Test unnamed#2 finished, took 32.819317ms
DEBUG: Starting test unnamed#3
DEBUG: Failed query exec, expr: "{__name__=~\".+\"}[24h0m1s]", err: query processing would load too many samples into memory in query execution
DEBUG: Test unnamed#3 finished, took 648.141484ms
DEBUG: Starting test unnamed#4
<rest cut out by krajorama>
```

Code after:
			}
			expSamples = append(expSamples, parsedSample{
				Labels:    lb,
				Value:     s.Value,
				Histogram: promqltest.HistogramTestExpression(hist),
			})
		}

		sort.Slice(expSamples, func(i, j int) bool {
			return labels.Compare(expSamples[i].Labels, expSamples[j].Labels) <= 0
		})
		sort.Slice(gotSamples, func(i, j int) bool {
			return labels.Compare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
		})
		if !cmp.Equal(expSamples, gotSamples, cmp.Comparer(labels.Equal)) {
			errs = append(errs, fmt.Errorf("    expr: %q, time: %s,\n        exp: %v\n        got: %v", testCase.Expr,
				testCase.EvalTime.String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
		}
	}

	if debug {
		ts := tg.maxEvalTime()
		// Potentially a test can be specified at a time with fractional seconds,
		// which PromQL cannot represent, so round up to the next whole second.
		ts = (ts + time.Second).Truncate(time.Second)
		expr := fmt.Sprintf(`{__name__=~".+"}[%v]`, ts)
		q, err := suite.QueryEngine().NewInstantQuery(context.Background(), suite.Queryable(), nil, expr, mint.Add(ts))
		if err != nil {
			fmt.Printf("DEBUG: Failed querying, expr: %q, err: %v\n", expr, err)
			return errs
		}
		res := q.Exec(suite.Context())
		if res.Err != nil {
			fmt.Printf("DEBUG: Failed query exec, expr: %q, err: %v\n", expr, res.Err)
			return errs
		}
		switch v := res.Value.(type) {
		case promql.Matrix:
			fmt.Printf("DEBUG: Dump of all data (input_series and rules) at %v:\n", ts)
			fmt.Println(v.String())
		default:
			fmt.Printf("DEBUG: Got unexpected type %T\n", v)
			return errs
		}
	}

	if len(errs) > 0 {
		return errs
	}
	return nil
}