Repository: openshift/sippy
Pull request #2551: TRT-2056: Track more regression data and show last failure in triaged table
Author: login: dgoodwin
Labels: name: approved
description: Indicates a PR has been approved by an approver from all required OWNERS files.
name: lgtm
description: Indicates that a PR is ready to be merged.
name: jira/valid-reference
description: Indicates that this PR references a valid Jira ticket of any type.
Created: 2025-05-02T16:21:19Z | Merged: 2025-05-09T20:36:45Z

For the duration a regression is open, track the max number of failures we saw in the sample window. This will be helpful for determining what minFail we should use, and provide insights related to regressions that go away on their own.

Doesn't work great with triage right now for the same reason we see 100% passing on a triaged test details page, we decrement traiged runs today and don't return any info on them. This will be better in the new traige where job runs only get decremented when the regression is considered closed.

Also removed temp bigquery migration code for the postgres rollout that
I forgot to come back for.



Also tracks the last failure time for the regression. 

And finally displays that in the triaged portion of the regressed tests modal. This will allow us to better see if a triaged regression is on-going, without having to go to the other tabs and find a specific row.



Diff:
diff --git a/pkg/api/componentreadiness/regressiontracker.go b/pkg/api/componentreadiness/regressiontracker.go
index 3ee2484664..7d8a7ac75a 100644
--- a/pkg/api/componentreadiness/regressiontracker.go
+++ b/pkg/api/componentreadiness/regressiontracker.go
@@ -4,22 +4,19 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"reflect"
 	"strings"
 	"time"
 
 	"github.com/openshift/sippy/pkg/api/componentreadiness/middleware/regressiontracker"
-	"github.com/openshift/sippy/pkg/db/models"
-	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
-	"google.golang.org/api/iterator"
-
 	crtype "github.com/openshift/sippy/pkg/apis/api/componentreport"
 	"github.com/openshift/sippy/pkg/apis/cache"
 	configv1 "github.com/openshift/sippy/pkg/apis/config/v1"
 	v1 "github.com/openshift/sippy/pkg/apis/sippy/v1"
 	sippybigquery "github.com/openshift/sippy/pkg/bigquery"
 	"github.com/openshift/sippy/pkg/db"
+	"github.com/openshift/sippy/pkg/db/models"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -39,46 +36,7 @@ type RegressionStore interface {
 	// a regressed test, so people using custom reporting can see what is regressed in main as well.
 	ListCurrentRegressionsForRelease(release string) ([]*models.TestRegression, error)
 	OpenRegression(view crtype.View, newRegressedTest crtype.ReportTestSummary) (*models.TestRegression, error)
-	ReOpenRegression(reg *models.TestRegression) error
-	CloseRegression(reg *models.TestRegression, closedAt time.Time) error
-}
-
-// TODO: temporary, just being used for migration code on initial rollout, can be deleted as soon as postgres imports
-// the current regressions.
-//
-// ListCurrentRegressionsForRelease returns *all* regressions for all releases. We operate on the assumption that
-// only one view is allowed to have regression tracking enabled (i.e. 4.18-main) per release, which is validated
-// when the views file is loaded. This is because we want to display regression tracking data on any report that shows
-// a regressed test, so users using custom reporting can see what is regressed in main as well.
-func ListCurrentRegressions(ctx context.Context, client *sippybigquery.Client) ([]*crtype.TestRegressionBigQuery, error) {
-	// Use max snapshot date to get the most recently appended view of the regressions.
-	queryString := fmt.Sprintf("SELECT * FROM %s.test_regressions_append WHERE snapshot = (SELECT MAX(snapshot) FROM %s.test_regressions_append)",
-		client.Dataset, client.Dataset)
-
-	sampleQuery := client.BQ.Query(queryString)
-
-	regressions := make([]*crtype.TestRegressionBigQuery, 0)
-	log.Infof("Fetching current test regressions with: %s", sampleQuery.Q)
-
-	it, err := sampleQuery.Read(ctx)
-	if err != nil {
-		log.WithError(err).Error("error querying triaged incidents from bigquery")
-		return regressions, err
-	}
-
-	for {
-		var regression crtype.TestRegressionBigQuery
-		err := it.Next(&regression)
-		if err == iterator.Done {
-			break
-		}
-		if err != nil {
-			log.WithError(err).Error("error parsing triaged incident from bigquery")
-			return nil, errors.Wrap(err, "error parsing triaged incident from bigquery")
-		}
-		regressions = append(regressions, &regression)
-	}
-	return regressions, nil
+	UpdateRegression(reg *models.TestRegression) error
 }
 
 type PostgresRegressionStore struct {
@@ -98,8 +56,8 @@ func (prs *PostgresRegressionStore) ListCurrentRegressionsForRelease(release str
 		Where("closed IS NULL OR closed > ?", time.Now().Add(-regressionHysteresisDays*24*time.Hour))
 	res := q.Scan(&regressions)
 	return regressions, res.Error
-
 }
+
 func (prs *PostgresRegressionStore) OpenRegression(view crtype.View, newRegressedTest crtype.ReportTestSummary) (*models.TestRegression, error) {
 
 	variants := []string{}
@@ -109,12 +67,14 @@ func (prs *PostgresRegressionStore) OpenRegression(view crtype.View, newRegresse
 	log.Infof("variants: %s", strings.Join(variants, ","))
 
 	newRegression := &models.TestRegression{
-		View:     view.Name,
-		Release:  view.SampleRelease.Release,
-		TestID:   newRegressedTest.TestID,
-		TestName: newRegressedTest.TestName,
-		Opened:   time.Now(),
-		Variants: variants,
+		View:        view.Name,
+		Release:     view.SampleRelease.Release,
+		TestID:      newRegressedTest.TestID,
+		TestName:    newRegressedTest.TestName,
+		Opened:      time.Now(),
+		Variants:    variants,
+		MaxFailures: newRegressedTest.SampleStats.FailureCount,
+		LastFailure: sql.NullTime{Valid: true, Time: *newRegressedTest.LastFailure},
 	}
 	res := prs.dbc.DB.Create(newRegression)
 	if res.Error != nil {
@@ -125,14 +85,7 @@ func (prs *PostgresRegressionStore) OpenRegression(view crtype.View, newRegresse
 
 }
 
-func (prs *PostgresRegressionStore) ReOpenRegression(reg *models.TestRegression) error {
-	reg.Closed = sql.NullTime{Valid: false}
-	res := prs.dbc.DB.Save(&reg)
-	return res.Error
-}
-
-func (prs *PostgresRegressionStore) CloseRegression(reg *models.TestRegression, closedAt time.Time) error {
-	reg.Closed = sql.NullTime{Valid: true, Time: closedAt}
+func (prs *PostgresRegressionStore) UpdateRegression(reg *models.TestRegression) error {
 	res := prs.dbc.DB.Save(&reg)
 	return res.Error
 }
@@ -177,48 +130,6 @@ type RegressionTracker struct {
 // component report to the regression tables.
 func (rt *RegressionTracker) Run(ctx context.Context) error {
 
-	// TODO: remove this temporary migration code for initial rollout
-	var totalRegressions int64
-	rt.dbc.DB.Model(&models.TestRegression{}).Count(&totalRegressions)
-	if totalRegressions == 0 {
-		rt.logger.Warn("no test regressions found in postgres, migrating bigquery regressions over")
-		allBigQueryRegressions, err := ListCurrentRegressions(ctx, rt.bigqueryClient)
-		if err != nil {
-			rt.logger.WithError(err).Error("error listing bigquery regressions")
-			return err
-		}
-		rt.logger.Infof("loaded %d test regressions from bigquery", len(allBigQueryRegressions))
-		for _, reg := range allBigQueryRegressions {
-			newRegression := &models.TestRegression{
-				View:     reg.View,
-				Release:  reg.Release,
-				TestID:   reg.TestID,
-				TestName: reg.TestName,
-				Opened:   reg.Opened,
-			}
-			if !reg.Closed.Valid {
-				newRegression.Closed = sql.NullTime{
-					Valid: false,
-				}
-			} else {
-				newRegression.Closed = sql.NullTime{
-					Valid: true,
-					Time:  reg.Closed.Timestamp,
-				}
-			}
-			variants := []string{}
-			for _, v := range reg.Variants {
-				variants = append(variants, fmt.Sprintf("%s:%s", v.Key, v.Value))
-			}
-			newRegression.Variants = variants
-
-			res := rt.dbc.DB.Create(newRegression)
-			if res.Error != nil {
-				return res.Error
-			}
-		}
-	}
-
 	// Run the existing logic
 	var err error
 	for _, view := range rt.views {
@@ -297,11 +208,33 @@ func (rt *RegressionTracker) SyncRegressionsForReport(ctx context.Context, view
 		}
 	}
 
-	var newRegs, reopenedRegs, untouchedRegs, closedRegs int
+	var openedRegs, reopenedRegs, ongoingRegs, closedRegs, statsUpdatedRegs int
 	matchedOpenRegressions := []*models.TestRegression{} // all the matches we found, used to determine what had no match
 	rLog.Infof("syncing %d open regressions", len(allRegressedTests))
 	for _, regTest := range allRegressedTests {
 		if openReg := regressiontracker.FindOpenRegression(view.Name, regTest.TestID, regTest.Variants, regressions); openReg != nil {
+
+			// Update any tracking params on the regression if we see better values:
+			var modifiedRegression bool
+			if regTest.SampleStats.FailureCount > openReg.MaxFailures {
+				openReg.MaxFailures = regTest.SampleStats.FailureCount
+				modifiedRegression = true
+			}
+			if regTest.LastFailure != nil {
+				if !openReg.LastFailure.Valid || regTest.LastFailure.After(openReg.LastFailure.Time) {
+					openReg.LastFailure = sql.NullTime{Valid: true, Time: *regTest.LastFailure}
+					modifiedRegression = true
+				}
+			}
+			if modifiedRegression {
+				statsUpdatedRegs++
+				err := rt.backend.UpdateRegression(openReg)
+				if err != nil {
+					rLog.WithError(err).Errorf("error updating regression: %v", openReg)
+					return errors.Wrapf(err, "error updating regression: %v", openReg)
+				}
+			}
+
 			if openReg.Closed.Valid {
 				// if the regression returned has a closedRegs date, we found a recently closedRegs
 				// regression for this test. We'll re-use it to limit churn as sometimes tests may drop
@@ -309,22 +242,23 @@ func (rt *RegressionTracker) SyncRegressionsForReport(ctx context.Context, view
 				rLog.Infof("re-opening existing regression: %v", openReg)
 				if !rt.dryRun {
 					reopenedRegs++
-					err := rt.backend.ReOpenRegression(openReg)
+					openReg.Closed = sql.NullTime{Valid: false}
+					err := rt.backend.UpdateRegression(openReg)
 					if err != nil {
 						rLog.WithError(err).Errorf("error re-opening regression: %v", openReg)
 						return errors.Wrapf(err, "error re-opening regression: %v", openReg)
 					}
 				}
 			} else {
-				untouchedRegs++
+				// Still consider untouched even if we bumped the max failures count
+				ongoingRegs++
 				rLog.WithFields(log.Fields{
 					"test": regTest.TestName,
 				}).Debugf("reusing already opened regression: %v", openReg)
-
 			}
 			matchedOpenRegressions = append(matchedOpenRegressions, openReg)
 		} else {
-			newRegs++
+			openedRegs++
 			rLog.Infof("opening new regression: %v", regTest)
 			if !rt.dryRun {
 				// Open a new regression:
@@ -342,8 +276,10 @@ func (rt *RegressionTracker) SyncRegressionsForReport(ctx context.Context, view
 	now := time.Now()
 	for _, regression := range regressions {
 		var matched bool
+		// We don't want to reject a regression as unmatched because it's max failures count is different.
+		// We also do not want to wipe out the max failures value when closing.
 		for _, m := range matchedOpenRegressions {
-			if reflect.DeepEqual(m, regression) {
+			if m.ID == regression.ID {
 				matched = true
 				break
 			}
@@ -353,7 +289,8 @@ func (rt *RegressionTracker) SyncRegressionsForReport(ctx context.Context, view
 			rLog.Infof("found a regression no longer appearing in the report which should be closedRegs: %v", regression)
 			closedRegs++
 			if !rt.dryRun {
-				err := rt.backend.CloseRegression(regression, now)
+				regression.Closed = sql.NullTime{Valid: true, Time: now}
+				err := rt.backend.UpdateRegression(regression)
 				if err != nil {
 					rLog.WithError(err).Errorf("error closing regression: %v", regression)
 					return errors.Wrap(err, "error closing regression")
@@ -362,7 +299,8 @@ func (rt *RegressionTracker) SyncRegressionsForReport(ctx context.Context, view
 		}
 
 	}
-	rLog.Infof("regression tracking sync completed, opened=%d, reopenedRegs=%d, closedRegs=%d untouchedRegs=%d", newRegs, reopenedRegs, closedRegs, untouchedRegs)
+	rLog.Infof("regression tracking sync completed, opened=%d, reopened=%d, closed=%d ongoing=%d statsUpdated=%d",
+		openedRegs, reopenedRegs, closedRegs, ongoingRegs, statsUpdatedRegs)
 
 	return nil
 }
diff --git a/pkg/db/models/triage.go b/pkg/db/models/triage.go
index 3120cb191b..f3a33ee484 100644
--- a/pkg/db/models/triage.go
+++ b/pkg/db/models/triage.go
@@ -84,4 +84,10 @@ type TestRegression struct {
 	Opened   time.Time      `json:"opened" gorm:"not null"`
 	Closed   sql.NullTime   `json:"closed"`
 	Triages  []Triage       `json:"-" gorm:"many2many:triage_regressions;"`
+	// LastFailure is the last failure in the sample we saw while this regression was open.
+	LastFailure sql.NullTime `json:"last_failure"`
+	// MaxFailures is the maximum number of failures we found in the reporting window while this regression was open.
+	// This is intended to help inform the min fail thresholds we should be using, and what kind of regressions
+	// disappear on their own.
+	MaxFailures int `json:"max_failures"`
 }
diff --git a/sippy-ng/src/component_readiness/TriagedRegressionTestList.js b/sippy-ng/src/component_readiness/TriagedRegressionTestList.js
index 699ea3f14e..1885482ee8 100644
--- a/sippy-ng/src/component_readiness/TriagedRegressionTestList.js
+++ b/sippy-ng/src/component_readiness/TriagedRegressionTestList.js
@@ -72,6 +72,26 @@ export default function TriagedRegressionTestList(props) {
         </Tooltip>
       ),
     },
+    {
+      field: 'last_failure',
+      headerName: 'Last Failure',
+      flex: 12,
+      valueGetter: (params) => {
+        if (!params.row.last_failure.Valid) {
+          return null
+        }
+        return new Date(params.row.last_failure.Time).getTime()
+      },
+      renderCell: (params) => {
+        if (!params.value) return ''
+        const lastFailureDate = new Date(params.value)
+        return (
+          <div className="last-failure">
+            {relativeTime(lastFailureDate, new Date())}
+          </div>
+        )
+      },
+    },
   ]
 
   return (
diff --git a/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go b/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go
index 9879783817..0fa1870445 100644
--- a/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go
+++ b/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go
@@ -78,14 +78,16 @@ func Test_RegressionTracker(t *testing.T) {
 
 		// Now close it:
 		assert.False(t, lookup.Closed.Valid)
-		err = tracker.CloseRegression(lookup, time.Now())
+		lookup.Closed = sql.NullTime{Valid: true, Time: time.Now()}
+		err = tracker.UpdateRegression(lookup)
 		assert.NoError(t, err)
 		// look it up again because we're being ridiculous:
 		dbc.DB.First(&lookup)
 
 		assert.True(t, lookup.Closed.Valid)
 
-		err = tracker.ReOpenRegression(lookup)
+		lookup.Closed = sql.NullTime{Valid: false}
+		err = tracker.UpdateRegression(lookup)
 		assert.NoError(t, err)
 		dbc.DB.First(&lookup)
 		assert.False(t, lookup.Closed.Valid)


Reviews:
user: login: smg247
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-09T15:16:35Z
user: login: dgoodwin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-09T17:21:40Z
user: login: dgoodwin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-09T17:22:09Z

Review comments:
diff_hunk: @@ -125,16 +85,13 @@ func (prs *PostgresRegressionStore) OpenRegression(view crtype.View, newRegresse
 
 }
 
-func (prs *PostgresRegressionStore) ReOpenRegression(reg *models.TestRegression) error {
-	reg.Closed = sql.NullTime{Valid: false}
+func (prs *PostgresRegressionStore) UpdateRegression(reg *models.TestRegression) error {
 	res := prs.dbc.DB.Save(&reg)
 	return res.Error
 }
 
 func (prs *PostgresRegressionStore) CloseRegression(reg *models.TestRegression, closedAt time.Time) error {
-	reg.Closed = sql.NullTime{Valid: true, Time: closedAt}
-	res := prs.dbc.DB.Save(&reg)
-	return res.Error
+	return prs.UpdateRegression(reg)
path: pkg/api/componentreadiness/regressiontracker.go
user: login: smg247
body: this doesn't use the `closedAt` param anymore?
created_at: 2025-05-09T15:10:55Z
author_association: MEMBER
diff_hunk: @@ -353,7 +293,8 @@ func (rt *RegressionTracker) SyncRegressionsForReport(ctx context.Context, view
 			rLog.Infof("found a regression no longer appearing in the report which should be closedRegs: %v", regression)
 			closedRegs++
 			if !rt.dryRun {
-				err := rt.backend.CloseRegression(regression, now)
+				regression.Closed = sql.NullTime{Valid: true, Time: now}
+				err := rt.backend.UpdateRegression(regression)
path: pkg/api/componentreadiness/regressiontracker.go
user: login: smg247
body: I would expect the `CloseRegression` method to be used here (after fixing the implementation). Otherwise, that could be removed.
created_at: 2025-05-09T15:15:16Z
author_association: MEMBER
diff_hunk: @@ -353,7 +293,8 @@ func (rt *RegressionTracker) SyncRegressionsForReport(ctx context.Context, view
 			rLog.Infof("found a regression no longer appearing in the report which should be closedRegs: %v", regression)
 			closedRegs++
 			if !rt.dryRun {
-				err := rt.backend.CloseRegression(regression, now)
+				regression.Closed = sql.NullTime{Valid: true, Time: now}
+				err := rt.backend.UpdateRegression(regression)
path: pkg/api/componentreadiness/regressiontracker.go
user: login: dgoodwin
body: well spotted, removed is what I planned but I forgot the impl in there. Thought was to just provide a generic update function.
created_at: 2025-05-09T17:21:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -353,7 +293,8 @@ func (rt *RegressionTracker) SyncRegressionsForReport(ctx context.Context, view
 			rLog.Infof("found a regression no longer appearing in the report which should be closedRegs: %v", regression)
 			closedRegs++
 			if !rt.dryRun {
-				err := rt.backend.CloseRegression(regression, now)
+				regression.Closed = sql.NullTime{Valid: true, Time: now}
+				err := rt.backend.UpdateRegression(regression)
path: pkg/api/componentreadiness/regressiontracker.go
user: login: dgoodwin
body: Updated now.
created_at: 2025-05-09T17:22:09Z
author_association: CONTRIBUTOR

PR comments:
user: login: openshift-ci-robot
created_at: 2025-05-07T19:19:18Z
body: @dgoodwin: This pull request references [TRT-2056](https://issues.redhat.com//browse/TRT-2056) which is a valid jira issue.

Warning: The referenced jira issue has an invalid target version for the target branch this PR targets: expected the bug to target the "4.20.0" version, but no target version was set.

<details>

In response to [this](https://github.com/openshift/sippy/pull/2551):

>For the duration a regression is open, track the max number of failures we saw in the sample window. This will be helpful for determining what minFail we should use, and provide insights related to regressions that go away on their own.
>
>Doesn't work great with triage right now for the same reason we see 100% passing on a triaged test details page, we decrement traiged runs today and don't return any info on them. This will be better in the new traige where job runs only get decremented when the regression is considered closed.
>
>Also removed temp bigquery migration code for the postgres rollout that
>I forgot to come back for.
>
>
>
>Also tracks the last failure time for the regression. 
>
>And finally displays that in the triaged portion of the regressed tests modal. This will allow us to better see if a triaged regression is on-going, without having to go to the other tabs and find a specific row.
>


Instructions for interacting with me using PR comments are available [here](https://prow.ci.openshift.org/command-help?repo=openshift%2Fsippy).  If you have questions or suggestions related to my behavior, please file an issue against the [openshift-eng/jira-lifecycle-plugin](https://github.com/openshift-eng/jira-lifecycle-plugin/issues/new) repository.
</details>
author_association: NONE
user: login: dgoodwin
created_at: 2025-05-08T21:51:10Z
body: /test e2e

author_association: CONTRIBUTOR
user: login: smg247
created_at: 2025-05-09T18:30:21Z
body: /lgtm
author_association: MEMBER
user: login: openshift-ci[bot]
created_at: 2025-05-09T18:31:02Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/openshift/sippy/pull/2551#" title="Author self-approved">dgoodwin</a>*, *<a href="https://github.com/openshift/sippy/pull/2551#issuecomment-2867569547" title="LGTM">smg247</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=openshift%2Fsippy).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/openshift/sippy/blob/main/OWNERS)~~ [dgoodwin,smg247]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR
user: login: openshift-ci[bot]
created_at: 2025-05-09T20:31:47Z
body: @dgoodwin: all tests **passed!**


[Full PR test history](https://prow.ci.openshift.org/pr-history?org=openshift&repo=sippy&pr=2551). [Your PR dashboard](https://prow.ci.openshift.org/pr?query=is:pr+state:open+author:dgoodwin).

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->
author_association: CONTRIBUTOR