Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: when deployment controller cleans up old ReplicaSet, it sorts all old ReplicaSets by createTimeStamp and delete the oldest one, which may cause Deployment roll back to a wrong version. How to reproduce it (as minimally and precisely as possible): Suppose we set the revisionHistoryLimit of a Deployment to 3, and currently we have 3 Replicas: r1, r2, r3 (the createTimeStamp and Revision both are r3 r2 r1); if we upgrade and roll back 3 times continuously, there will be an expected case: upgrade: &nbsp; ReplicaSet will be: r4, r3, r2 &nbsp; createTimeStamp: r4 r3 r2 &nbsp; reversion: r4 r3 r2 rollback: should rollback to v3 &nbsp; ReplicaSet will be: r4, r3, r2 &nbsp; createTimeStamp: r4 r3 r2 &nbsp; reversion: r3 r4 r2 upgrade: &nbsp; ReplicaSet will be: r5, r4, r3 &nbsp; createTimeStamp: r5 r4 r3 &nbsp; reversion: r5 r3 r4 rollback: should rollback to v3 &nbsp; ReplicaSet will be: r4, r3, r2 &nbsp; createTimeStamp: r5 r4 r3 &nbsp; reversion: r3 r5 r4 upgrade: &nbsp; ReplicaSet will be: r6, r5, r4 &nbsp; createTimeStamp: r6 r5 r4 &nbsp; reversion: r6 r5 r4 rollback: should rollback to r3, but instead, we rollbacked to r5, and this may cause error. What you expected to happen: when deployment controller cleans up old ReplicaSet, it should sort old ReplicaSets by Revision
/sig api-machinery /sig apps
does this make sense to you? /assign /cc
?
/remove-sig api-machinery
This requirement makes sense. Unassign API machinery - SIG Apps should own this.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Discussion:
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", <ins> "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", </ins> "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",

return nil } <del> sort.Sort(controller.ReplicaSetsByCreationTimestamp(cleanableRSes)) </del> <ins> sort.Sort(deploymentutil.ReplicaSetsByRevision(cleanableRSes)) </ins> klog.V(4).Infof("Looking to cleanup old replica sets for deployment %q", deployment.Name) for i := int32(0); i < diff; i++ {

apps "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" <ins> "k8s.io/apimachinery/pkg/util/sets" </ins> "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" testclient "k8s.io/client-go/testing"

} } } <ins> func TestDeploymentController_cleanupDeploymentOrder(t *testing.T) { selector := map[string]string{"foo": "bar"} now := metav1.Now() duration := time.Minute newRSWithRevisionAndCreationTimestamp := func(name string, replicas int, selector map[string]string, timestamp time.Time, revision string) *apps.ReplicaSet { rs := rs(name, replicas, selector, metav1.NewTime(timestamp)) if revision != "" { rs.Annotations = map[string]string{ deploymentutil.RevisionAnnotation: revision, } } rs.Status = apps.ReplicaSetStatus{ Replicas: int32(replicas), } return rs } // for all test cases, creationTimestamp order keeps as: rs1 < rs2 < rs3 < r4 tests := []struct { oldRSs               []*apps.ReplicaSet revisionHistoryLimit int32 expectedDeletedRSs   sets.String }{ { // revision order: rs1 < rs2, delete rs1 oldRSs: []*apps.ReplicaSet{ newRSWithRevisionAndCreationTimestamp("foo-1", 0, selector, now.Add(-1*duration), "1"), newRSWithRevisionAndCreationTimestamp("foo-2", 0, selector, now.Time, "2"), }, revisionHistoryLimit: 1, expectedDeletedRSs:   sets.NewString("foo-1"), }, { // revision order: rs2 < rs1, delete rs2 oldRSs: []*apps.ReplicaSet{ newRSWithRevisionAndCreationTimestamp("foo-1", 0, selector, now.Add(-1*duration), "2"), newRSWithRevisionAndCreationTimestamp("foo-2", 0, selector, now.Time, "1"), }, revisionHistoryLimit: 1, expectedDeletedRSs:   sets.NewString("foo-2"), }, { // rs1 has revision but rs2 doesn't have revision, delete rs2 oldRSs: []*apps.ReplicaSet{ newRSWithRevisionAndCreationTimestamp("foo-1", 0, selector, now.Add(-1*duration), "1"), newRSWithRevisionAndCreationTimestamp("foo-2", 0, selector, now.Time, ""), }, revisionHistoryLimit: 1, expectedDeletedRSs:   sets.NewString("foo-2"), }, { // rs1 doesn't have revision while rs2 has revision, delete rs1 oldRSs: []*apps.ReplicaSet{ newRSWithRevisionAndCreationTimestamp("foo-1", 0, selector, now.Add(-1*duration), ""), newRSWithRevisionAndCreationTimestamp("foo-2", 0, selector, now.Time, "2"), }, revisionHistoryLimit: 1, expectedDeletedRSs:   sets.NewString("foo-1"), }, { // revision order: rs1 < rs2 < r3, but rs1 has replicas, delete rs2 oldRSs: []*apps.ReplicaSet{ newRSWithRevisionAndCreationTimestamp("foo-1", 1, selector, now.Add(-1*duration), "1"), newRSWithRevisionAndCreationTimestamp("foo-2", 0, selector, now.Time, "2"), newRSWithRevisionAndCreationTimestamp("foo-3", 0, selector, now.Add(duration), "3"), }, revisionHistoryLimit: 1, expectedDeletedRSs:   sets.NewString("foo-2"), }, { // revision order: rs1 < rs2 < r3, both rs1 && rs2 have replicas, don't delete oldRSs: []*apps.ReplicaSet{ newRSWithRevisionAndCreationTimestamp("foo-1", 1, selector, now.Add(-1*duration), "1"), newRSWithRevisionAndCreationTimestamp("foo-2", 1, selector, now.Time, "2"), newRSWithRevisionAndCreationTimestamp("foo-3", 0, selector, now.Add(duration), "3"), }, revisionHistoryLimit: 1, expectedDeletedRSs:   sets.NewString(), }, { // revision order: rs2 < rs4 < rs1 < rs3, delete rs2 && rs4 oldRSs: []*apps.ReplicaSet{ newRSWithRevisionAndCreationTimestamp("foo-1", 0, selector, now.Add(-1*duration), "3"), newRSWithRevisionAndCreationTimestamp("foo-2", 0, selector, now.Time, "1"), newRSWithRevisionAndCreationTimestamp("foo-3", 0, selector, now.Add(duration), "4"), newRSWithRevisionAndCreationTimestamp("foo-4", 0, selector, now.Add(2*duration), "2"), }, revisionHistoryLimit: 2, expectedDeletedRSs:   sets.NewString("foo-2", "foo-4"), }, } for i := range tests { test := tests[i] t.Logf("scenario %d", i) fake := &fake.Clientset{} informers := informers.NewSharedInformerFactory(fake, controller.NoResyncPeriodFunc()) controller, err := NewDeploymentController(informers.Apps().V1().Deployments(), informers.Apps().V1().ReplicaSets(), informers.Core().V1().Pods(), fake) if err != nil { t.Fatalf("error creating Deployment controller: %v", err) } controller.eventRecorder = &record.FakeRecorder{} controller.dListerSynced = alwaysReady controller.rsListerSynced = alwaysReady controller.podListerSynced = alwaysReady for _, rs := range test.oldRSs { informers.Apps().V1().ReplicaSets().Informer().GetIndexer().Add(rs) } stopCh := make(chan struct{}) defer close(stopCh) informers.Start(stopCh) d := newDeployment("foo", 1, &test.revisionHistoryLimit, nil, nil, map[string]string{"foo": "bar"}) controller.cleanupDeployment(test.oldRSs, d) deletedRSs := sets.String{} for _, action := range fake.Actions() { deleteAction, ok := action.(testclient.DeleteActionImpl) if !ok { t.Logf("Found not-delete action with verb %v. Ignoring.", action.GetVerb()) continue } if deleteAction.GetResource().Resource != "replicasets" { continue } deletedRSs.Insert(deleteAction.GetName()) } t.Logf("&test.revisionHistoryLimit: %d, &test.deletedReplicaSets: %v", test.revisionHistoryLimit, deletedRSs) if !test.expectedDeletedRSs.Equal(deletedRSs) { t.Errorf("expect to delete old replica sets %v, but got %v", test.expectedDeletedRSs, deletedRSs) continue } } } </ins>

return deployments, nil } <ins> // ReplicaSetsByRevision sorts a list of ReplicaSet by revision, using their creation timestamp or name as a tie breaker. // By using the creation timestamp, this sorts from old to new replica sets. type ReplicaSetsByRevision []*apps.ReplicaSet func (o ReplicaSetsByRevision) Len() int      { return len(o) } func (o ReplicaSetsByRevision) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o ReplicaSetsByRevision) Less(i, j int) bool { revision1, err1 := Revision(o[i]) revision2, err2 := Revision(o[j]) if err1 != nil || err2 != nil || revision1 == revision2 { return controller.ReplicaSetsByCreationTimestamp(o).Less(i, j) } return revision1 < revision2 } </ins>