Change:
What happened: kube-controller-manager with enabled option --controllers=,tokencleaner is not removing newly created tokens. It works only after kube-controller-manager restart or after editing secret with token. What you expected to happen: It should remove all expired tokens from kube-system namespace, without restarting or touching secrets. How to reproduce it (as minimally and precisely as possible): enable tokencleaner in kube-controller-manager start controller manager create bootstrap token, you can use script wait 5 minutes token is still there restart kube-controller-manager OR edit newly created token token should be gone after a while Anything else we need to know?: It could be related with Environment: Kubernetes version (use ): Server Version: version.Info{Major:"1", Minor:"13", GitVersion:"v1.13.0", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-12-03T20:56:12Z", GoVersion:"go1.11.2", Compiler:"gc", Platform:"linux/amd64"}
Do you see the following log ? thanks
no errors at all
/sig auth

Discussion:
"//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", <ins> "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", </ins> "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",

"fmt" "time" <del> "k8s.io/api/core/v1" </del> <ins> v1 "k8s.io/api/core/v1" </ins> apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime"

func (tc *TokenCleaner) evalSecret(o interface{}) { secret := o.(*v1.Secret) <del> if bootstrapsecretutil.HasExpired(secret, time.Now()) { </del> <ins> ttl, alreadyExpired := bootstrapsecretutil.GetExpiration(secret, time.Now()) if alreadyExpired { </ins> klog.V(3).Infof("Deleting expired secret %s/%s", secret.Namespace, secret.Name) var options *metav1.DeleteOptions if len(secret.UID) > 0 {

if err != nil && !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) { klog.V(3).Infof("Error deleting Secret: %v", err) } <ins> } else if ttl > 0 { tc.queue.AddAfter(o, ttl) </ins> } }

"github.com/davecgh/go-spew/spew" "k8s.io/apimachinery/pkg/runtime/schema" <ins> "k8s.io/apimachinery/pkg/util/wait" </ins> "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes/fake"

verifyActions(t, expected, cl.Actions()) } <ins> func TestCleanerExpiredAt(t *testing.T) { cleaner, cl, secrets, err := newTokenCleaner() if err != nil { t.Fatalf("error creating TokenCleaner: %v", err) } secret := newTokenSecret("tokenID", "tokenSecret") addSecretExpiration(secret, timeString(2*time.Second)) expected := []core.Action{} verifyFunc := func() { secrets.Informer().GetIndexer().Add(secret) cleaner.evalSecret(secret) verifyActions(t, expected, cl.Actions()) } // token has not expired currently verifyFunc() if cleaner.queue.Len() != 0 { t.Errorf("not using the queue, the length should be 0, now: %v", cleaner.queue.Len()) } var conditionFunc = func() (bool, error) { if cleaner.queue.Len() == 1 { return true, nil } return false, nil } err = wait.Poll(100*time.Millisecond, wait.ForeverTestTimeout, conditionFunc) if err != nil { t.Fatalf("secret is put back into the queue, the queue length should be 1, error: %vn", err) } // secret was eventually deleted expected = []core.Action{ core.NewDeleteAction( schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, api.NamespaceSystem, secret.ObjectMeta.Name), } verifyFunc() } </ins>

"strings" "time" <del> "k8s.io/api/core/v1" </del> <ins> v1 "k8s.io/api/core/v1" </ins> "k8s.io/apimachinery/pkg/util/sets" "k8s.io/cluster-bootstrap/token/api" legacyutil "k8s.io/cluster-bootstrap/token/util"

// HasExpired will identify whether the secret expires func HasExpired(secret *v1.Secret, currentTime time.Time) bool { <ins> _, expired := GetExpiration(secret, currentTime) return expired } // GetExpiration checks if the secret expires // isExpired indicates if the secret is already expired. // timeRemaining indicates how long until it does expire. // if the secret has no expiration timestamp, returns 0, false. // if there is an error parsing the secret's expiration timestamp, returns 0, true. func GetExpiration(secret *v1.Secret, currentTime time.Time) (timeRemaining time.Duration, isExpired bool) { </ins> expiration := GetData(secret, api.BootstrapTokenExpirationKey) <del> if len(expiration) > 0 { expTime, err2 := time.Parse(time.RFC3339, expiration) if err2 != nil { klog.V(3).Infof("Unparseable expiration time (%s) in %s/%s Secret: %v. Treating as expired.", expiration, secret.Namespace, secret.Name, err2) return true } if currentTime.After(expTime) { klog.V(3).Infof("Expired bootstrap token in %s/%s Secret: %v", secret.Namespace, secret.Name, expiration) return true } </del> <ins> if len(expiration) == 0 { return 0, false } expTime, err := time.Parse(time.RFC3339, expiration) if err != nil { klog.V(3).Infof("Unparseable expiration time (%s) in %s/%s Secret: %v. Treating as expired.", expiration, secret.Namespace, secret.Name, err) return 0, true } timeRemaining = expTime.Sub(currentTime) if timeRemaining <= 0 { klog.V(3).Infof("Expired bootstrap token in %s/%s Secret: %v", secret.Namespace, secret.Name, expiration) return 0, true </ins> } <del> return false </del> <ins> return timeRemaining, false </ins> } // ParseName parses the name of the secret to extract the secret ID.