Repository: argoproj/argo (go)

Issue:
workflow template ref does not work in other namespace
## Summary 

Reference workflow template from namespace other than `argo` dose not work. 

Submit workflow which reference template in `argo` namespace works .

## Diagnostics

What version of Argo Workflows are you running?

```
[root@sre4-inf examples]# argo version
argo: v2.9.5
  BuildDate: 2020-08-06T22:11:59Z
  GitCommit: 5759a0e198d333fa8c3e0aeee433d93808c0dc72
  GitTreeState: clean
  GitTag: v2.9.5
  GoVersion: go1.13
  Compiler: gc
  Platform: linux/amd64
[root@sre4-inf examples]#
```

### Reproduce 

- Install argo from https://github.com/argoproj/argo/blob/master/manifests/install.yaml
- Create a new namespace `kubectl create ns my-namespace`
- Create workflow template in `my-namespace` from this https://github.com/argoproj/argo/blob/master/examples/workflow-template/templates.yaml
- Create workflow  in `my-namespace` from this https://github.com/argoproj/argo/blob/master/examples/workflow-template/workflow-template-ref-with-entrypoint-arg-passing.yaml

```
Paste the logs from the workflow controller:
kubectl logs -n argo $(kubectl get pods -l app=workflow-controller -n argo -o name) | grep ${workflow}
```

```
[root@sre4-inf workflow-template]# argo submit  workflow-template-ref-with-entrypoint-arg-passing.yaml -n  my-namespace
Name:                workflow-template-hello-world-q25zd
Namespace:            my-namespace
ServiceAccount:      default
Status:              Pending
Created:             Wed Aug 12 19:04:12 -0700 (1 minute from now)
Parameters:
  message:           hello world
[root@sre4-inf workflow-template]#

[root@sre4-inf workflow-template]# argo get workflow-template-hello-world-q25zd
Name:                workflow-template-hello-world-q25zd
Namespace:            my-namespace
ServiceAccount:      default
Status:              Error
Message:             workflowtemplates.argoproj.io "workflow-template-whalesay-template" not found
Conditions:
 Completed           True
Created:             Wed Aug 12 19:04:12 -0700 (2 minutes ago)
Started:             Wed Aug 12 19:04:12 -0700 (2 minutes ago)
Finished:            Wed Aug 12 19:04:12 -0700 (2 minutes ago)
Duration:            0 seconds
Parameters:
  message:           hello world

[root@sre4-inf workflow-template]# argo template list
NAME
workflow-template-inner-dag
workflow-template-inner-steps
workflow-template-random-fail-template
workflow-template-submittable
workflow-template-whalesay-template
[root@sre4-inf workflow-template]#
```
---
<!-- Issue Author: Don't delete this message to encourage other users to support your issue! -->
**Message from the maintainers**:

Impacted by this bug? Give it a 👍. We prioritise the issues with the most 👍.


Pull request description:
fix: workflow template ref does not work in other namespace
Checklist:

* [ ] Either (a) I've created an [enhancement proposal](https://github.com/argoproj/argo/issues/new/choose) and discussed it with the community, (b) this is a bug fix, or (c) this is a chore.
* [ ] The title of the PR is (a) [conventional](https://www.conventionalcommits.org/en/v1.0.0/), (b) states what changed, and (c) suffixes the related issues number. E.g. `"fix(controller): Updates such and such. Fixes #1234"`.  
* [ ] I've signed the CLA.
* [ ] I have written unit and/or e2e tests for my change. PRs without these are unlikely to be merged.
* [ ] My builds are green. Try syncing with master if they are not. 
* [ ] My organization is added to [USERS.md](https://github.com/argoproj/argo/blob/master/USERS.md).
Fixed #3759

Patch:
diff --git a/workflow/controller/controller.go b/workflow/controller/controller.go
index 34a00ebec..88fd51c01 100644
--- a/workflow/controller/controller.go
+++ b/workflow/controller/controller.go
@@ -160,7 +160,7 @@ func (wfc *WorkflowController) Run(ctx context.Context, wfWorkers, podWorkers in
 	log.Infof("Workers: workflow: %d, pod: %d", wfWorkers, podWorkers)
 
 	wfc.wfInformer = util.NewWorkflowInformer(wfc.restConfig, wfc.GetManagedNamespace(), workflowResyncPeriod, wfc.tweakListOptions)
-	wfc.wftmplInformer = informer.NewTolerantWorkflowTemplateInformer(wfc.dynamicInterface, workflowTemplateResyncPeriod, wfc.namespace)
+	wfc.wftmplInformer = informer.NewTolerantWorkflowTemplateInformer(wfc.dynamicInterface, workflowTemplateResyncPeriod, wfc.managedNamespace)
 
 	wfc.addWorkflowInformerHandlers()
 	wfc.podInformer = wfc.newPodInformer()
diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go
index 519eed3af..99334db30 100644
--- a/workflow/controller/operator.go
+++ b/workflow/controller/operator.go
@@ -2154,7 +2154,7 @@ func (woc *wfOperationCtx) executeScript(nodeName string, templateScope string,
 func (woc *wfOperationCtx) checkForbiddenErrorAndResubmitAllowed(err error, nodeName string, tmpl *wfv1.Template) (*wfv1.NodeStatus, error) {
 	if (apierr.IsForbidden(err) || apierr.IsTooManyRequests(err)) && isResubmitAllowed(tmpl) {
 		// Our error was most likely caused by a lack of resources. If pod resubmission is allowed, keep the node pending
-		woc.requeue(0)
+		woc.requeue(10 * time.Second)
 		return woc.markNodePending(nodeName, err), nil
 	}
 	return nil, err
diff --git a/workflow/creator/creator.go b/workflow/creator/creator.go
index 4bc165ddb..30d46c538 100644
--- a/workflow/creator/creator.go
+++ b/workflow/creator/creator.go
@@ -14,6 +14,10 @@ import (
 func Label(ctx context.Context, obj metav1.Object) {
 	claims := auth.GetClaimSet(ctx)
 	if claims != nil {
-		labels.Label(obj, common.LabelKeyCreator, regexp.MustCompile("[^-_.a-z0-9A-Z]").ReplaceAllString(claims.Sub, "-"))
+		value := regexp.MustCompile("[^-_.a-z0-9A-Z]").ReplaceAllString(claims.Sub, "-")
+		if len(value) > 63 {
+			value = value[len(value)-63:]
+		}
+		labels.Label(obj, common.LabelKeyCreator, value)
 	}
 }


Test patch:
diff --git a/workflow/creator/creator_test.go b/workflow/creator/creator_test.go
index 30d03785f..464b9e6b1 100644
--- a/workflow/creator/creator_test.go
+++ b/workflow/creator/creator_test.go
@@ -2,6 +2,7 @@ package creator
 
 import (
 	"context"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -25,4 +26,11 @@ func TestLabel(t *testing.T) {
 			assert.Contains(t, wf.Labels, common.LabelKeyCreator)
 		}
 	})
+	t.Run("TooLong", func(t *testing.T) {
+		wf := &wfv1.Workflow{}
+		Label(context.WithValue(context.TODO(), auth.ClaimSetKey, &jws.ClaimSet{Sub: strings.Repeat("x", 63) + "y"}), wf)
+		if assert.NotEmpty(t, wf.Labels) {
+			assert.Equal(t, strings.Repeat("x", 62)+"y", wf.Labels[common.LabelKeyCreator])
+		}
+	})
 }