Repository: argoproj/argo (go)

Issue:
Increase max reconciliation time
# Summary

We'd want to support 10k+ node workflows, and these tend to fail to schedule all their nodes within in the 10s deadline. In fact, it is possible for a workflow to get so large it can never fully schedule its nodes.

In #2705 (May) we increased the default number of workflow-workflows from 8 to 32. 

Let`s increase the reconciliation time (or make it configurable)

# Use Cases

10k+ node workflows.

---
<!-- Issue Author: Don't delete this message to encourage other users to support your issue! -->
**Message from the maintainers**:

Impacted by this bug? Give it a 👍. We prioritise the issues with the most 👍.

Pull request description:
feat(controller): Make MAX_OPERATION_TIME configurable. Close #4239
Signed-off-by: Alex Collins <alex_collins@intuit.com>

Checklist:

* [x] My organization is added to [USERS.md](https://github.com/argoproj/argo/blob/master/USERS.md).

<!--

Please leave your PR in draft if you don't need a review yet. 

To fix failing `CI / Codegen`, run `make codegen`. 

-->

This change is to test to see if increasing this improve issues like #4560 

Patch:
diff --git a/util/env/env.go b/util/env/env.go
new file mode 100644
index 000000000..76f2f5a36
--- /dev/null
+++ b/util/env/env.go
@@ -0,0 +1,21 @@
+package env
+
+import (
+	"os"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func LookupEnvDurationOr(key string, o time.Duration) time.Duration {
+	v, found := os.LookupEnv(key)
+	if found {
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			log.WithField(key, v).WithError(err).Panic("failed to parse")
+		} else {
+			return d
+		}
+	}
+	return o
+}
diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go
index b6962798a..403753e37 100644
--- a/workflow/controller/operator.go
+++ b/workflow/controller/operator.go
@@ -37,6 +37,7 @@ import (
 	wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
 	"github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
 	"github.com/argoproj/argo/util"
+	envutil "github.com/argoproj/argo/util/env"
 	errorsutil "github.com/argoproj/argo/util/errors"
 	"github.com/argoproj/argo/util/intstr"
 	"github.com/argoproj/argo/util/resource"
@@ -116,8 +117,10 @@ var (
 
 // maxOperationTime is the maximum time a workflow operation is allowed to run
 // for before requeuing the workflow onto the workqueue.
-const maxOperationTime = 10 * time.Second
-const defaultRequeueTime = maxOperationTime
+var (
+	maxOperationTime   = envutil.LookupEnvDurationOr("MAX_OPERATION_TIME", 30*time.Second)
+	defaultRequeueTime = envutil.LookupEnvDurationOr("DEFAULT_REQUEUE_TIME", maxOperationTime/2)
+)
 
 // failedNodeStatus is a subset of NodeStatus that is only used to Marshal certain fields into a JSON of failed nodes
 type failedNodeStatus struct {


Test patch:
diff --git a/util/env/env_test.go b/util/env/env_test.go
new file mode 100644
index 000000000..65e705063
--- /dev/null
+++ b/util/env/env_test.go
@@ -0,0 +1,18 @@
+package env
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLookupEnvDurationOr(t *testing.T) {
+	defer func() { _ = os.Unsetenv("FOO") }()
+	assert.Equal(t, time.Second, LookupEnvDurationOr("", time.Second), "default value")
+	_ = os.Setenv("FOO", "bar")
+	assert.Panics(t, func() { LookupEnvDurationOr("FOO", time.Second) }, "bad value")
+	_ = os.Setenv("FOO", "1h")
+	assert.Equal(t, time.Hour, LookupEnvDurationOr("FOO", time.Second), "env var value")
+}