Repository: fluxcd/flux
Pull request #2481 (2019-09-26 14:56:25)

Make sync operations timeout configurable
Add `--sync-timeout` to fluxd command flags and `sync.timeout` to the Helm chart (defaults to `1m` same as before).

Workaround for: #2477 and #1857

hiddeco: In #1857, where the timeouts were originally reported, we discussed ways of optimization instead of making it a configurable flag. However, shortly after this the manifest factorization feature was introduced, which gives people the power to run anything during sync. As optimizing the unknown is a hard thing to do, I think this change now is the obvious (and easiest) solution.

Patch:
diff --git a/chart/flux/README.md b/chart/flux/README.md
index 63150a39..d3e696c7 100755
--- a/chart/flux/README.md
+++ b/chart/flux/README.md
@@ -243,6 +243,7 @@ The following tables lists the configurable parameters of the Flux chart and the
 | `service.type`                                    | `ClusterIP`                                          | Service type to be used (exposing the Flux API outside of the cluster is not advised)
 | `service.port`                                    | `3030`                                               | Service port to be used
 | `sync.state`                                      | `git`                                                | Where to keep sync state; either a tag in the upstream repo (`git`), or as an annotation on the SSH secret (`secret`)
+| `sync.timeout`                                    | `None`                                               |  Duration after which sync operations time out (defaults to `1m`)
 | `git.url`                                         | `None`                                               | URL of git repo with Kubernetes manifests
 | `git.readonly`                                    | `false`                                              | If `true`, the git repo will be considered read-only, and Flux will not attempt to write to it
 | `git.branch`                                      | `master`                                             | Branch of git repo to use for Kubernetes manifests
diff --git a/chart/flux/templates/deployment.yaml b/chart/flux/templates/deployment.yaml
index df9b25fc..391bc1dd 100644
--- a/chart/flux/templates/deployment.yaml
+++ b/chart/flux/templates/deployment.yaml
@@ -184,6 +184,9 @@ spec:
           - --k8s-secret-name={{ .Values.git.secretName | default (printf "%s-git-deploy" (include "flux.fullname" .)) }}
           - --memcached-hostname={{ .Values.memcached.hostnameOverride | default (printf "%s-memcached" (include "flux.fullname" .)) }}
           - --sync-state={{ .Values.sync.state }}
+          {{- if .Values.sync.timeout }}
+          - --sync-timeout={{ .Values.sync.timeout }}
+          {{- end }}
           {{- if .Values.memcached.createClusterIP }}
           - --memcached-service=
           {{- end }}
diff --git a/chart/flux/values.yaml b/chart/flux/values.yaml
index ddc268f7..4b7a5e04 100644
--- a/chart/flux/values.yaml
+++ b/chart/flux/values.yaml
@@ -147,6 +147,8 @@ gpgKeys:
 sync:
   # use `.sync.state: secret` to store flux's state as an annotation on the secret (instead of a git tag)
   state: git
+  # Duration after which sync operations time out (defaults to 1m)
+  timeout:
 
 git:
   # URL of git repo with Kubernetes manifests; e.g. git.url=ssh://git@github.com/fluxcd/flux-get-started
diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go
index 2703be0a..9036ed88 100644
--- a/cmd/fluxd/main.go
+++ b/cmd/fluxd/main.go
@@ -136,6 +136,7 @@ func main() {
 
 		// syncing
 		syncInterval = fs.Duration("sync-interval", 5*time.Minute, "apply config in git to cluster at least this often, even if there are no new commits")
+		syncTimeout  = fs.Duration("sync-timeout", 1*time.Minute, "duration after which sync operations time out")
 		syncGC       = fs.Bool("sync-garbage-collection", false, "experimental; delete resources that were created by fluxd, but are no longer in the git repo")
 		dryGC        = fs.Bool("sync-garbage-collection-dry", false, "experimental; only log what would be garbage collected, rather than deleting. Implies --sync-garbage-collection")
 		syncState    = fs.String("sync-state", fluxsync.GitTagStateMode, fmt.Sprintf("method used by flux for storing state (one of {%s})", strings.Join([]string{fluxsync.GitTagStateMode, fluxsync.NativeStateMode}, ",")))
@@ -696,6 +697,7 @@ func main() {
 		GitSecretEnabled:          *gitSecret,
 		LoopVars: &daemon.LoopVars{
 			SyncInterval:        *syncInterval,
+			SyncTimeout:         *syncTimeout,
 			SyncState:           syncProvider,
 			AutomationInterval:  *automationInterval,
 			GitTimeout:          *gitTimeout,
diff --git a/docs/references/daemon.md b/docs/references/daemon.md
index 62e39c9c..a559ee61 100644
--- a/docs/references/daemon.md
+++ b/docs/references/daemon.md
@@ -59,6 +59,7 @@ Version controlling of cluster manifests provides reproducibility and a historic
 | --git-timeout                                    | `20s`                    | duration after which git operations time out
 | **syncing:** control over how config is applied to the cluster
 | --sync-interval                                  | `5m`                     | apply the git config to the cluster at least this often. New commits may provoke more frequent syncs
+| --sync-timeout                                   | `1m`                     |  duration after which sync operations time out
 | --sync-garbage-collection                        | `false`                  | experimental: when set, fluxd will delete resources that it created, but are no longer present in git
 | **registry cache:** (none of these need overriding, usually)
 | --memcached-hostname                             | `memcached`                        | hostname for memcached service to use for caching image metadata
diff --git a/docs/references/fluxyaml-config-files.md b/docs/references/fluxyaml-config-files.md
index 48c5cc6e..690a9c4c 100644
--- a/docs/references/fluxyaml-config-files.md
+++ b/docs/references/fluxyaml-config-files.md
@@ -168,6 +168,10 @@ In addition, `updaters` are provided with some environment variables:
   * `FLUX_POLICY_VALUE`: value of the policy to be added or updated in the controller. If the `FLUX_POLICY_VALUE`
   environment variable is not set, it means the policy should be removed.
 
+Please note that the default timeout for sync commands is set to one minute.
+If you run into errors like `error executing generator command: context deadline exceeded`,
+you can increase the timeout with the `--sync-timeout` fluxd command flag or the `sync.timeout` Helm chart option.
+
 ### Combining generators, updaters and raw manifest files
 
 The `.flux.yaml` files support including multiple generators and updaters. Here is an example combining multiple
diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go
index 7e4c7d37..e9059cd4 100644
--- a/pkg/daemon/daemon.go
+++ b/pkg/daemon/daemon.go
@@ -31,15 +31,6 @@ import (
 	"github.com/fluxcd/flux/pkg/update"
 )
 
-const (
-	// This is set to be in sympathy with the request / RPC timeout (i.e., empirically)
-	defaultHandlerTimeout = 10 * time.Second
-	// A job can take an arbitrary amount of time but we want to have
-	// a (generous) threshold for considering a job stuck and
-	// abandoning it
-	defaultJobTimeout = 60 * time.Second
-)
-
 // Daemon is the fully-functional state of a daemon (compare to
 // `NotReadyDaemon`).
 type Daemon struct {
@@ -277,7 +268,7 @@ func (d *Daemon) makeJobFromUpdate(update updateFunc) jobFunc {
 // executeJob runs a job func and keeps track of its status, so the
 // daemon can report it when asked.
 func (d *Daemon) executeJob(id job.ID, do jobFunc, logger log.Logger) (job.Result, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultJobTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), d.SyncTimeout)
 	defer cancel()
 	d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusRunning})
 	result, err := do(ctx, id, logger)
@@ -372,7 +363,7 @@ func (d *Daemon) UpdateManifests(ctx context.Context, spec update.Spec) (job.ID,
 func (d *Daemon) sync() jobFunc {
 	return func(ctx context.Context, jobID job.ID, logger log.Logger) (job.Result, error) {
 		var result job.Result
-		ctx, cancel := context.WithTimeout(ctx, defaultJobTimeout)
+		ctx, cancel := context.WithTimeout(ctx, d.SyncTimeout)
 		defer cancel()
 		err := d.Repo.Refresh(ctx)
 		if err != nil {
diff --git a/pkg/daemon/loop.go b/pkg/daemon/loop.go
index b936b890..fca18dcd 100644
--- a/pkg/daemon/loop.go
+++ b/pkg/daemon/loop.go
@@ -15,6 +15,7 @@ import (
 
 type LoopVars struct {
 	SyncInterval        time.Duration
+	SyncTimeout         time.Duration
 	AutomationInterval  time.Duration
 	GitTimeout          time.Duration
 	GitVerifySignatures bool


Test patch:
diff --git a/pkg/daemon/daemon_test.go b/pkg/daemon/daemon_test.go
index ad55a0b9..c7891877 100644
--- a/pkg/daemon/daemon_test.go
+++ b/pkg/daemon/daemon_test.go
@@ -741,7 +741,7 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *mock.Mock, *mockEventWr
 		JobStatusCache: &job.StatusCache{Size: 100},
 		EventWriter:    events,
 		Logger:         logger,
-		LoopVars:       &LoopVars{GitTimeout: timeout, SyncState: gitSync},
+		LoopVars:       &LoopVars{SyncTimeout: timeout, GitTimeout: timeout, SyncState: gitSync},
 	}
 
 	start := func() {
diff --git a/pkg/daemon/sync_test.go b/pkg/daemon/sync_test.go
index 39525a02..d159c35f 100644
--- a/pkg/daemon/sync_test.go
+++ b/pkg/daemon/sync_test.go
@@ -75,7 +75,7 @@ func daemon(t *testing.T) (*Daemon, func()) {
 		JobStatusCache: &job.StatusCache{Size: 100},
 		EventWriter:    events,
 		Logger:         log.NewLogfmtLogger(os.Stdout),
-		LoopVars:       &LoopVars{GitTimeout: timeout},
+		LoopVars:       &LoopVars{SyncTimeout: timeout, GitTimeout: timeout},
 	}
 	return d, func() {
 		close(shutdown)