Repository: rancher/fleet
Pull request #3407 (2025-02-28 11:43:31)

Retries in fleet apply and single patch in webhook
Adds a retry mechanism to `fleet apply` after observing several errors due to conflicts under high loads. As the job created is not retrying by itself the `GitRepo` gets hung.

Also changes from 1 Status Update and Patch to one single patch in the webhook handling to mitigate conflicts and possible race conditions.

Refers to: https://github.com/rancher/fleet/issues/3067




Patch:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c22455f1..f89c1fc6 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -22,7 +22,6 @@ jobs:
 
     env:
       IS_HOTFIX: ${{ contains(github.ref, '-hotfix-') }}
-      REGISTRY: "${{ contains(github.ref, '-hotfix-') && 'stgregistry.suse.com' || 'docker.io' }}"
 
     if: github.repository == 'rancher/fleet'
     steps:
@@ -108,14 +107,6 @@ jobs:
           username: ${{ env.DOCKER_USERNAME }}
           password: ${{ env.DOCKER_PASSWORD }}
 
-      - name: Log into Staging registry
-        uses: docker/login-action@v3
-        if: ${{ env.IS_HOTFIX == 'true' }}
-        with:
-          username: ${{ env.STAGE_REGISTRY_USERNAME }}
-          password: ${{ env.STAGE_REGISTRY_PASSWORD }}
-          registry: ${{ env.REGISTRY }}
-
       - name: Log into Prime registry
         uses: docker/login-action@v3
         with:
@@ -174,6 +165,7 @@ jobs:
           done
 
       - name: Upload charts to release
+        if: ${{ env.IS_HOTFIX == 'false' }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           repo: "rancher"
@@ -198,6 +190,7 @@ jobs:
           find dist/ -name '*.tgz' -exec gh release upload $tag {} +
 
       - name: Add charts to branch
+        if: ${{ env.IS_HOTFIX == 'false' }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
index 86fd1d3e..4e7d0a32 100644
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@@ -6,7 +6,6 @@ project_name: fleet
 env:
   - CGO_ENABLED=0
   - IS_HOTFIX={{ if isEnvSet "IS_HOTFIX"}}{{ .Env.IS_HOTFIX }}{{else}}false{{end}}
-  - REGISTRY={{ if isEnvSet "REGISTRY" }}{{ .Env.REGISTRY }}{{else}}docker.io{{end}}
 
 release:
   prerelease: auto
@@ -89,6 +88,7 @@ dockers:
   -
     use: buildx
     id: fleet-amd64-public
+    skip_push: "{{ .Env.IS_HOTFIX }}"
 
     # GOOS of the built binaries/packages that should be used.
     goos: linux
@@ -103,7 +103,7 @@ dockers:
 
     # Templates of the Docker image names.
     image_templates:
-    - "{{ .Env.REGISTRY }}/rancher/fleet:{{ .Tag }}-linux-amd64"
+    - "docker.io/rancher/fleet:{{ .Tag }}-linux-amd64"
 
     # Path to the Dockerfile (from the project root).
     dockerfile: package/Dockerfile
@@ -123,13 +123,14 @@ dockers:
   -
     use: buildx
     id: fleet-arm64-public
+    skip_push: "{{ .Env.IS_HOTFIX }}"
     goos: linux
     goarch: arm64
     ids:
     - fleet-controller
     - fleet-cli
     image_templates:
-    - "{{ .Env.REGISTRY }}/rancher/fleet:{{ .Tag }}-linux-arm64"
+    - "docker.io/rancher/fleet:{{ .Tag }}-linux-arm64"
     dockerfile: package/Dockerfile
     build_flag_templates:
     - "--pull"
@@ -146,12 +147,13 @@ dockers:
   -
     use: buildx
     id: fleet-agent-amd64-public
+    skip_push: "{{ .Env.IS_HOTFIX }}"
     goos: linux
     goarch: amd64
     ids:
     - fleet-agent
     image_templates:
-    - "{{ .Env.REGISTRY }}/rancher/fleet-agent:{{ .Tag }}-linux-amd64"
+    - "docker.io/rancher/fleet-agent:{{ .Tag }}-linux-amd64"
     dockerfile: package/Dockerfile.agent
     build_flag_templates:
     - "--pull"
@@ -166,12 +168,13 @@ dockers:
   -
     use: buildx
     id: fleet-agent-arm64-public
+    skip_push: "{{ .Env.IS_HOTFIX }}"
     goos: linux
     goarch: arm64
     ids:
     - fleet-agent
     image_templates:
-    - "{{ .Env.REGISTRY }}/rancher/fleet-agent:{{ .Tag }}-linux-arm64"
+    - "docker.io/rancher/fleet-agent:{{ .Tag }}-linux-arm64"
     dockerfile: package/Dockerfile.agent
     build_flag_templates:
     - "--pull"
@@ -298,17 +301,19 @@ dockers:
 
 docker_manifests:
   # https://goreleaser.com/customization/docker_manifest/
-  - name_template: "{{ .Env.REGISTRY }}/rancher/fleet:{{ .Tag }}"
+  - name_template: "docker.io/rancher/fleet:{{ .Tag }}"
     id: fleet-manifest-public
+    skip_push: "{{ .Env.IS_HOTFIX }}"
     image_templates:
-    - "{{ .Env.REGISTRY }}/rancher/fleet:{{ .Tag }}-linux-amd64"
-    - "{{ .Env.REGISTRY }}/rancher/fleet:{{ .Tag }}-linux-arm64"
+    - "docker.io/rancher/fleet:{{ .Tag }}-linux-amd64"
+    - "docker.io/rancher/fleet:{{ .Tag }}-linux-arm64"
 
-  - name_template: "{{ .Env.REGISTRY }}/rancher/fleet-agent:{{ .Tag }}"
+  - name_template: "docker.io/rancher/fleet-agent:{{ .Tag }}"
     id: fleet-agent-manifest-public
+    skip_push: "{{ .Env.IS_HOTFIX }}"
     image_templates:
-    - "{{ .Env.REGISTRY }}/rancher/fleet-agent:{{ .Tag }}-linux-amd64"
-    - "{{ .Env.REGISTRY }}/rancher/fleet-agent:{{ .Tag }}-linux-arm64"
+    - "docker.io/rancher/fleet-agent:{{ .Tag }}-linux-amd64"
+    - "docker.io/rancher/fleet-agent:{{ .Tag }}-linux-arm64"
 
     # Disabled until `docker manifest create` supports provenance meta data
     # - name_template: "{{ .Env.PRIME_REGISTRY }}/rancher/fleet:{{ .Tag }}"
diff --git a/go.mod b/go.mod
index 715c8949..77dabb7e 100644
--- a/go.mod
+++ b/go.mod
@@ -15,10 +15,10 @@ require (
 	github.com/chartmuseum/helm-push v0.10.4
 	github.com/cheggaaa/pb v1.0.29
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
-	github.com/docker/docker v28.0.0+incompatible
+	github.com/docker/docker v28.0.1+incompatible
 	github.com/evanphx/json-patch v5.9.11+incompatible
 	github.com/go-git/go-billy/v5 v5.6.2
-	github.com/go-git/go-git/v5 v5.13.2
+	github.com/go-git/go-git/v5 v5.14.0
 	github.com/go-logr/logr v1.4.2
 	github.com/go-playground/webhooks/v6 v6.4.0
 	github.com/gobwas/glob v0.2.3
@@ -95,7 +95,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
-	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/cloudflare/circl v1.6.0 // indirect
 	github.com/containerd/containerd v1.7.24 // indirect
 	github.com/containerd/errdefs v0.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
@@ -103,7 +103,7 @@ require (
 	github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
 	github.com/cpuguy83/dockercfg v0.3.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
-	github.com/cyphar/filepath-securejoin v0.3.6 // indirect
+	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/cli v27.5.0+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
@@ -132,7 +132,7 @@ require (
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gobuffalo/flect v1.0.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
 	github.com/google/btree v1.1.3 // indirect
@@ -203,7 +203,7 @@ require (
 	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
-	github.com/skeema/knownhosts v1.3.0 // indirect
+	github.com/skeema/knownhosts v1.3.1 // indirect
 	github.com/spf13/cast v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
diff --git a/go.sum b/go.sum
index 9c87c2f9..35f3d405 100644
--- a/go.sum
+++ b/go.sum
@@ -696,8 +696,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
-github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
+github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk=
+github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -732,8 +732,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
-github.com/cyphar/filepath-securejoin v0.3.6 h1:4d9N5ykBnSp5Xn2JkhocYDkOpURL/18CYMpo6xB9uWM=
-github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
+github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s=
+github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -746,8 +746,8 @@ github.com/docker/cli v27.5.0+incompatible h1:aMphQkcGtpHixwwhAXJT1rrK/detk2JIvD
 github.com/docker/cli v27.5.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM=
-github.com/docker/docker v28.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v28.0.1+incompatible h1:FCHjSRdXhNRFjlHMTv4jUNlIBbTeRjrWfeFuJp7jpo0=
+github.com/docker/docker v28.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo=
 github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
@@ -762,8 +762,8 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arX
 github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/elazarl/goproxy v1.4.0 h1:4GyuSbFa+s26+3rmYNSuUVsx+HgPrV1bk1jXI0l9wjM=
-github.com/elazarl/goproxy v1.4.0/go.mod h1:X/5W/t+gzDyLfHW4DrMdpjqYjpXsURlBt9lpBDxZZZQ=
+github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
+github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
 github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
 github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -822,8 +822,8 @@ github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UN
 github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
-github.com/go-git/go-git/v5 v5.13.2 h1:7O7xvsK7K+rZPKW6AQR1YyNhfywkv7B8/FsP3ki6Zv0=
-github.com/go-git/go-git/v5 v5.13.2/go.mod h1:hWdW5P4YZRjmpGHwRH2v3zkWcNl6HeXaXQEMGb3NJ9A=
+github.com/go-git/go-git/v5 v5.14.0 h1:/MD3lCrGjCen5WfEAzKg00MJJffKhC8gzS80ycmCi60=
+github.com/go-git/go-git/v5 v5.14.0/go.mod h1:Z5Xhoia5PcWA3NF8vRLURn9E5FRhSl7dGj9ItW3Wk5k=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -876,8 +876,9 @@ github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
+github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -1240,8 +1241,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
-github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
-github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/rubenv/sql-migrate v1.7.1 h1:f/o0WgfO/GqNuVg+6801K/KW3WdDSupzSjDYODmiUq4=
 github.com/rubenv/sql-migrate v1.7.1/go.mod h1:Ob2Psprc0/3ggbM6wCzyYVFFuc6FyZrb2AS+ezLDFb4=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -1262,8 +1263,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY=
-github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
+github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8=
+github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
diff --git a/internal/cmd/cli/apply.go b/internal/cmd/cli/apply.go
index 14b9416d..7fb661c6 100644
--- a/internal/cmd/cli/apply.go
+++ b/internal/cmd/cli/apply.go
@@ -5,8 +5,10 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
 	"github.com/rancher/fleet/internal/bundlereader"
@@ -16,9 +18,15 @@ import (
 	"github.com/rancher/fleet/internal/cmd/cli/writer"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/util/yaml"
 )
 
+const (
+	FleetApplyConflictRetriesEnv = "FLEET_APPLY_CONFLICT_RETRIES"
+	defaultApplyConflictRetries  = 1
+)
+
 type readFile func(name string) ([]byte, error)
 
 // NewApply returns a subcommand to create bundles from directories
@@ -68,6 +76,24 @@ func (r *Apply) PersistentPre(_ *cobra.Command, _ []string) error {
 }
 
 func (a *Apply) Run(cmd *cobra.Command, args []string) error {
+	// Apply retries on conflict errors.
+	// We could have race conditions updating the Bundle in high load situations
+	var err error
+	retries, err := GetOnConflictRetries()
+	if err != nil {
+		logrus.Errorf("failed parsing env variable %s, using defaults, err: %v", FleetApplyConflictRetriesEnv, err)
+	}
+	for range retries {
+		err = a.run(cmd, args)
+		if !errors.IsConflict(err) {
+			break
+		}
+	}
+
+	return err
+}
+
+func (a *Apply) run(cmd *cobra.Command, args []string) error {
 	labels := a.Label
 	if a.Commit == "" {
 		a.Commit = currentCommit()
@@ -210,3 +236,19 @@ func currentCommit() string {
 	}
 	return ""
 }
+
+func GetOnConflictRetries() (int, error) {
+	s := os.Getenv(FleetApplyConflictRetriesEnv)
+	if s != "" {
+		// check if we have a valid value
+		// it must be an integer
+		r, err := strconv.Atoi(s)
+		if err != nil {
+			return defaultApplyConflictRetries, err
+		} else {
+			return r, nil
+		}
+	}
+
+	return defaultApplyConflictRetries, nil
+}
diff --git a/internal/cmd/controller/gitops/reconciler/gitjob_controller.go b/internal/cmd/controller/gitops/reconciler/gitjob_controller.go
index 33364d45..1805598d 100644
--- a/internal/cmd/controller/gitops/reconciler/gitjob_controller.go
+++ b/internal/cmd/controller/gitops/reconciler/gitjob_controller.go
@@ -15,6 +15,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/reugn/go-quartz/quartz"
 
+	fleetcli "github.com/rancher/fleet/internal/cmd/cli"
 	fleetutil "github.com/rancher/fleet/internal/cmd/controller/errorutil"
 	"github.com/rancher/fleet/internal/cmd/controller/finalize"
 	"github.com/rancher/fleet/internal/cmd/controller/imagescan"
@@ -778,7 +779,7 @@ func (r *GitJobReconciler) newJobSpec(ctx context.Context, gitrepo *v1alpha1.Git
 
 	saName := names.SafeConcatName("git", gitrepo.Name)
 	logger := log.FromContext(ctx)
-	args, envs := argsAndEnvs(gitrepo, logger.V(1).Enabled(), CACertsFilePathOverride)
+	args, envs := argsAndEnvs(gitrepo, logger, CACertsFilePathOverride)
 
 	return &batchv1.JobSpec{
 		BackoffLimit: &zero,
@@ -834,13 +835,13 @@ func (r *GitJobReconciler) newJobSpec(ctx context.Context, gitrepo *v1alpha1.Git
 	}, nil
 }
 
-func argsAndEnvs(gitrepo *v1alpha1.GitRepo, debug bool, CACertsPathOverride string) ([]string, []corev1.EnvVar) {
+func argsAndEnvs(gitrepo *v1alpha1.GitRepo, logger logr.Logger, CACertsPathOverride string) ([]string, []corev1.EnvVar) {
 	args := []string{
 		"fleet",
 		"apply",
 	}
 
-	if debug {
+	if logger.V(1).Enabled() {
 		args = append(args, "--debug", "--debug-level", "9")
 	}
 
@@ -876,11 +877,19 @@ func argsAndEnvs(gitrepo *v1alpha1.GitRepo, debug bool, CACertsPathOverride stri
 		}
 	}
 
+	fleetApplyRetries, err := fleetcli.GetOnConflictRetries()
+	if err != nil {
+		logger.Error(err, "failed parsing env variable, using defaults", "env_var_name", fleetcli.FleetApplyConflictRetriesEnv)
+	}
 	env := []corev1.EnvVar{
 		{
 			Name:  "HOME",
 			Value: fleetHomeDir,
 		},
+		{
+			Name:  fleetcli.FleetApplyConflictRetriesEnv,
+			Value: strconv.Itoa(fleetApplyRetries),
+		},
 	}
 	if gitrepo.Spec.HelmSecretNameForPaths != "" {
 		helmArgs := []string{
diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go
index cbe87a06..00fba75a 100644
--- a/pkg/webhook/webhook.go
+++ b/pkg/webhook/webhook.go
@@ -24,9 +24,8 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	ktypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/types"
 	kcache "k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/util/retry"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -224,33 +223,29 @@ func (w *Webhook) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 
 			if gitrepo.Status.WebhookCommit != revision && revision != "" {
 				var gitRepoFromCluster v1alpha1.GitRepo
-				if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
-					err := w.client.Get(
-						ctx,
-						ktypes.NamespacedName{
-							Name:      gitrepo.Name,
-							Namespace: gitrepo.Namespace,
-						}, &gitRepoFromCluster,
-					)
-					if err != nil {
-						return err
-					}
-					gitRepoFromCluster.Status.WebhookCommit = revision
-					return w.client.Status().Update(ctx, &gitRepoFromCluster)
-				}); err != nil {
+				err := w.client.Get(
+					ctx,
+					types.NamespacedName{
+						Name:      gitrepo.Name,
+						Namespace: gitrepo.Namespace,
+					}, &gitRepoFromCluster,
+				)
+				if err != nil {
 					w.logAndReturn(rw, err)
 					return
 				}
+				orig := gitRepoFromCluster.DeepCopy()
+				gitRepoFromCluster.Status.WebhookCommit = revision
 				// if PollingInterval is not set and webhook is configured, set it to 1 hour
-				if gitrepo.Spec.PollingInterval == nil {
-					p := client.MergeFrom(&gitRepoFromCluster)
+				if gitRepoFromCluster.Spec.PollingInterval == nil {
 					gitRepoFromCluster.Spec.PollingInterval = &metav1.Duration{
 						Duration: webhookDefaultSyncInterval * time.Second,
 					}
-					if err := w.client.Patch(ctx, &gitRepoFromCluster, p); err != nil {
-						w.logAndReturn(rw, err)
-						return
-					}
+				}
+				p := client.MergeFrom(orig)
+				if err := w.client.Status().Patch(ctx, &gitRepoFromCluster, p); err != nil {
+					w.logAndReturn(rw, err)
+					return
 				}
 			}
 		}


Test patch:
diff --git a/internal/cmd/controller/gitops/reconciler/gitjob_test.go b/internal/cmd/controller/gitops/reconciler/gitjob_test.go
index e4841590..3e708344 100644
--- a/internal/cmd/controller/gitops/reconciler/gitjob_test.go
+++ b/internal/cmd/controller/gitops/reconciler/gitjob_test.go
@@ -1392,6 +1392,10 @@ func TestGenerateJob_EnvVars(t *testing.T) {
 					Name:  "HOME",
 					Value: "/fleet-home",
 				},
+				{
+					Name:  "FLEET_APPLY_CONFLICT_RETRIES",
+					Value: "1",
+				},
 				{
 					Name:  "GIT_SSH_COMMAND",
 					Value: "ssh -o stricthostkeychecking=accept-new",
@@ -1414,6 +1418,10 @@ func TestGenerateJob_EnvVars(t *testing.T) {
 					Name:  "HOME",
 					Value: "/fleet-home",
 				},
+				{
+					Name:  "FLEET_APPLY_CONFLICT_RETRIES",
+					Value: "1",
+				},
 				{
 					Name:  "COMMIT",
 					Value: "commit",
@@ -1439,6 +1447,52 @@ func TestGenerateJob_EnvVars(t *testing.T) {
 			},
 			osEnv: map[string]string{"HTTP_PROXY": "httpProxy", "HTTPS_PROXY": "httpsProxy"},
 		},
+		"retries_valid": {
+			gitrepo: &fleetv1.GitRepo{
+				Spec: fleetv1.GitRepoSpec{},
+				Status: fleetv1.GitRepoStatus{
+					Commit: "commit",
+				},
+			},
+			expectedContainerEnvVars: []corev1.EnvVar{
+				{
+					Name:  "HOME",
+					Value: "/fleet-home",
+				},
+				{
+					Name:  "FLEET_APPLY_CONFLICT_RETRIES",
+					Value: "3",
+				},
+				{
+					Name:  "COMMIT",
+					Value: "commit",
+				},
+			},
+			osEnv: map[string]string{"FLEET_APPLY_CONFLICT_RETRIES": "3"},
+		},
+		"retries_not_valid": {
+			gitrepo: &fleetv1.GitRepo{
+				Spec: fleetv1.GitRepoSpec{},
+				Status: fleetv1.GitRepoStatus{
+					Commit: "commit",
+				},
+			},
+			expectedContainerEnvVars: []corev1.EnvVar{
+				{
+					Name:  "HOME",
+					Value: "/fleet-home",
+				},
+				{
+					Name:  "FLEET_APPLY_CONFLICT_RETRIES",
+					Value: "1",
+				},
+				{
+					Name:  "COMMIT",
+					Value: "commit",
+				},
+			},
+			osEnv: map[string]string{"FLEET_APPLY_CONFLICT_RETRIES": "this_is_not_an_int"},
+		},
 	}
 
 	for name, test := range tests {
diff --git a/pkg/webhook/webhook_test.go b/pkg/webhook/webhook_test.go
index e8c53f72..dd43c25e 100644
--- a/pkg/webhook/webhook_test.go
+++ b/pkg/webhook/webhook_test.go
@@ -385,16 +385,12 @@ func TestGitHubRightSecretAndCommitUpdated(t *testing.T) {
 
 	// Status().Update() mock call
 	mockClient.EXPECT().Status().Return(statusClient).Times(1)
-	statusClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Do(
-		func(ctx context.Context, repo *v1alpha1.GitRepo, opts ...interface{}) {
+	statusClient.EXPECT().Patch(gomock.Any(), gomock.Any(), gomock.Any()).Do(
+		func(ctx context.Context, repo *v1alpha1.GitRepo, _ client.Patch, opts ...interface{}) {
 			// check that the commit is the expected one
 			if repo.Status.WebhookCommit != expectedCommit {
 				t.Errorf("expecting gitrepo webhook commit %s, got %s", expectedCommit, repo.Status.WebhookCommit)
 			}
-		},
-	).Times(1)
-	mockClient.EXPECT().Patch(gomock.Any(), gomock.Any(), gomock.Any()).Do(
-		func(ctx context.Context, repo *v1alpha1.GitRepo, _ client.Patch, opts ...interface{}) {
 			if repo.Spec.PollingInterval.Duration != time.Hour {
 				t.Errorf("expecting gitrepo polling interval 1h, got %s", repo.Spec.PollingInterval.Duration)
 			}