Repository: apache/apisix-ingress-controller

Issue:
bug: Some CRD missing status sub-resource
### Current Behavior

Due to the lack of definition for status sub-resource, and status cannot be recorded.

**Missing CRDs for status sub-resource:**

- [ ] [ApisixClusterConfig](https://github.com/apache/apisix-ingress-controller/blob/master/samples/deploy/crd/v1/ApisixConsumer.yaml)
- [ ] [ApisixConsumer](https://github.com/apache/apisix-ingress-controller/blob/master/samples/deploy/crd/v1/ApisixClusterConfig)

Similar to #1619 , we need to add the definition of the status field and related e2e testing in the v2 apiversion.

https://github.com/apache/apisix-ingress-controller/blob/8e437006dc227c1bccf3fec93d70cdac40dc31a3/samples/deploy/crd/v1/ApisixUpstream.yaml#L827-L844

### Environment

- APISIX Ingress controller version (run `apisix-ingress-controller version --long`) 1.6.0
- Kubernetes cluster version (run `kubectl version`)
- OS version if running APISIX Ingress controller in a bare-metal environment (run `uname -a`)


Discussion:
@Chever-John: Let me try it:)@AlinsRan: Assigned, Thanks.@afzal442: Hi, LMK if there is any update. I'd like to take it up.

cc @tao12345666333 @tao12345666333: Assigned. Thanks @Chever-John: Emmmm, there's some work going on right now. I can give it another two days.
(fixing e2e-test phase@afzal442: Alright @Chever-John ! Of course you can.@Chever-John: @afzal442 Thanks:)

I will keep you all informed if there is an update notice, for example, if I cannot proceed, and you are welcome to continue with the work.
Have a happy day!

Solution patch:
diff --git a/.github/workflows/e2e-test-ci-v2-cron-dev.yml b/.github/workflows/e2e-test-ci-v2-cron-dev.yml
index b0f5f214..19e65731 100644
--- a/.github/workflows/e2e-test-ci-v2-cron-dev.yml
+++ b/.github/workflows/e2e-test-ci-v2-cron-dev.yml
@@ -33,6 +33,7 @@ concurrency:
 
 jobs:
   changes:
+    if: ${{ (!github.event.pull_request.draft && github.event.schedule == '0 0 * * *') }}
     runs-on: ubuntu-20.04
     outputs:
       docs: ${{ steps.filter.outputs.docs }}
@@ -113,6 +114,7 @@ jobs:
             ${REGISTRY}/apisix-ingress-controller:dev \
             ${REGISTRY}/httpbin:dev \
             ${REGISTRY}/test-backend:dev \
+            ${REGISTRY}/test-timeout:dev \
             ${REGISTRY}/echo-server:dev \
             ${REGISTRY}/busybox:dev \
             | pigz > docker-dev.tar.gz
@@ -204,7 +206,6 @@ jobs:
         env:
           E2E_FOCUS: "${{ matrix.suite }}"
           ENABLE_PROXY: "false"
-          INGRESS_CLASS: "${{ matrix.ingress-class-values }}"
           E2E_SKIP_BUILD: "1"
           E2E_FLAKE_ATTEMPTS: "2"
           E2E_ENV: "ci"
diff --git a/.github/workflows/e2e-test-ci-v2-cron.yml b/.github/workflows/e2e-test-ci-v2-cron.yml
index f3730503..c765f6d5 100644
--- a/.github/workflows/e2e-test-ci-v2-cron.yml
+++ b/.github/workflows/e2e-test-ci-v2-cron.yml
@@ -113,6 +113,7 @@ jobs:
             ${REGISTRY}/apisix-ingress-controller:dev \
             ${REGISTRY}/httpbin:dev \
             ${REGISTRY}/test-backend:dev \
+            ${REGISTRY}/test-timeout:dev \
             ${REGISTRY}/echo-server:dev \
             ${REGISTRY}/busybox:dev \
             | pigz > docker-v2.tar.gz
@@ -197,4 +198,5 @@ jobs:
           E2E_FLAKE_ATTEMPTS: "2"
           E2E_ENV: "ci"
         run: |
+          echo ${E2E_FOCUS} | grep "suite-cluster" && export E2E_NODES=1
           make e2e-test
diff --git a/.github/workflows/e2e-test-ci.yml b/.github/workflows/e2e-test-ci.yml
index 1c3166ff..56e430c6 100644
--- a/.github/workflows/e2e-test-ci.yml
+++ b/.github/workflows/e2e-test-ci.yml
@@ -114,6 +114,7 @@ jobs:
             ${REGISTRY}/apisix-ingress-controller:dev \
             ${REGISTRY}/httpbin:dev \
             ${REGISTRY}/test-backend:dev \
+            ${REGISTRY}/test-timeout:dev \
             ${REGISTRY}/echo-server:dev \
             ${REGISTRY}/busybox:dev \
             | pigz > docker.tar.gz
diff --git a/.github/workflows/k8s-timer-ci.yml b/.github/workflows/k8s-timer-ci.yml
index 5a8eefe3..9632de0b 100644
--- a/.github/workflows/k8s-timer-ci.yml
+++ b/.github/workflows/k8s-timer-ci.yml
@@ -103,6 +103,7 @@ jobs:
             ${REGISTRY}/apisix-ingress-controller:dev \
             ${REGISTRY}/httpbin:dev \
             ${REGISTRY}/test-backend:dev \
+            ${REGISTRY}/test-timeout:dev \
             ${REGISTRY}/echo-server:dev \
             ${REGISTRY}/busybox:dev \
             | pigz > docker.tar.gz
diff --git a/Makefile b/Makefile
index 2c60bd17..9778c4ff 100644
--- a/Makefile
+++ b/Makefile
@@ -103,8 +103,9 @@ ifeq ($(E2E_SKIP_BUILD), 0)
 	docker tag kennethreitz/httpbin $(REGISTRY)/httpbin:$(IMAGE_TAG)
 
 	docker build -t test-backend:$(IMAGE_TAG) --build-arg ENABLE_PROXY=$(ENABLE_PROXY) ./test/e2e/testbackend
+	docker build -t test-timeout:$(IMAGE_TAG) --build-arg ENABLE_PROXY=$(ENABLE_PROXY) ./test/e2e/testtimeout	
 	docker tag test-backend:$(IMAGE_TAG) $(REGISTRY)/test-backend:$(IMAGE_TAG)
-
+	docker tag test-timeout:$(IMAGE_TAG) $(REGISTRY)/test-timeout:$(IMAGE_TAG)
 	docker tag apache/apisix-ingress-controller:$(IMAGE_TAG) $(REGISTRY)/apisix-ingress-controller:$(IMAGE_TAG)
 
 	docker pull jmalloc/echo-server:latest
@@ -122,6 +123,7 @@ ifeq ($(E2E_SKIP_BUILD), 0)
 	docker push $(REGISTRY)/etcd:$(IMAGE_TAG)
 	docker push $(REGISTRY)/httpbin:$(IMAGE_TAG)
 	docker push $(REGISTRY)/test-backend:$(IMAGE_TAG)
+	docker push $(REGISTRY)/test-timeout:$(IMAGE_TAG)
 	docker push $(REGISTRY)/apisix-ingress-controller:$(IMAGE_TAG)
 	docker push $(REGISTRY)/echo-server:$(IMAGE_TAG)
 	docker push $(REGISTRY)/busybox:$(IMAGE_TAG)
@@ -305,6 +307,7 @@ kind-load-images:
             $(REGISTRY)/apisix-ingress-controller:dev \
             $(REGISTRY)/httpbin:dev \
             $(REGISTRY)/test-backend:dev \
+			$(REGISTRY)/test-timeout:dev \
             $(REGISTRY)/echo-server:dev \
             $(REGISTRY)/busybox:dev
 
diff --git a/pkg/providers/ingress/translation/annotations.go b/pkg/providers/ingress/translation/annotations.go
index b341dfc6..76a43334 100644
--- a/pkg/providers/ingress/translation/annotations.go
+++ b/pkg/providers/ingress/translation/annotations.go
@@ -24,7 +24,7 @@ import (
 	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations/plugins"
 	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations/regex"
 	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations/servicenamespace"
-	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations/upstreamscheme"
+	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations/upstream"
 	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations/websocket"
 	apisix "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1"
 )
@@ -36,7 +36,7 @@ type Ingress struct {
 	EnableWebSocket  bool
 	PluginConfigName string
 	ServiceNamespace string
-	UpstreamScheme   string
+	Upstream         upstream.Upstream
 }
 
 var (
@@ -46,7 +46,7 @@ var (
 		"EnableWebSocket":  websocket.NewParser(),
 		"PluginConfigName": pluginconfig.NewParser(),
 		"ServiceNamespace": servicenamespace.NewParser(),
-		"UpstreamScheme":   upstreamscheme.NewParser(),
+		"Upstream":         upstream.NewParser(),
 	}
 )
 
diff --git a/pkg/providers/ingress/translation/annotations/types.go b/pkg/providers/ingress/translation/annotations/types.go
index 503e79b2..aac0a0ba 100644
--- a/pkg/providers/ingress/translation/annotations/types.go
+++ b/pkg/providers/ingress/translation/annotations/types.go
@@ -27,6 +27,12 @@ const (
 	AnnotationsEnableWebSocket  = AnnotationsPrefix + "enable-websocket"
 	AnnotationsPluginConfigName = AnnotationsPrefix + "plugin-config-name"
 	AnnotationsUpstreamScheme   = AnnotationsPrefix + "upstream-scheme"
+
+	//support retries and timeouts on upstream
+	AnnotationsUpstreamRetry          = AnnotationsPrefix + "upstream-retries"
+	AnnotationsUpstreamTimeoutConnect = AnnotationsPrefix + "upstream-connect-timeout"
+	AnnotationsUpstreamTimeoutRead    = AnnotationsPrefix + "upstream-read-timeout"
+	AnnotationsUpstreamTimeoutSend    = AnnotationsPrefix + "upstream-send-timeout"
 )
 
 const (
diff --git a/pkg/providers/ingress/translation/annotations/upstream/upstream.go b/pkg/providers/ingress/translation/annotations/upstream/upstream.go
new file mode 100644
index 00000000..02692d9f
--- /dev/null
+++ b/pkg/providers/ingress/translation/annotations/upstream/upstream.go
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package upstream
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations"
+	apisixv1 "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1"
+)
+
+func NewParser() annotations.IngressAnnotationsParser {
+	return &Upstream{}
+}
+
+type Upstream struct {
+	Scheme         string
+	Retry          int
+	TimeoutRead    int
+	TimeoutConnect int
+	TimeoutSend    int
+}
+
+func (u *Upstream) Parse(e annotations.Extractor) (interface{}, error) {
+	scheme := strings.ToLower(e.GetStringAnnotation(annotations.AnnotationsUpstreamScheme))
+	if scheme != "" {
+		_, ok := apisixv1.ValidSchemes[scheme]
+		if !ok {
+			keys := make([]string, 0, len(apisixv1.ValidSchemes))
+			for key := range apisixv1.ValidSchemes {
+				keys = append(keys, key)
+			}
+			return nil, fmt.Errorf("scheme %s is not supported, Only { %s } are supported", scheme, strings.Join(keys, ", "))
+		}
+		u.Scheme = scheme
+	}
+
+	retry := e.GetStringAnnotation(annotations.AnnotationsUpstreamRetry)
+	if retry != "" {
+		t, err := strconv.Atoi(retry)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse retry as an integer: %s", err.Error())
+		}
+		u.Retry = t
+	}
+
+	timeoutConnect := strings.TrimSuffix(e.GetStringAnnotation(annotations.AnnotationsUpstreamTimeoutConnect), "s")
+	if timeoutConnect != "" {
+		t, err := strconv.Atoi(timeoutConnect)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse timeout as an integer: %s", err.Error())
+		}
+		u.TimeoutConnect = t
+	}
+
+	timeoutRead := strings.TrimSuffix(e.GetStringAnnotation(annotations.AnnotationsUpstreamTimeoutRead), "s")
+	if timeoutRead != "" {
+		t, err := strconv.Atoi(timeoutRead)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse timeout as an integer: %s", err.Error())
+		}
+		u.TimeoutRead = t
+	}
+
+	timeoutSend := strings.TrimSuffix(e.GetStringAnnotation(annotations.AnnotationsUpstreamTimeoutSend), "s")
+	if timeoutSend != "" {
+		t, err := strconv.Atoi(timeoutSend)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse timeout as an integer: %s", err.Error())
+		}
+		u.TimeoutSend = t
+	}
+
+	return *u, nil
+}
diff --git a/pkg/providers/ingress/translation/annotations/upstreamscheme/upstreamscheme.go b/pkg/providers/ingress/translation/annotations/upstreamscheme/upstreamscheme.go
deleted file mode 100644
index f9ce46b6..00000000
--- a/pkg/providers/ingress/translation/annotations/upstreamscheme/upstreamscheme.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//	http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package upstreamscheme
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations"
-	apisixv1 "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1"
-)
-
-type upstreamscheme struct{}
-
-func NewParser() annotations.IngressAnnotationsParser {
-	return &upstreamscheme{}
-}
-
-func (w *upstreamscheme) Parse(e annotations.Extractor) (interface{}, error) {
-	scheme := strings.ToLower(e.GetStringAnnotation(annotations.AnnotationsUpstreamScheme))
-	if scheme == "" {
-		return nil, nil
-	}
-	_, ok := apisixv1.ValidSchemes[scheme]
-	if ok {
-		return scheme, nil
-	}
-
-	keys := make([]string, 0, len(apisixv1.ValidSchemes))
-	for key := range apisixv1.ValidSchemes {
-		keys = append(keys, key)
-	}
-
-	return nil, fmt.Errorf("scheme %s is not supported, Only { %s } are supported", scheme, strings.Join(keys, ", "))
-}
diff --git a/pkg/providers/ingress/translation/translator.go b/pkg/providers/ingress/translation/translator.go
index 82ac87f4..30f30c3d 100644
--- a/pkg/providers/ingress/translation/translator.go
+++ b/pkg/providers/ingress/translation/translator.go
@@ -201,8 +201,28 @@ func (t *translator) translateIngressV1(ing *networkingv1.Ingress, skipVerify bo
 						return nil, err
 					}
 				}
-				if ingress.UpstreamScheme != "" {
-					ups.Scheme = ingress.UpstreamScheme
+				if ingress.Upstream.Scheme != "" {
+					ups.Scheme = ingress.Upstream.Scheme
+				}
+				if ingress.Upstream.Retry > 0 {
+					retry := ingress.Upstream.Retry
+					ups.Retries = &retry
+				}
+				if ups.Timeout == nil {
+					ups.Timeout = &apisixv1.UpstreamTimeout{
+						Read:    60,
+						Send:    60,
+						Connect: 60,
+					}
+				}
+				if ingress.Upstream.TimeoutConnect > 0 {
+					ups.Timeout.Connect = ingress.Upstream.TimeoutConnect
+				}
+				if ingress.Upstream.TimeoutRead > 0 {
+					ups.Timeout.Read = ingress.Upstream.TimeoutRead
+				}
+				if ingress.Upstream.TimeoutSend > 0 {
+					ups.Timeout.Send = ingress.Upstream.TimeoutSend
 				}
 				ctx.AddUpstream(ups)
 			}
@@ -306,8 +326,8 @@ func (t *translator) translateIngressV1beta1(ing *networkingv1beta1.Ingress, ski
 						return nil, err
 					}
 				}
-				if ingress.UpstreamScheme != "" {
-					ups.Scheme = ingress.UpstreamScheme
+				if ingress.Upstream.Scheme != "" {
+					ups.Scheme = ingress.Upstream.Scheme
 				}
 				ctx.AddUpstream(ups)
 			}
@@ -359,7 +379,26 @@ func (t *translator) translateIngressV1beta1(ing *networkingv1beta1.Ingress, ski
 			if len(ingress.Plugins) > 0 {
 				route.Plugins = *(ingress.Plugins.DeepCopy())
 			}
-
+			if ingress.Upstream.Retry > 0 {
+				retry := ingress.Upstream.Retry
+				ups.Retries = &retry
+			}
+			if ups.Timeout == nil {
+				ups.Timeout = &apisixv1.UpstreamTimeout{
+					Read:    60,
+					Send:    60,
+					Connect: 60,
+				}
+			}
+			if ingress.Upstream.TimeoutConnect > 0 {
+				ups.Timeout.Connect = ingress.Upstream.TimeoutConnect
+			}
+			if ingress.Upstream.TimeoutRead > 0 {
+				ups.Timeout.Read = ingress.Upstream.TimeoutRead
+			}
+			if ingress.Upstream.TimeoutSend > 0 {
+				ups.Timeout.Send = ingress.Upstream.TimeoutSend
+			}
 			if ingress.PluginConfigName != "" {
 				route.PluginConfigId = id.GenID(apisixv1.ComposePluginConfigName(ing.Namespace, ingress.PluginConfigName))
 			}
diff --git a/samples/deploy/crd/v1/ApisixClusterConfig.yaml b/samples/deploy/crd/v1/ApisixClusterConfig.yaml
index 355883e7..5b703260 100644
--- a/samples/deploy/crd/v1/ApisixClusterConfig.yaml
+++ b/samples/deploy/crd/v1/ApisixClusterConfig.yaml
@@ -73,3 +73,27 @@ spec:
                           type: number
                           minimum: 0.00001
                           maximum: 1
+            status:
+              type: object
+              properties:
+                observedGeneration:
+                  type: integer
+                conditions:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      type:
+                        type: string
+                      status:
+                        type: string
+                      lastTransitionTime:
+                        type: string
+                      reason:
+                        type: string
+                      message:
+                        type: string
+                  required:
+                    - type
+                    - status
+                    - lastTransitionTime
diff --git a/samples/deploy/crd/v1/ApisixConsumer.yaml b/samples/deploy/crd/v1/ApisixConsumer.yaml
index 1bcdd0a3..6ccc5897 100644
--- a/samples/deploy/crd/v1/ApisixConsumer.yaml
+++ b/samples/deploy/crd/v1/ApisixConsumer.yaml
@@ -224,4 +224,29 @@ spec:
                             name:
                               type: string
                               minLength: 1
-                          required: null
+                          required:
+                            - name
+            status:
+              type: object
+              properties:
+                observedGeneration:
+                  type: integer
+                conditions:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      type:
+                        type: string
+                      status:
+                        type: string
+                      lastTransitionTime:
+                        type: string
+                      reason:
+                        type: string
+                      message:
+                        type: string
+                  required:
+                    - type
+                    - status
+                    - lastTransitionTime



Test patch:
diff --git a/docs/en/latest/concepts/annotations.md b/docs/en/latest/concepts/annotations.md
index 26af6a21..a3292b1e 100644
--- a/docs/en/latest/concepts/annotations.md
+++ b/docs/en/latest/concepts/annotations.md
@@ -413,3 +413,63 @@ spec:
                 port:
                   number: 80
 ```
+
+## Upstream retries
+
+This annotation can be used to configure retries among multiple nodes in an upstream. You may want the proxy to retry when requests occur faults like transient network errors or service unavailable, By default the retry count is 1. You can change it by specifying the retries field.
+
+The following configuration configures the retries to 3, which indicates there'll be at most 3 requests sent to Kubernetes service httpbin's endpoints.
+
+One should bear in mind that passing a request to the next endpoint is only possible if nothing has been sent to a client yet. That is, if an error or timeout occurs in the middle of the transferring of a response, fixing this is impossible.
+
+```yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    k8s.apisix.apache.org/upstream-retries: "3"
+  name: ingress-ext-v1beta1
+spec:
+  ingressClassName: apisix
+  rules:
+    - host: httpbin.org
+      http:
+        paths:
+          - path: /ip
+            pathType: Exact
+            backend:
+              service:
+                name: httpbin
+                port:
+                  number: 80
+```
+
+## Upstream timeout
+
+This annotation can be used to configure different types of timeout on an upstream. The default connect, read and send timeout are 60s, which might not be proper for some applications.
+
+The below example sets the read, connect and send timeout to 5s, 10s, 10s respectively.
+
+```yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    k8s.apisix.apache.org/upstream-read-timeout.: "5s"
+    k8s.apisix.apache.org/upstream-connect-timeout: "10s"
+    k8s.apisix.apache.org/upstream-send-timeout: "10s"
+  name: ingress-ext-v1beta1
+spec:
+  ingressClassName: apisix
+  rules:
+    - host: httpbin.org
+      http:
+        paths:
+          - path: /ip
+            pathType: Exact
+            backend:
+              service:
+                name: httpbin
+                port:
+                  number: 80
+```
diff --git a/pkg/providers/ingress/translation/annotations/upstream/upstream_test.go b/pkg/providers/ingress/translation/annotations/upstream/upstream_test.go
new file mode 100644
index 00000000..4c998762
--- /dev/null
+++ b/pkg/providers/ingress/translation/annotations/upstream/upstream_test.go
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package upstream_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations"
+	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations/upstream"
+)
+
+func TestIPRestrictionHandler(t *testing.T) {
+	anno := map[string]string{
+		annotations.AnnotationsUpstreamScheme: "grpcs",
+	}
+	u := upstream.NewParser()
+
+	out, err := u.Parse(annotations.NewExtractor(anno))
+	ups, ok := out.(upstream.Upstream)
+	if !ok {
+		t.Fatalf("could not parse upstream")
+	}
+	assert.Nil(t, err, "checking given error")
+	assert.Equal(t, "grpcs", ups.Scheme)
+
+	anno[annotations.AnnotationsUpstreamScheme] = "gRPC"
+	out, err = u.Parse(annotations.NewExtractor(anno))
+	ups, ok = out.(upstream.Upstream)
+	if !ok {
+		t.Fatalf("could not parse upstream")
+	}
+	assert.Nil(t, err, "checking given error")
+	assert.Equal(t, "grpc", ups.Scheme)
+
+	anno[annotations.AnnotationsUpstreamScheme] = "nothing"
+	out, err = u.Parse(annotations.NewExtractor(anno))
+	assert.NotNil(t, err, "checking given error")
+	assert.Nil(t, out, "checking given output")
+}
+
+func TestRetryParsing(t *testing.T) {
+	anno := map[string]string{
+		annotations.AnnotationsUpstreamRetry: "2",
+	}
+	u := upstream.NewParser()
+	out, err := u.Parse(annotations.NewExtractor(anno))
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	ups, ok := out.(upstream.Upstream)
+	if !ok {
+		t.Fatalf("could not parse upstream")
+	}
+	assert.Nil(t, err, "checking given error")
+	assert.Equal(t, 2, ups.Retry)
+
+	anno[annotations.AnnotationsUpstreamRetry] = "asdf"
+	out, err = u.Parse(annotations.NewExtractor(anno))
+	assert.NotNil(t, err, "checking given error")
+}
+
+func TestTimeoutParsing(t *testing.T) {
+	anno := map[string]string{
+		annotations.AnnotationsUpstreamTimeoutConnect: "2s",
+		annotations.AnnotationsUpstreamTimeoutRead:    "3s",
+		annotations.AnnotationsUpstreamTimeoutSend:    "4s",
+	}
+	u := upstream.NewParser()
+	out, err := u.Parse(annotations.NewExtractor(anno))
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	ups, ok := out.(upstream.Upstream)
+	if !ok {
+		t.Fatalf("could not parse upstream")
+	}
+	assert.Nil(t, err, "checking given error")
+	assert.Equal(t, 2, ups.TimeoutConnect)
+	assert.Equal(t, 3, ups.TimeoutRead)
+	assert.Equal(t, 4, ups.TimeoutSend)
+	anno[annotations.AnnotationsUpstreamRetry] = "asdf"
+	out, err = u.Parse(annotations.NewExtractor(anno))
+	assert.NotNil(t, err, "checking given error")
+}
diff --git a/pkg/providers/ingress/translation/annotations/upstreamscheme/upstreamscheme_test.go b/pkg/providers/ingress/translation/annotations/upstreamscheme/upstreamscheme_test.go
deleted file mode 100644
index 74741fc4..00000000
--- a/pkg/providers/ingress/translation/annotations/upstreamscheme/upstreamscheme_test.go
+++ /dev/null
@@ -1,44 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//	http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package upstreamscheme
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/apache/apisix-ingress-controller/pkg/providers/ingress/translation/annotations"
-)
-
-func TestIPRestrictionHandler(t *testing.T) {
-	anno := map[string]string{
-		annotations.AnnotationsUpstreamScheme: "grpcs",
-	}
-	u := NewParser()
-
-	out, err := u.Parse(annotations.NewExtractor(anno))
-	assert.Nil(t, err, "checking given error")
-	assert.Equal(t, "grpcs", out)
-
-	anno[annotations.AnnotationsUpstreamScheme] = "gRPC"
-	out, err = u.Parse(annotations.NewExtractor(anno))
-	assert.Nil(t, err, "checking given error")
-	assert.Equal(t, "grpc", out)
-
-	anno[annotations.AnnotationsUpstreamScheme] = "nothing"
-	out, err = u.Parse(annotations.NewExtractor(anno))
-	assert.NotNil(t, err, "checking given error")
-	assert.Nil(t, out, "checking given output")
-}
diff --git a/test/e2e/scaffold/cluster_config.go b/test/e2e/scaffold/cluster_config.go
index 4a1b4a88..6f5e78f0 100644
--- a/test/e2e/scaffold/cluster_config.go
+++ b/test/e2e/scaffold/cluster_config.go
@@ -53,7 +53,10 @@ func (s *Scaffold) NewApisixClusterConfig(name string, enable bool, enablePrefer
 	if err := s.CreateResourceFromString(cc); err != nil {
 		return err
 	}
-	time.Sleep(5 * time.Second)
+	s.addFinalizers(func() {
+		_ = s.DeleteResourceFromString(cc)
+	})
+
 	return nil
 }
 
diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go
index 491a334b..97dce6fa 100644
--- a/test/e2e/scaffold/scaffold.go
+++ b/test/e2e/scaffold/scaffold.go
@@ -474,6 +474,7 @@ func (s *Scaffold) beforeEach() {
 		s.DeployAdminaAPIMode()
 	}
 	s.DeployTestService()
+	s.DeployRetryTimeout()
 }
 
 func (s *Scaffold) DeployAdminaAPIMode() {
@@ -520,6 +521,21 @@ func (s *Scaffold) DeployCompositeMode() {
 	assert.Nil(s.t, err, "creating apisix tunnels")
 }
 
+func (s *Scaffold) DeployRetryTimeout() {
+	//Two endpoints are blocking(10 second) and one is non blocking
+	//Testing timeout
+	//With 1 retry and a timeout of 5 sec, it should return 504(timeout)
+	//With 1 retry and a timeout of 15 sec, it should success
+
+	//Testing retry
+	//With 1 retry and a timeout of 5 sec, it should return 504(timeout)
+	//With 2 retry and a timeout of 5 sec, it should success
+	err := s.NewDeploymentForRetryTimeoutTest()
+	assert.Nil(s.t, err, "error creating deployments for retry and timeout")
+	err = s.NewServiceForRetryTimeoutTest()
+	assert.Nil(s.t, err, "error creating services for retry and timeout")
+}
+
 func (s *Scaffold) DeployTestService() {
 	var err error
 
diff --git a/test/e2e/scaffold/test_backend.go b/test/e2e/scaffold/test_backend.go
index f4eb8979..afa4975a 100644
--- a/test/e2e/scaffold/test_backend.go
+++ b/test/e2e/scaffold/test_backend.go
@@ -89,6 +89,68 @@ spec:
               name: "grpc-mtls"
               protocol: "TCP"
 `
+
+	_testTimeoutAndRetryDeploymentWithTimeout = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gobackend-deployment2
+spec:
+  replicas: 2  # You can adjust the number of replicas as needed
+  selector:
+    matchLabels:
+      app: gobackend
+  template:
+    metadata:
+      labels:
+        app: gobackend
+    spec:
+      containers:
+      - name: gobackend
+        imagePullPolicy: IfNotPresent
+        image: "127.0.0.1:5000/test-timeout:dev"
+        command: ["/app/gobackend", "fail"]
+        ports:
+        - containerPort: 9280
+`
+
+	_testTimeoutAndRetryDeploymentWithNoTimeout = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gobackend-deployment1
+spec:
+  replicas: 1  # You can adjust the number of replicas as needed
+  selector:
+    matchLabels:
+      app: gobackend
+  template:
+    metadata:
+      labels:
+        app: gobackend
+    spec:
+      containers:
+      - name: gobackend
+        imagePullPolicy: IfNotPresent
+        image: "127.0.0.1:5000/test-timeout:dev"
+        ports:
+        - containerPort: 9280
+`
+
+	_testTimeoutAndRetryService = `
+apiVersion: v1
+kind: Service
+metadata:
+  name: gobackend-service
+spec:
+  selector:
+    app: gobackend
+  ports:
+    - protocol: TCP
+      port: 9280  
+      targetPort: 9280
+`
+
 	_testBackendService = `
 apiVersion: v1
 kind: Service
@@ -172,6 +234,23 @@ spec:
 `
 )
 
+func (s *Scaffold) NewServiceForRetryTimeoutTest() error {
+	if err := s.CreateResourceFromString(_testTimeoutAndRetryService); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *Scaffold) NewDeploymentForRetryTimeoutTest() error {
+	if err := s.CreateResourceFromString(_testTimeoutAndRetryDeploymentWithTimeout); err != nil {
+		return err
+	}
+	if err := s.CreateResourceFromString(_testTimeoutAndRetryDeploymentWithNoTimeout); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (s *Scaffold) newTestBackend() (*corev1.Service, error) {
 	backendDeployment := fmt.Sprintf(s.FormatRegistry(_testBackendDeploymentTemplate), 1)
 	if err := s.CreateResourceFromString(backendDeployment); err != nil {
diff --git a/test/e2e/suite-annotations/upstreamretry.go b/test/e2e/suite-annotations/upstreamretry.go
new file mode 100644
index 00000000..768e2d75
--- /dev/null
+++ b/test/e2e/suite-annotations/upstreamretry.go
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package annotations
+
+import (
+	"net/http"
+	"time"
+
+	ginkgo "github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
+)
+
+var _ = ginkgo.Describe("suite-annotations: annotations.networking/v1 upstream", func() {
+	s := scaffold.NewDefaultScaffold()
+	ginkgo.It("Test timeout: 1 retry and long timeout", func() {
+		ing := `
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    k8s.apisix.apache.org/upstream-retries: "1"
+    k8s.apisix.apache.org/upstream-read-timeout: "20s"
+  name: ingress-ext-v1beta1
+spec:
+  ingressClassName: apisix
+  rules:
+    - host: e2e.apisix.local
+      http:
+        paths:
+         - path: /retry
+           pathType: Exact
+           backend:
+             service:
+               name: gobackend-service
+               port:
+                 number: 9280
+`
+		err := s.CreateResourceFromString(ing)
+		assert.Nil(ginkgo.GinkgoT(), err, "creating ingress")
+		time.Sleep(5 * time.Second)
+		err = s.EnsureNumApisixUpstreamsCreated(1)
+		assert.Nil(ginkgo.GinkgoT(), err, "checking upstreams")
+		time.Sleep(2 * time.Second)
+
+		respGet := s.NewAPISIXClient().GET("/retry").WithHeader("Host", "e2e.apisix.local").Expect()
+		respGet.Status(http.StatusOK)
+	})
+
+	ginkgo.It("Test retry: 2 retry and short timeout", func() {
+		ing := `
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    k8s.apisix.apache.org/upstream-retries: "2"
+    k8s.apisix.apache.org/upstream-read-timeout: "2s"
+  name: ingress-ext-v1beta1
+spec:
+  ingressClassName: apisix
+  rules:
+    - host: e2e.apisix.local
+      http:
+        paths:
+         - path: /retry
+           pathType: Exact
+           backend:
+             service:
+               name: gobackend-service
+               port:
+                 number: 9280
+`
+		err := s.CreateResourceFromString(ing)
+		assert.Nil(ginkgo.GinkgoT(), err, "creating ingress")
+		time.Sleep(5 * time.Second)
+		err = s.EnsureNumApisixUpstreamsCreated(1)
+		assert.Nil(ginkgo.GinkgoT(), err, "checking upstreams")
+		time.Sleep(2 * time.Second)
+
+		respGet := s.NewAPISIXClient().GET("/retry").WithHeader("Host", "e2e.apisix.local").Expect()
+		respGet.Status(http.StatusOK)
+	})
+})
diff --git a/test/e2e/suite-cluster/status.go b/test/e2e/suite-cluster/status.go
new file mode 100644
index 00000000..0aac5b74
--- /dev/null
+++ b/test/e2e/suite-cluster/status.go
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package cluster
+
+import (
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
+)
+
+var _ = ginkgo.Describe("suite-cluster: CRDs status subresource Testing", func() {
+	s := scaffold.NewDefaultScaffold()
+
+	ginkgo.It("check ApisixClusterConfig status is recorded", func() {
+		// create ApisixClusterConfig resource
+		assert.Nil(ginkgo.GinkgoT(), s.NewApisixClusterConfig("default", true, true), "create cluster config error")
+		time.Sleep(6 * time.Second)
+
+		// status should be recorded as successful
+		output, err := s.GetOutputFromString("acc", "default", "-o", "yaml")
+		assert.Nil(ginkgo.GinkgoT(), err, "Get output of ApisixClusterConfig resource")
+		assert.Contains(ginkgo.GinkgoT(), output, "type: ResourcesAvailable", "status.conditions.type is recorded")
+		assert.Contains(ginkgo.GinkgoT(), output, "reason: ResourcesSynced", "status.conditions.reason  is recorded")
+		assert.Contains(ginkgo.GinkgoT(), output, `status: "True"`, "status.conditions.status  is recorded")
+		assert.Contains(ginkgo.GinkgoT(), output, "message: Sync Successfully", "status.conditions.message  is recorded")
+	})
+})
diff --git a/test/e2e/suite-ingress/suite-ingress-features/status.go b/test/e2e/suite-ingress/suite-ingress-features/status.go
index 5c9513f2..a51fa5ab 100644
--- a/test/e2e/suite-ingress/suite-ingress-features/status.go
+++ b/test/e2e/suite-ingress/suite-ingress-features/status.go
@@ -123,7 +123,7 @@ spec:
 `
 		assert.Nil(ginkgo.GinkgoT(), s.CreateVersionedApisixResource(apc))
 		time.Sleep(6 * time.Second)
-		// status should be recorded as successfulen
+		// status should be recorded as successful
 		output, err := s.GetOutputFromString("apc", "test-apc", "-o", "yaml")
 		assert.Nil(ginkgo.GinkgoT(), err, "Get output of ApisixPluginConfig resource")
 		assert.Contains(ginkgo.GinkgoT(), output, "type: ResourcesAvailable", "status.conditions.type is recorded")
@@ -213,7 +213,7 @@ wrw7im4TNSAdwVX4Y1F4svJ2as5SJn5QYGAzXDixNuwzXYrpP9rzA2s=
 		host := "api6.com"
 		assert.Nil(ginkgo.GinkgoT(), s.NewApisixTls(tlsName, host, secretName), "create tls error")
 		time.Sleep(6 * time.Second)
-		// status should be recorded as successfulen
+		// status should be recorded as successful
 		output, err := s.GetOutputFromString("atls", tlsName, "-o", "yaml")
 		assert.Nil(ginkgo.GinkgoT(), err, "Get output of ApisixPluginConfig resource")
 		assert.Contains(ginkgo.GinkgoT(), output, "type: ResourcesAvailable", "status.conditions.type is recorded")
@@ -233,8 +233,19 @@ wrw7im4TNSAdwVX4Y1F4svJ2as5SJn5QYGAzXDixNuwzXYrpP9rzA2s=
 	})
 
 	//TODO: ApisixGlobal
-	//TODO: ApisixConsumer  CRD missing status definition
-	//TODO: ApisixClusterConfig CRD missing status definition
+	ginkgo.It("check ApisixConsumer status is recorded", func() {
+		// create ApisixConsumer resource
+		assert.Nil(ginkgo.GinkgoT(), s.ApisixConsumerBasicAuthCreated("test-apisix-consumer", "foo", "bar"), "create consumer error")
+		time.Sleep(6 * time.Second)
+
+		// status should be recorded as successful
+		output, err := s.GetOutputFromString("ac", "test-apisix-consumer", "-o", "yaml")
+		assert.Nil(ginkgo.GinkgoT(), err, "Get output of ApisixConsumer resource")
+		assert.Contains(ginkgo.GinkgoT(), output, "type: ResourcesAvailable", "status.conditions.type is recorded")
+		assert.Contains(ginkgo.GinkgoT(), output, "reason: ResourcesSynced", "status.conditions.reason  is recorded")
+		assert.Contains(ginkgo.GinkgoT(), output, `status: "True"`, "status.conditions.status  is recorded")
+		assert.Contains(ginkgo.GinkgoT(), output, "message: Sync Successfully", "status.conditions.message  is recorded")
+	})
 })
 
 var _ = ginkgo.Describe("suite-ingress-features: Ingress LB Status Testing", func() {
diff --git a/test/e2e/testdata/apisix-gw-config-with-sd.yaml b/test/e2e/testdata/apisix-gw-config-with-sd.yaml
index 8d1ef5d0..bd95f34a 100644
--- a/test/e2e/testdata/apisix-gw-config-with-sd.yaml
+++ b/test/e2e/testdata/apisix-gw-config-with-sd.yaml
@@ -48,4 +48,4 @@ discovery:
     servers:
       - "10.96.0.10:53"          # use the real address of your dns server.
                                  # currently we use KIND as the standard test environment, so here we can hard-code the default DNS address first.
-                                 # TODO: can be modified to fill dynamically 
+                                 # TODO: can be modified to fill dynamically 
\ No newline at end of file
diff --git a/test/e2e/testdata/apisix-stream-disabled.yaml b/test/e2e/testdata/apisix-stream-disabled.yaml
index 458b707a..00ce34ae 100644
--- a/test/e2e/testdata/apisix-stream-disabled.yaml
+++ b/test/e2e/testdata/apisix-stream-disabled.yaml
@@ -39,4 +39,4 @@ etcd:
   timeout: 30                     # 30 seconds
 plugin_attr:
   prometheus:
-    enable_export_server: false
+    enable_export_server: false
\ No newline at end of file
diff --git a/test/e2e/testdata/ldap/docker-compose.yaml b/test/e2e/testdata/ldap/docker-compose.yaml
index 364aef39..20e019c2 100644
--- a/test/e2e/testdata/ldap/docker-compose.yaml
+++ b/test/e2e/testdata/ldap/docker-compose.yaml
@@ -30,4 +30,4 @@ services:
       - LDAP_ADMIN_PASSWORD=admin
       - LDAP_ROOT=dc=ldap,dc=example,dc=org
       - LDAP_USERS=jack
-      - LDAP_PASSWORDS=jackPassword
+      - LDAP_PASSWORDS=jackPassword
\ No newline at end of file
diff --git a/test/e2e/testdata/wolf-rbac/cmd.sh b/test/e2e/testdata/wolf-rbac/cmd.sh
index 13aff671..c5e3328d 100755
--- a/test/e2e/testdata/wolf-rbac/cmd.sh
+++ b/test/e2e/testdata/wolf-rbac/cmd.sh
@@ -82,4 +82,4 @@ elif [ $OPTION = "stop" ]; then
     rm -rf db-psql.sql
 else
     echo "argument is one of [ip, start, stop]"
-fi
+fi
\ No newline at end of file
diff --git a/test/e2e/testdata/wolf-rbac/docker-compose.yaml b/test/e2e/testdata/wolf-rbac/docker-compose.yaml
index 3835589a..0be212ce 100644
--- a/test/e2e/testdata/wolf-rbac/docker-compose.yaml
+++ b/test/e2e/testdata/wolf-rbac/docker-compose.yaml
@@ -74,4 +74,4 @@ services:
       RBAC_SERVER_URL: http://server:12180
       RBAC_APP_ID: restful-demo
       AGENT_PORT: 12184
-      EXTENSION_CONFIG: include /opt/wolf/agent/conf/no-permission-demo.conf;
+      EXTENSION_CONFIG: include /opt/wolf/agent/conf/no-permission-demo.conf;
\ No newline at end of file
diff --git a/test/e2e/testtimeout/Dockerfile b/test/e2e/testtimeout/Dockerfile
new file mode 100644
index 00000000..1e4aec78
--- /dev/null
+++ b/test/e2e/testtimeout/Dockerfile
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM golang:latest
+
+WORKDIR /app
+
+COPY . .
+
+RUN go build -o gobackend .
+
+EXPOSE 9280
+
+ENTRYPOINT [ "/app/gobackend" ]
\ No newline at end of file
diff --git a/test/e2e/testtimeout/go.mod b/test/e2e/testtimeout/go.mod
new file mode 100644
index 00000000..6840ad39
--- /dev/null
+++ b/test/e2e/testtimeout/go.mod
@@ -0,0 +1,3 @@
+module github.com/apache/apisix-ingress-controller/test/e2e/testtimeout
+
+go 1.17
diff --git a/test/e2e/testtimeout/main.go b/test/e2e/testtimeout/main.go
new file mode 100644
index 00000000..2b95c28a
--- /dev/null
+++ b/test/e2e/testtimeout/main.go
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+)
+
+type Server struct {
+	timeout int
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	if s.timeout == 0 {
+		w.WriteHeader(200)
+		fmt.Println("PASSING")
+		return
+	}
+	time.Sleep(time.Duration(s.timeout * int(time.Second)))
+	fmt.Println("served after waiting")
+	return
+}
+
+func main() {
+	fmt.Println("starting server...")
+	var s Server
+	if len(os.Args) > 1 && os.Args[1] == "fail" {
+		s.timeout = 10
+	} else {
+		s.timeout = 0
+	}
+	http.Handle("/retry", &s)
+	http.ListenAndServe(":9280", nil)
+}