Repository: avenga/couper
Pull request #339 (2021-10-13 11:09:17)

Backend health
This PR introduces a `health` block for defined backends to check for their current state.

---
<details>
    <summary>Reviewer checklist</summary>
    <ul>
        <li>Read PR description: a summary about the changes is required</li>
        <li>Changelog updated</li>
        <li>Documentation: docs/{Reference, Cli, ...}, Docker and cli help/usage</li>
        <li>Pulled branch, manually tested</li>
        <li>Verified requirements are met</li>
        <li>Reviewed the code</li>
        <li>Reviewed the related tests</li>
    </ul>
</details>



Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fbf50e0a..0bd257ab 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,8 @@ Unreleased changes are available as `avenga/couper:edge` container.
 * **Added**
   * Couper [reads and merges configuration files](./docs/CLI.md#global-options) from a given directory ([#437](https://github.com/avenga/couper/pull/437))
     * provided via `-d` command-line flag or `COUPER_FILE_DIRECTORY` environment variable
+  * `beta_health`-block to `backend`-block to enable continuous health-checks for defined backends ([#313](https://github.com/avenga/couper/pull/313))
+    * `backends.<name>.health` variable to access the current health-check state _(subject to change)_
 
 * **Fixed**
   * Keys in object type attribute values are only handled case-insensitively if reasonable (e.g. they represent HTTP methods or header field values) ([#461](https://github.com/avenga/couper/pull/461))
diff --git a/config/backend.go b/config/backend.go
index 912323b8..208fb470 100644
--- a/config/backend.go
+++ b/config/backend.go
@@ -18,6 +18,7 @@ var (
 type Backend struct {
 	DisableCertValidation  bool     `hcl:"disable_certificate_validation,optional"`
 	DisableConnectionReuse bool     `hcl:"disable_connection_reuse,optional"`
+	Health                 *Health  `hcl:"beta_health,block"`
 	HTTP2                  bool     `hcl:"http2,optional"`
 	MaxConnections         int      `hcl:"max_connections,optional"`
 	Name                   string   `hcl:"name,label"`
diff --git a/config/health_check.go b/config/health_check.go
new file mode 100644
index 00000000..a0370fa0
--- /dev/null
+++ b/config/health_check.go
@@ -0,0 +1,104 @@
+package config
+
+import (
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/hashicorp/hcl/v2"
+
+	"github.com/avenga/couper/utils"
+)
+
+var defaultHealthCheck = &HealthCheck{
+	FailureThreshold: 2,
+	Interval:         time.Second,
+	Timeout:          time.Second,
+	ExpectStatus:     map[int]bool{200: true, 204: true, 301: true},
+	ExpectText:       "",
+}
+
+type HealthCheck struct {
+	FailureThreshold uint
+	Interval         time.Duration
+	Timeout          time.Duration
+	ExpectStatus     map[int]bool
+	ExpectText       string
+	Request          *http.Request
+	RequestUIDFormat string
+}
+
+type Headers map[string]string
+
+type Health struct {
+	FailureThreshold uint     `hcl:"failure_threshold,optional"`
+	Interval         string   `hcl:"interval,optional"`
+	Timeout          string   `hcl:"timeout,optional"`
+	Path             string   `hcl:"path,optional"`
+	ExpectStatus     int      `hcl:"expect_status,optional"`
+	ExpectText       string   `hcl:"expect_text,optional"`
+	Headers          Headers  `hcl:"headers,optional"`
+	Remain           hcl.Body `hcl:",remain"`
+}
+
+func NewHealthCheck(baseURL string, options *Health, settings *Settings) (*HealthCheck, error) {
+	healthCheck := *defaultHealthCheck
+	healthCheck.RequestUIDFormat = settings.RequestIDFormat
+
+	var err error
+	if options != nil {
+		if options.Interval != "" {
+			healthCheck.Interval, err = time.ParseDuration(options.Interval)
+			if err != nil {
+				return nil, err
+			}
+			healthCheck.Timeout = healthCheck.Interval
+		}
+		if options.Timeout != "" {
+			healthCheck.Timeout, err = time.ParseDuration(options.Timeout)
+			if err != nil {
+				return nil, err
+			}
+		}
+		if healthCheck.Timeout > healthCheck.Interval {
+			healthCheck.Timeout = healthCheck.Interval
+		}
+		if options.FailureThreshold != 0 {
+			healthCheck.FailureThreshold = options.FailureThreshold
+		}
+		if options.ExpectStatus != 0 {
+			healthCheck.ExpectStatus = map[int]bool{options.ExpectStatus: true}
+		}
+		healthCheck.ExpectText = options.ExpectText
+
+		request, err := http.NewRequest(http.MethodGet, baseURL, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		if options.Path != "" {
+			request.URL = request.URL.ResolveReference(createURL(options.Path))
+		}
+
+		if options.Headers != nil {
+			for key, value := range options.Headers {
+				request.Header.Add(key, value)
+			}
+		}
+
+		if ua := request.Header.Get("User-Agent"); ua == "" {
+			request.Header.Set("User-Agent", "Couper / "+utils.VersionName+" health-check")
+		}
+
+		healthCheck.Request = request
+	}
+	return &healthCheck, err
+}
+
+func createURL(pathQuery string) *url.URL {
+	uri, _ := url.ParseRequestURI("http://HOST/../" + pathQuery)
+	uri.Scheme = ""
+	uri.Host = ""
+	u := url.URL{}
+	return u.ResolveReference(uri)
+}
diff --git a/config/runtime/backend.go b/config/runtime/backend.go
index 14c457f1..ef968cb2 100644
--- a/config/runtime/backend.go
+++ b/config/runtime/backend.go
@@ -1,12 +1,14 @@
 package runtime
 
 import (
+	"fmt"
 	"math"
 	"net/http"
 
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/gohcl"
 	"github.com/sirupsen/logrus"
+	"github.com/zclconf/go-cty/cty"
 
 	"github.com/avenga/couper/backend"
 	"github.com/avenga/couper/cache"
@@ -69,13 +71,27 @@ func newBackend(evalCtx *hcl.EvalContext, backendCtx hcl.Body, log *logrus.Entry
 		MaxConnections:         beConf.MaxConnections,
 	}
 
-	openAPIopts, err := validation.NewOpenAPIOptions(beConf.OpenAPI)
+	options := &transport.BackendOptions{}
+	var err error
+	options.OpenAPI, err = validation.NewOpenAPIOptions(beConf.OpenAPI)
 	if err != nil {
 		return nil, err
 	}
 
-	options := &transport.BackendOptions{
-		OpenAPI: openAPIopts,
+	if beConf.Health != nil {
+		origin, diags := eval.ValueFromBodyAttribute(evalCtx, backendCtx, "origin")
+		if diags != nil {
+			return nil, diags
+		}
+
+		if origin == cty.NilVal {
+			return nil, fmt.Errorf("missing origin for backend %q", beConf.Name)
+		}
+
+		options.HealthCheck, err = config.NewHealthCheck(origin.AsString(), beConf.Health, settings)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	oauthContent, _, _ := backendCtx.PartialContent(config.OAuthBlockSchema)
diff --git a/docs/LOGS.md b/docs/LOGS.md
index 3f8d74b2..dc8134c3 100644
--- a/docs/LOGS.md
+++ b/docs/LOGS.md
@@ -77,7 +77,7 @@ These fields are found in the [Log Types](#log-types) `couper_access` and `coupe
 | `"timings":`  |             | field regarding timing                                                                                                            |
 |               | `{`         |                                                                                                                                   |
 |               | `"total"`   | total time taken                                                                                                                  |
-|               | `)`         |                                                                                                                                   |
+|               | `}`         |                                                                                                                                   |
 | `“uid"`       |             | unique request id configurable in [Settings](./REFERENCE.md#settings-block)                                                       |
 | `"url"`       |             | complete url (`<proto>://<host>:<port><path>` or `<origin><path>`)                                                                |
 
@@ -117,7 +117,7 @@ These fields are found in the [Log Type](#log-types) `couper_backend` in additio
 |                         | `"tls"`     | time taken between attempt and success at tls handshake                                                                           |
 |                         | `"total"`   | total time taken                                                                                                                  |
 |                         | `"ttfb"`    | time to first byte/between establishing connection and receiving first byte                                                       |
-|                         | `)`         |                                                                                                                                   |
+|                         | `}`         |                                                                                                                                   |
 | `"token_request"`       |             | entry regarding request for token                                                                                                 |
 | `"token_request_retry"` |             | how many `token_request` attempts were made                                                                                       |
 | `"uid"`                 |             | unique request id configurable in [Settings](./REFERENCE.md#settings-block)                                                       |
@@ -136,7 +136,7 @@ These fields are found in the [Log Type](#log-types) `couper_daemon` in addition
 |              | `{`             |                                                                                                                            |
 |              | `"max-retries"` | maximum retry count, see [Global Options](./CLI.md#global-options)                                                         |
 |              | `"retry-delay"` | configured delay of each retry, see [Global Options](./CLI.md#global-options)                                              |
-|              | `)`             |                                                                                                                            |
+|              | `}`             |                                                                                                                            |
 
 ## Custom Logging
 
diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md
index 8c870c0b..9e44a887 100644
--- a/docs/REFERENCE.md
+++ b/docs/REFERENCE.md
@@ -26,6 +26,7 @@
     - [SAML Block](#saml-block)
     - [Settings Block](#settings-block)
     - [Defaults Block](#defaults-block)
+    - [Health Block (Beta)](#health-block)
     - [Error Handler Block](#error-handler-block)
   - [Access Control](#access-control)
   - [Health-Check](#health-check)
@@ -37,6 +38,7 @@
     - [backend_requests](#backend_requests)
     - [backend_response](#backend_response)
     - [backend_responses](#backend_responses)
+    - [backends](#backends)
   - [Functions](#functions)
   - [Modifiers](#modifiers)
     - [Request Header](#request-header)
@@ -173,8 +175,7 @@ The `request` block creates and executes a request to a backend service.
 
 | Block name | Context                           | Label                                                                                                                                                                                                                                                                      | Nested block(s)                                                                                                             |
 |:-----------|:----------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------|
-| `request`  | [Endpoint Block](#endpoint-block) | &#9888; A [Proxy Block](#proxy-block) or [Request Block](#request-block) w/o a label has an implicit label `"default"`. Only **one** [Proxy Block](#proxy-block) or [Request Block](#request-block) w/ label `"default"` per [Endpoint Block](#endpoint-block) is allowed. | [Backend Block](#backend-block) (&#9888; required, if no `backend` block reference is defined or no `url` attribute is set. |
-
+| `request`  | [Endpoint Block](#endpoint-block)  |&#9888; A [Proxy Block](#proxy-block) or [Request Block](#request-block) w/o a label has an implicit label `"default"`. Only **one** [Proxy Block](#proxy-block) or [Request Block](#request-block) w/ label `"default"` per [Endpoint Block](#endpoint-block) is allowed.|[Backend Block](#backend-block) (&#9888; required, if no `backend` block reference is defined or no `url` attribute is set.|
 <!-- TODO: add available http methods -->
 
 | Attribute(s)      | Type                                  | Default | Description                                                                                                                                                                                                                                                                                      | Characteristic(s)                                                                                                                                                                      | Example           |
@@ -212,7 +213,7 @@ The `backend` block defines the connection to a local/remote backend service.
 
 |Block name|Context|Label|Nested block(s)|
 | :----------| :-----------| :-----------| :-----------|
-|`backend`| [Definitions Block](#definitions-block), [Proxy Block](#proxy-block), [Request Block](#request-block), [OAuth2 CC Block](#oauth2-block), [JWT Block](#jwt-block), [OAuth2 AC Block (beta)](#beta-oauth2-block), [OIDC Block](#oidc-block)| &#9888; required, when defined in [Definitions Block](#definitions-block)| [OpenAPI Block](#openapi-block), [OAuth2 CC Block](#oauth2-block)|
+|`backend`| [Definitions Block](#definitions-block), [Proxy Block](#proxy-block), [Request Block](#request-block), [OAuth2 CC Block](#oauth2-block), [JWT Block](#jwt-block), [OAuth2 AC Block (beta)](#beta-oauth2-block), [OIDC Block](#oidc-block)| &#9888; required, when defined in [Definitions Block](#definitions-block)| [OpenAPI Block](#openapi-block), [OAuth2 CC Block](#oauth2-block), [Health Block](#health-block)|
 
 | Attribute(s) | Type |Default|Description|Characteristic(s)| Example|
 | :------------------------------ | :--------------- | :--------------- | :--------------- | :--------------- | :--------------- |
@@ -527,13 +528,33 @@ Some information from the assertion consumed at the ACS endpoint is provided in
 - the session expiry date `SessionNotOnOrAfter` (as UNIX timestamp: `request.context.<label>.exp`)
 - the attributes (`request.context.<label>.attributes.<name>`)
 
+### Health Block
+
+Defines a recurring health check request for its backend. Results can be obtained via the [`backends.<label>.health` variables](#backends).
+Changes in health states and related requests will be logged. Default User-Agent will be `Couper / <version> health-check` if not provided
+via `headers` attribute.
+
+| Block name    | Context                           | Label | Nested block |
+|:--------------|:----------------------------------|:------|:-------------|
+| `beta_health` | [`backend` block](#backend-block) | –     |              |
+
+| Attributes          | Type                  | Default             | Description                                        | Characteristics       | Example                            |
+|:--------------------|:----------------------|:--------------------|:---------------------------------------------------|:----------------------|:-----------------------------------|
+| `expect_status`     | number                | `200`, `204`, `301` | wanted response status code                        |                       | `expect_status =  418`             |
+| `expect_text`       | string                | –                   | text response body must contain                    |                       | `expect_text = alive`              |
+| `failure_threshold` | number                | `2`                 | failed checks needed to consider backend unhealthy |                       | `failure_threshold = 3`            |
+| `headers`           | map                   | –                   | request headers                                    |                       | `headers = {User-Agent: "health"}` |
+| `interval`          | [duration](#duration) | `"2s"`              | time interval for recheck                          |                       | `timeout = "5s"`                   |
+| `path`              | string                | –                   | URL path/query on backend host                     |                       | `path = "/health"`                 |
+| `timeout`           | [duration](#duration) | `"2s"`              | maximum allowed time limit                         | bounded by `interval` | `timeout = "3s"`                   |
+
 ### Settings Block
 
 The `settings` block lets you configure the more basic and global behavior of your
 gateway instance.
 
 | Context | Label    | Nested block(s) |
-| :------ | :------- | :-------------- |
+|:--------|:---------|:----------------|
 | -       | no label | -               |
 
 | Attribute(s)                    | Type   | Default             | Description | Characteristic(s) | Example |
@@ -689,6 +710,15 @@ and for OIDC additionally:
 - `id_token_claims`: a map of claims from the ID token
 - `userinfo`: a map of claims retrieved from the userinfo endpoint
 
+
+### `backends`
+
+`backends.<label>` allows access to backend information.
+
+| Variable | Type   | Description                           | Example                                              |
+|:---------|:-------|:--------------------------------------|:-----------------------------------------------------|
+| `health` | object | current [health state](#health-block) | `{"error": "", "healthy": true, "state": "healthy"}` |
+
 ### `backend_request`
 
 `backend_request` holds information about the current backend request. It is only
diff --git a/eval/context.go b/eval/context.go
index f9d86d80..ff3ebf98 100644
--- a/eval/context.go
+++ b/eval/context.go
@@ -3,6 +3,7 @@ package eval
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"mime"
 	"net/http"
@@ -23,6 +24,7 @@ import (
 	"github.com/avenga/couper/config/env"
 	"github.com/avenga/couper/config/request"
 	"github.com/avenga/couper/eval/lib"
+	"github.com/avenga/couper/handler/transport/probe_map"
 	"github.com/avenga/couper/internal/seetie"
 	"github.com/avenga/couper/oauth2/oidc"
 	"github.com/avenga/couper/utils"
@@ -158,6 +160,21 @@ func (c *Context) WithClientRequest(req *http.Request) *Context {
 	ctx.eval.Variables[BackendRequests] = cty.ObjectVal(make(map[string]cty.Value))
 	ctx.eval.Variables[BackendResponses] = cty.ObjectVal(make(map[string]cty.Value))
 
+	backendsVariable := map[string]interface{}{}
+	probe_map.BackendProbes.Range(func(backendName, value interface{}) bool {
+		health := value.(probe_map.HealthInfo)
+		backendsVariable[fmt.Sprint(backendName)] = map[string]interface{}{
+			Health: map[string]interface{}{
+				"healthy": health.Healthy,
+				"error":   health.Error,
+				"state":   health.State,
+			},
+		}
+		return true
+	})
+
+	ctx.eval.Variables[Backends] = seetie.MapToValue(backendsVariable)
+
 	ctx.updateRequestRelatedFunctions(origin)
 	ctx.updateFunctions()
 
diff --git a/eval/variables.go b/eval/variables.go
index 3e1f3e65..a13ba0e6 100644
--- a/eval/variables.go
+++ b/eval/variables.go
@@ -1,11 +1,13 @@
 package eval
 
 const (
+	Backends         = "backends"
 	BackendRequest   = "backend_request"
 	BackendRequests  = "backend_requests"
 	BackendResponse  = "backend_response"
 	BackendResponses = "backend_responses"
 	BackendDefault   = "default"
+	State            = "state"
 	Body             = "body"
 	ClientRequest    = "request"
 	CTX              = "context"
@@ -14,6 +16,7 @@ const (
 	Environment      = "env"
 	FormBody         = "form_body"
 	Headers          = "headers"
+	Health           = "health"
 	HttpStatus       = "status"
 	ID               = "id"
 	JsonBody         = "json_body"
diff --git a/handler/transport/backend.go b/handler/transport/backend.go
index 7396b583..0d63cb35 100644
--- a/handler/transport/backend.go
+++ b/handler/transport/backend.go
@@ -53,30 +53,34 @@ type Backend struct {
 
 // NewBackend creates a new <*Backend> object by the given <*Config>.
 func NewBackend(ctx hcl.Body, tc *Config, opts *BackendOptions, log *logrus.Entry) http.RoundTripper {
-	var logEntry *logrus.Entry
+	var (
+		healthCheck  *config.HealthCheck
+		openAPI      *validation.OpenAPI
+		tokenRequest TokenRequest
+	)
 
-	if tc.BackendName != "" {
-		logEntry = log.WithField("backend", tc.BackendName)
-	} else {
-		logEntry = log.WithField("backend", "default")
-	}
-
-	var openAPI *validation.OpenAPI
-	var tr TokenRequest
 	if opts != nil {
+		healthCheck = opts.HealthCheck
 		openAPI = validation.NewOpenAPI(opts.OpenAPI)
-		tr = opts.AuthBackend
+		tokenRequest = opts.AuthBackend
 	}
 
 	backend := &Backend{
 		context:          ctx,
-		logEntry:         logEntry,
+		logEntry:         log.WithField("backend", tc.BackendName),
 		name:             tc.BackendName,
 		openAPIValidator: openAPI,
-		tokenRequest:     tr,
+		tokenRequest:     tokenRequest,
 		transportConf:    tc,
 	}
-	backend.upstreamLog = logging.NewUpstreamLog(logEntry, backend, tc.NoProxyFromEnv)
+
+	backend.upstreamLog = logging.NewUpstreamLog(backend.logEntry, backend, tc.NoProxyFromEnv)
+
+	distinct := !strings.HasPrefix(tc.Hostname, "anonymous_")
+	if distinct && healthCheck != nil {
+		NewProbe(backend.logEntry, tc.BackendName, healthCheck)
+	}
+
 	return backend.upstreamLog
 }
 
diff --git a/handler/transport/backend_options.go b/handler/transport/backend_options.go
index 021d9ccd..654a1533 100644
--- a/handler/transport/backend_options.go
+++ b/handler/transport/backend_options.go
@@ -3,13 +3,15 @@ package transport
 import (
 	"net/http"
 
+	"github.com/avenga/couper/config"
 	"github.com/avenga/couper/handler/validation"
 )
 
 // BackendOptions represents the transport <BackendOptions> object.
 type BackendOptions struct {
-	OpenAPI     *validation.OpenAPIOptions
 	AuthBackend TokenRequest
+	HealthCheck *config.HealthCheck
+	OpenAPI     *validation.OpenAPIOptions
 }
 
 type TokenRequest interface {
diff --git a/handler/transport/probe.go b/handler/transport/probe.go
new file mode 100644
index 00000000..8dd66204
--- /dev/null
+++ b/handler/transport/probe.go
@@ -0,0 +1,170 @@
+package transport
+
+import (
+	"context"
+	goerror "errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/avenga/couper/config"
+	"github.com/avenga/couper/config/request"
+	"github.com/avenga/couper/errors"
+	"github.com/avenga/couper/eval"
+	"github.com/avenga/couper/handler/middleware"
+	probe "github.com/avenga/couper/handler/transport/probe_map"
+	"github.com/avenga/couper/logging"
+)
+
+const (
+	StateInvalid state = iota
+	StateOk
+	StateFailing
+	StateDown
+)
+
+var healthStateLabels = []string{
+	"invalid",
+	"healthy",
+	"failing",
+	"unhealthy",
+}
+
+var _ context.Context = &eval.Context{}
+
+type state int
+
+func (s state) String() string {
+	return healthStateLabels[s]
+}
+
+type Probe struct {
+	//configurable settings
+	backendName string
+	log         *logrus.Entry
+	opts        *config.HealthCheck
+
+	//variables reflecting status of probe
+	client  *http.Client
+	counter uint
+	failure uint
+	state   state
+	status  int
+
+	uidFunc middleware.UIDFunc
+}
+
+func NewProbe(log *logrus.Entry, backendName string, opts *config.HealthCheck) {
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Transport: logging.NewUpstreamLog(log, http.DefaultTransport,
+			false), // always false due to defaultTransport
+	}
+
+	p := &Probe{
+		backendName: backendName,
+		log:         log,
+		opts:        opts,
+
+		client: client,
+		state:  StateInvalid,
+
+		uidFunc: middleware.NewUIDFunc(opts.RequestUIDFormat),
+	}
+
+	go p.probe()
+}
+
+func (p *Probe) probe() {
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), p.opts.Timeout)
+		ctx = context.WithValue(ctx, request.RoundTripName, "health-check")
+		uid := p.uidFunc()
+		ctx = context.WithValue(ctx, request.UID, uid)
+
+		res, err := p.client.Do(p.opts.Request.Clone(ctx))
+		cancel()
+
+		p.counter++
+		prevState := p.state
+		p.status = 0
+		if res != nil {
+			p.status = res.StatusCode
+		}
+
+		var errorMessage string
+		if err != nil || !p.opts.ExpectStatus[res.StatusCode] || !contains(res.Body, p.opts.ExpectText) {
+			if p.failure++; p.failure < p.opts.FailureThreshold {
+				p.state = StateFailing
+			} else {
+				p.state = StateDown
+			}
+			if err == nil {
+				if !p.opts.ExpectStatus[res.StatusCode] {
+					errorMessage = "unexpected statusCode: " + strconv.Itoa(p.status)
+				} else {
+					errorMessage = "unexpected text"
+				}
+			} else {
+				unwrapped := goerror.Unwrap(err)
+				if unwrapped != nil {
+					err = unwrapped
+				}
+				if gerr, ok := err.(errors.GoError); ok {
+					errorMessage = gerr.LogError()
+				} else {
+					errorMessage = err.Error()
+				}
+			}
+		} else {
+			p.failure = 0
+			p.state = StateOk
+			errorMessage = ""
+		}
+
+		if prevState != p.state {
+			newState := p.state.String()
+			probe.BackendProbes.Store(p.backendName, probe.HealthInfo{
+				State:   newState,
+				Error:   errorMessage,
+				Healthy: p.state != StateDown,
+			})
+
+			message := fmt.Sprintf("new health state: %s", newState)
+
+			log := p.log.WithField("uid", uid)
+			switch p.state {
+			case StateOk:
+				log.Info(message)
+			case StateFailing:
+				log.Warn(message)
+			case StateDown:
+				log.WithError(fmt.Errorf(errorMessage + ": " + message)).Error()
+			}
+		}
+
+		time.Sleep(p.opts.Interval)
+	}
+}
+
+func (p Probe) String() string {
+	return fmt.Sprintf("check #%d for backend %q: state: %s (%d/%d), HTTP status: %d", p.counter, p.backendName, p.state, p.failure, p.opts.FailureThreshold, p.status)
+}
+
+func contains(reader io.ReadCloser, text string) bool {
+	defer reader.Close() // free resp body related connection
+
+	if text == "" {
+		return true
+	}
+
+	bytes, _ := io.ReadAll(reader)
+	return strings.Contains(string(bytes), text)
+}
diff --git a/handler/transport/probe_map/probe_map.go b/handler/transport/probe_map/probe_map.go
new file mode 100644
index 00000000..47022f57
--- /dev/null
+++ b/handler/transport/probe_map/probe_map.go
@@ -0,0 +1,11 @@
+package probe_map
+
+import "sync"
+
+var BackendProbes sync.Map
+
+type HealthInfo struct {
+	State   string
+	Healthy bool
+	Error   string
+}


Test patch:
diff --git a/config/configload/load_test.go b/config/configload/load_test.go
index a4c64e69..9a734a94 100644
--- a/config/configload/load_test.go
+++ b/config/configload/load_test.go
@@ -1,11 +1,15 @@
 package configload_test
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"testing"
 
+	"github.com/avenga/couper/cache"
 	"github.com/avenga/couper/config/configload"
+	"github.com/avenga/couper/config/runtime"
+	"github.com/avenga/couper/internal/test"
 )
 
 func TestPrepareBackendRefineAttributes(t *testing.T) {
@@ -70,3 +74,84 @@ definitions {
 		t.Error(err)
 	}
 }
+
+func TestHealthCheck(t *testing.T) {
+	tests := []struct {
+		name  string
+		hcl   string
+		error string
+	}{
+		{
+			"Bad interval",
+			`interval = "10sec"`,
+			`time: unknown unit "sec" in duration "10sec"`,
+		},
+		{
+			"Bad timeout",
+			`timeout = 1`,
+			`time: missing unit in duration "1"`,
+		},
+		{
+			"Bad failure_threshold",
+			`failure_threshold = -1`,
+			"couper.hcl:13,29-30: Unsuitable value type; Unsuitable value: value must be a whole number, between 0 and 18446744073709551615 inclusive",
+		},
+		{
+			"Bad expect status",
+			`expect_status = [200, 204]`,
+			"couper.hcl:13,25-26: Unsuitable value type; Unsuitable value: number required",
+		},
+		{
+			"OK",
+			`failure_threshold = 3
+			 timeout = "3s"
+			 interval = "5s"
+			 expect_text = 123
+			 expect_status = 200`,
+			"",
+		},
+	}
+
+	logger, _ := test.NewLogger()
+	log := logger.WithContext(context.TODO())
+
+	template := `
+		server {
+		  endpoint "/" {
+		    proxy {
+		      backend = "foo"
+		    }
+		  }
+		}
+		definitions {
+		  backend "foo" {
+		    origin = "..."
+		    beta_health {
+		      %%
+		    }
+		  }
+		}`
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(subT *testing.T) {
+			conf, err := configload.LoadBytes([]byte(strings.Replace(template, "%%", tt.hcl, -1)), "couper.hcl")
+
+			closeCh := make(chan struct{})
+			defer close(closeCh)
+			memStore := cache.New(log, closeCh)
+
+			if conf != nil {
+				_, err = runtime.NewServerConfiguration(conf, log, memStore)
+			}
+
+			var errorMsg = ""
+			if err != nil {
+				errorMsg = err.Error()
+			}
+
+			if tt.error != errorMsg {
+				subT.Errorf("%q: Unexpected configuration error:\n\tWant: %q\n\tGot:  %q", tt.name, tt.error, errorMsg)
+			}
+		})
+	}
+}
diff --git a/handler/transport/backend_test.go b/handler/transport/backend_test.go
index a4666724..a34869ca 100644
--- a/handler/transport/backend_test.go
+++ b/handler/transport/backend_test.go
@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"strings"
 	"testing"
 	"time"
@@ -354,3 +356,137 @@ func TestBackend_director(t *testing.T) {
 		})
 	}
 }
+
+func TestBackend_HealthCheck(t *testing.T) {
+	type expectation struct {
+		FailureThreshold uint
+		Interval         time.Duration
+		Timeout          time.Duration
+		ExpectStatus     map[int]bool
+		ExpectText       string
+		URL              *url.URL
+		RequestUIDFormat string
+	}
+
+	type testCase struct {
+		name        string
+		health      *config.Health
+		expectation expectation
+	}
+
+	defaultExpectStatus := map[int]bool{200: true, 204: true, 301: true}
+
+	for _, tc := range []testCase{
+		{
+			name:   "health check with default values",
+			health: &config.Health{},
+			expectation: expectation{
+				FailureThreshold: 2,
+				Interval:         time.Second,
+				Timeout:          time.Second,
+				ExpectStatus:     defaultExpectStatus,
+				ExpectText:       "",
+				RequestUIDFormat: "common",
+			},
+		},
+		{
+			name: "health check with configured values",
+			health: &config.Health{
+				FailureThreshold: 42,
+				Interval:         "1h",
+				Timeout:          "9m",
+				Path:             "/gsund??",
+				ExpectStatus:     418,
+				ExpectText:       "roger roger",
+			},
+			expectation: expectation{
+				FailureThreshold: 42,
+				Interval:         time.Hour,
+				Timeout:          9 * time.Minute,
+				ExpectStatus:     map[int]bool{418: true},
+				ExpectText:       "roger roger",
+				URL: &url.URL{
+					Scheme:   "http",
+					Host:     "origin:8080",
+					Path:     "/gsund",
+					RawQuery: "?",
+				},
+				RequestUIDFormat: "common",
+			},
+		},
+		{
+			name:   "uninitialised health check",
+			health: nil,
+			expectation: expectation{
+				FailureThreshold: 2,
+				Interval:         time.Second,
+				Timeout:          time.Second,
+				ExpectStatus:     defaultExpectStatus,
+				ExpectText:       "",
+				RequestUIDFormat: "common",
+			},
+		},
+		{
+			name: "timeout set indirectly by configured interval",
+			health: &config.Health{
+				Interval: "10s",
+			},
+			expectation: expectation{
+				FailureThreshold: 2,
+				Interval:         10 * time.Second,
+				Timeout:          10 * time.Second,
+				ExpectStatus:     defaultExpectStatus,
+				ExpectText:       "",
+				RequestUIDFormat: "common",
+			},
+		},
+		{
+			name: "timeout bounded by configured interval",
+			health: &config.Health{
+				Interval: "5s",
+				Timeout:  "10s",
+			},
+			expectation: expectation{
+				FailureThreshold: 2,
+				Interval:         5 * time.Second,
+				Timeout:          5 * time.Second,
+				ExpectStatus:     defaultExpectStatus,
+				ExpectText:       "",
+				RequestUIDFormat: "common",
+			},
+		},
+		{
+			name: "zero threshold",
+			health: &config.Health{
+				FailureThreshold: 0,
+			},
+			expectation: expectation{
+				FailureThreshold: 2,
+				Interval:         time.Second,
+				Timeout:          time.Second,
+				ExpectStatus:     defaultExpectStatus,
+				ExpectText:       "",
+				RequestUIDFormat: "common",
+			},
+		},
+	} {
+		t.Run(tc.name, func(subT *testing.T) {
+			h := test.New(subT)
+
+			health, err := config.NewHealthCheck("http://origin:8080/foo", tc.health, &config.DefaultSettings)
+			h.Must(err)
+
+			if tc.expectation.URL != nil {
+				if *tc.expectation.URL != *health.Request.URL {
+					t.Errorf("Unexpected health check URI:\n\tWant: %#v\n\tGot:  %#v", tc.expectation.URL, health.Request.URL)
+				}
+				tc.expectation.URL = nil
+			}
+			health.Request = nil
+
+			if fmt.Sprint(tc.expectation) != fmt.Sprint(*health) {
+				t.Errorf("Unexpected health options:\n\tWant: %v\n\tGot:  %v", tc.expectation, *health)
+			}
+		})
+	}
+}
diff --git a/internal/test/test_backend.go b/internal/test/test_backend.go
index eb82436b..2d3e8e18 100644
--- a/internal/test/test_backend.go
+++ b/internal/test/test_backend.go
@@ -73,8 +73,10 @@ func registerHTTPHandler(b *Backend) {
 	b.mux.HandleFunc("/anything", createAnythingHandler(http.StatusOK))
 	b.mux.HandleFunc("/", createAnythingHandler(http.StatusNotFound))
 	b.mux.HandleFunc("/ws", echo)
+	b.mux.HandleFunc("/redirect", redirect)
 	b.mux.HandleFunc("/pdf", pdf)
 	b.mux.HandleFunc("/small", small)
+	b.mux.HandleFunc("/health", health)
 	b.mux.HandleFunc("/jwks.json", jwks)
 	b.mux.HandleFunc("/.well-known/openid-configuration", oidc)
 	b.mux.HandleFunc("/error", func(w http.ResponseWriter, _ *http.Request) {
@@ -188,6 +190,22 @@ func echo(rw http.ResponseWriter, req *http.Request) {
 	wg.Wait()
 }
 
+func redirect(rw http.ResponseWriter, req *http.Request) {
+	queryParams := req.URL.Query()
+	if queryParams.Get("url") == "" {
+		rw.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	// Allow relative redirects only
+	uri, _ := url.ParseRequestURI(queryParams.Get("url"))
+	uri.Scheme = ""
+	uri.Host = ""
+
+	rw.Header().Set("Location", uri.String())
+	rw.WriteHeader(http.StatusFound)
+}
+
 func pdf(rw http.ResponseWriter, req *http.Request) {
 	_, currFile, _, _ := runtime.Caller(0)
 	rootDir := http.Dir(path.Join(path.Dir(currFile), "testdata"))
@@ -210,6 +228,10 @@ func small(rw http.ResponseWriter, req *http.Request) {
 	rw.Write([]byte("1234567890"))
 }
 
+func health(rw http.ResponseWriter, req *http.Request) {
+	rw.Write([]byte(" 👍 "))
+}
+
 func jwks(rw http.ResponseWriter, req *http.Request) {
 	_, currFile, _, _ := runtime.Caller(0)
 	rootDir := http.Dir(path.Join(path.Dir(currFile), "testdata"))
diff --git a/server/http_integration_test.go b/server/http_integration_test.go
index 08cdc592..f8420674 100644
--- a/server/http_integration_test.go
+++ b/server/http_integration_test.go
@@ -168,6 +168,15 @@ func newCouperWithConfig(couperConfig *config.Couper, helper *test.Helper) (func
 
 	for _, entry := range hook.AllEntries() {
 		if entry.Level < logrus.WarnLevel {
+			// ignore health-check startup errors
+			if req, ok := entry.Data["request"]; ok {
+				if reqFields, ok := req.(logging.Fields); ok {
+					n, _ := reqFields["name"]
+					if hc, ok := n.(string); ok && hc == "health-check" {
+						continue
+					}
+				}
+			}
 			defer os.Exit(1) // ok in loop, next line is the end
 			helper.Must(fmt.Errorf("error: %#v: %s", entry.Data, entry.Message))
 		}
@@ -1048,7 +1057,6 @@ func TestHTTPServer_LogFields(t *testing.T) {
 	req, err := http.NewRequest(http.MethodGet, "http://example.com:8080", nil)
 	helper.Must(err)
 
-	logHook.Reset()
 	res, err := client.Do(req)
 	helper.Must(err)
 
@@ -2623,6 +2631,130 @@ func TestHTTPServer_XFH_AcceptingForwardedUrl(t *testing.T) {
 	}
 }
 
+func TestHTTPServer_BackendProbes(t *testing.T) {
+	helper := test.New(t)
+	client := newClient()
+
+	confPath := path.Join("testdata/integration/config/14_couper.hcl")
+	shutdown, _ := newCouper(confPath, helper)
+	defer shutdown()
+
+	type testCase struct {
+		name   string
+		path   string
+		expect string
+	}
+
+	time.Sleep(2 * time.Second)
+	healthyJSON := `{"error":"","healthy":true,"state":"healthy"}`
+
+	for _, tc := range []testCase{
+		{
+			"unknown backend",
+			"/unknown",
+			`null`,
+		},
+		{
+			"healthy backend",
+			"/healthy/default",
+			healthyJSON,
+		},
+		{
+			"healthy backend w/ expect_status",
+			"/healthy/expect_status",
+			healthyJSON,
+		},
+		{
+			"healthy backend w/ expect_text",
+			"/healthy/expect_text",
+			healthyJSON,
+		},
+		{
+			"healthy backend w/ path",
+			"/healthy/path",
+			healthyJSON,
+		},
+		{
+			"healthy backend w/ headers",
+			"/healthy/headers",
+			healthyJSON,
+		},
+		{
+			"healthy backend w/ fallback ua header",
+			"/healthy/ua-header",
+			healthyJSON,
+		},
+		{
+			"healthy backend: check does not follow Location",
+			"/healthy/no_follow_redirect",
+			healthyJSON,
+		},
+		{
+			"unhealthy backend: timeout",
+			"/unhealthy/timeout",
+			`{"error":"backend error: proxyconnect tcp: dial tcp 127.0.0.1:9999: connect: connection refused","healthy":false,"state":"unhealthy"}`,
+		},
+		{
+			"unhealthy backend: unexpected status code",
+			"/unhealthy/bad_status",
+			`{"error":"unexpected statusCode: 404","healthy":false,"state":"unhealthy"}`,
+		},
+		{
+			"unhealthy backend w/ expect_status: unexpected status code",
+			"/unhealthy/bad_expect_status",
+			`{"error":"unexpected statusCode: 200","healthy":false,"state":"unhealthy"}`,
+		},
+		{
+			"unhealthy backend w/ expect_text: unexpected text",
+			"/unhealthy/bad_expect_text",
+			`{"error":"unexpected text","healthy":false,"state":"unhealthy"}`,
+		},
+		{
+			"unhealthy backend: unexpected status code",
+			"/unhealthy/bad_status",
+			`{"error":"unexpected statusCode: 404","healthy":false,"state":"unhealthy"}`,
+		},
+		{
+			"unhealthy backend w/ path: unexpected status code",
+			"/unhealthy/bad_path",
+			`{"error":"unexpected statusCode: 404","healthy":false,"state":"unhealthy"}`,
+		},
+		{
+			"unhealthy backend w/ headers: unexpected text",
+			"/unhealthy/headers",
+			`{"error":"unexpected text","healthy":false,"state":"unhealthy"}`,
+		},
+		{
+			"unhealthy backend: does not follow location",
+			"/unhealthy/no_follow_redirect",
+			`{"error":"unexpected statusCode: 302","healthy":false,"state":"unhealthy"}`,
+		},
+		{
+			"failing backend: timeout but threshold not reached",
+			"/failing",
+			`{"error":"backend error: proxyconnect tcp: dial tcp 127.0.0.1:9999: connect: connection refused","healthy":true,"state":"failing"}`,
+		},
+	} {
+		t.Run(tc.name, func(subT *testing.T) {
+			h := test.New(subT)
+
+			req, err := http.NewRequest(http.MethodGet, "http://localhost:8080"+tc.path, nil)
+			h.Must(err)
+
+			res, err := client.Do(req)
+			h.Must(err)
+
+			b, _ := io.ReadAll(res.Body)
+			body := string(b)
+			h.Must(res.Body.Close())
+
+			if body != tc.expect {
+				t.Errorf("%s: Unexpected states:\n\tWant: %s\n\tGot:  %s", tc.name, tc.expect, body)
+			}
+		})
+	}
+}
+
 func TestHTTPServer_backend_requests_variables(t *testing.T) {
 	client := newClient()
 
diff --git a/server/testdata/integration/config/14_couper.hcl b/server/testdata/integration/config/14_couper.hcl
new file mode 100644
index 00000000..e32f7a59
--- /dev/null
+++ b/server/testdata/integration/config/14_couper.hcl
@@ -0,0 +1,227 @@
+server {
+  endpoint "/unknown" {
+    response {
+      json_body = backends.UNKNOWN.health
+    }
+  }
+  endpoint "/healthy/default" {
+    response {
+      json_body = backends.healthy.health
+    }
+  }
+  endpoint "/healthy/expect_status" {
+    response {
+      json_body = backends.healthy_expect_status.health
+    }
+  }
+  endpoint "/healthy/expect_text" {
+    response {
+      json_body = backends.healthy_expect_text.health
+    }
+  }
+  endpoint "/healthy/path" {
+    response {
+      json_body = backends.healthy_path.health
+    }
+  }
+  endpoint "/healthy/headers" {
+    response {
+      json_body = backends.healthy_headers.health
+    }
+  }
+  endpoint "/healthy/ua-header" {
+    response {
+      json_body = backends.healthy_ua_header.health
+    }
+  }
+  endpoint "/healthy/no_follow_redirect" {
+    response {
+      json_body = backends.healthy_no_follow_redirect.health
+    }
+  }
+  endpoint "/unhealthy/timeout" {
+    response {
+      json_body = backends.unhealthy_timeout.health
+    }
+  }
+  endpoint "/unhealthy/bad_status" {
+    response {
+      json_body = backends.unhealthy_bad_status.health
+    }
+  }
+  endpoint "/unhealthy/bad_expect_status" {
+    response {
+      json_body = backends.unhealthy_bad_expect_status.health
+    }
+  }
+  endpoint "/unhealthy/bad_expect_text" {
+    response {
+      json_body = backends.unhealthy_bad_expect_text.health
+    }
+  }
+  endpoint "/unhealthy/bad_path" {
+    response {
+      json_body = backends.unhealthy_bad_path.health
+    }
+  }
+  endpoint "/unhealthy/headers" {
+    response {
+      json_body = backends.unhealthy_headers.health
+    }
+  }
+  endpoint "/unhealthy/no_follow_redirect" {
+    response {
+      json_body = backends.unhealthy_no_follow_redirect.health
+    }
+  }
+  endpoint "/failing" {
+    response {
+      json_body = backends.failing.health
+    }
+  }
+
+  endpoint "/dummy" {
+    request "default" {
+      backend = "healthy"
+    }
+    request "healthy_expect_status" {
+      backend = "healthy_expect_status"
+    }
+    request "healthy_expect_text" {
+      backend = "healthy_expect_text"
+    }
+    request "healthy_path" {
+      backend = "healthy_path"
+    }
+    request "healthy_headers" {
+      backend = "healthy_headers"
+    }
+    request "healthy_ua_header" {
+      backend = "healthy_ua_header"
+    }
+    request "healthy_no_follow_redirect" {
+      backend = "healthy_no_follow_redirect"
+    }
+    request "unhealthy_timeout" {
+      backend = "unhealthy_timeout"
+    }
+    request "unhealthy_bad_status" {
+      backend = "unhealthy_bad_status"
+    }
+    request "unhealthy_bad_expect_status" {
+      backend = "unhealthy_bad_expect_status"
+    }
+    request "unhealthy_bad_expect_text" {
+      backend = "unhealthy_bad_expect_text"
+    }
+    request "unhealthy_bad_path" {
+      backend = "unhealthy_bad_path"
+    }
+    request "unhealthy_headers" {
+      backend = "unhealthy_headers"
+    }
+    request "unhealthy_no_follow_redirect" {
+      backend = "unhealthy_no_follow_redirect"
+    }
+    request "failing" {
+      backend = "failing"
+    }
+  }
+}
+
+definitions {
+  backend "healthy" {
+    origin = "${env.COUPER_TEST_BACKEND_ADDR}/health"
+    beta_health {}
+  }
+  backend "healthy_expect_status" {
+    origin = "${env.COUPER_TEST_BACKEND_ADDR}/not-there"
+    beta_health {
+      expect_status = 404
+    }
+  }
+  backend "healthy_expect_text" {
+    origin = "${env.COUPER_TEST_BACKEND_ADDR}/health"
+    beta_health {
+      expect_text = "👍"
+    }
+  }
+  backend "healthy_path" {
+    origin = env.COUPER_TEST_BACKEND_ADDR
+    beta_health {
+      path = "/anything?foo=bar"
+      expect_text = "\"RawQuery\":\"foo=bar\""
+    }
+  }
+
+  backend "healthy_headers" {
+    origin = env.COUPER_TEST_BACKEND_ADDR
+    beta_health {
+      path = "/anything"
+      headers = {User-Agent: "Couper-Health-Check"}
+      expect_text = "\"UserAgent\":\"Couper-Health-Check\""
+    }
+  }
+
+  backend "healthy_ua_header" {
+    origin = env.COUPER_TEST_BACKEND_ADDR
+    beta_health {
+      path = "/anything"
+      expect_text = "\"UserAgent\":\"Couper / 0 health-check\""
+    }
+  }
+
+  backend "healthy_no_follow_redirect" {
+    origin = env.COUPER_TEST_BACKEND_ADDR
+    beta_health {
+      path = "/redirect?url=/health?redirected"
+      expect_status = 302
+    }
+  }
+  backend "unhealthy_timeout" {
+    origin = "http://1.2.3.4"
+    beta_health {}
+  }
+  backend "unhealthy_bad_status" {
+    origin = "${env.COUPER_TEST_BACKEND_ADDR}"
+    beta_health {}
+  }
+  backend "unhealthy_bad_expect_status" {
+    origin = "${env.COUPER_TEST_BACKEND_ADDR}/health"
+    beta_health {
+      expect_status = 500
+    }
+  }
+  backend "unhealthy_bad_expect_text" {
+    origin = "${env.COUPER_TEST_BACKEND_ADDR}/health"
+    beta_health {
+      expect_text = "down?"
+    }
+  }
+  backend "unhealthy_bad_path" {
+    origin = "${env.COUPER_TEST_BACKEND_ADDR}/health"
+    beta_health {
+      path = "/"
+    }
+  }
+  backend "unhealthy_headers" {
+    origin = "${env.COUPER_TEST_BACKEND_ADDR}/anything"
+    beta_health {
+      headers = {User-Agent = "FAIL"}
+      expect_text = "Go-http-client"
+    }
+  }
+  backend "unhealthy_no_follow_redirect" {
+    origin = env.COUPER_TEST_BACKEND_ADDR
+    beta_health {
+      path = "/redirect?url=/health?redirected"
+      expect_text = "👍"
+    }
+  }
+  backend "failing" {
+    origin = "http://1.2.3.4"
+    beta_health {
+      failure_threshold = 4
+    }
+  }
+}