Repository: avenga/couper
Pull request #508 (2022-05-18 11:35:36)

add circuit breaker for health-checked backends
and options to continue anyways

- Fix probe transport settings (proxy, certs)
- Add metric `backend_up`
- shutdown probes on `-watch` changes

---
<details>
    <summary>Reviewer checklist</summary>
    <ul>
        <li>Read PR description: a summary about the changes is required</li>
        <li>Changelog updated</li>
        <li>Documentation: docs/{Reference, Cli, ...}, Docker and cli help/usage</li>
        <li>Pulled branch, manually tested</li>
        <li>Verified requirements are met</li>
        <li>Reviewed the code</li>
        <li>Reviewed the related tests</li>
    </ul>
</details>



Patch:
diff --git a/cache/memory.go b/cache/memory.go
index 6eac1708..a3e8af3f 100644
--- a/cache/memory.go
+++ b/cache/memory.go
@@ -2,6 +2,7 @@ package cache
 
 import (
 	"runtime/debug"
+	"strings"
 	"sync"
 	"time"
 
@@ -60,6 +61,23 @@ func (ms *MemoryStore) Get(k string) interface{} {
 	return nil
 }
 
+func (ms *MemoryStore) GetAllWithPrefix(prefix string) []interface{} {
+	var list []interface{}
+
+	ms.mu.RLock()
+	defer ms.mu.RUnlock()
+
+	for k, v := range ms.db {
+		if time.Now().Unix() >= v.expAt || !strings.HasPrefix(k, prefix) {
+			continue
+		}
+
+		list = append(list, v.value)
+	}
+
+	return list
+}
+
 // Set stores a key/value pair for <ttl> second(s) into the <MemoryStore>.
 func (ms *MemoryStore) Set(k string, v interface{}, ttl int64) {
 	if ttl < 0 {
diff --git a/command/run.go b/command/run.go
index 1edfd51b..aa1fcaaf 100644
--- a/command/run.go
+++ b/command/run.go
@@ -20,6 +20,7 @@ import (
 	"github.com/avenga/couper/config/env"
 	"github.com/avenga/couper/config/runtime"
 	"github.com/avenga/couper/errors"
+	"github.com/avenga/couper/eval"
 	"github.com/avenga/couper/server"
 	"github.com/avenga/couper/server/writer"
 	"github.com/avenga/couper/telemetry"
@@ -98,6 +99,9 @@ func (r *Run) Execute(args Args, config *config.Couper, logEntry *logrus.Entry)
 	*r.settings = *config.Settings
 	r.settingsMu.Unlock()
 
+	// apply command context
+	config.Context = config.Context.(*eval.Context).WithContext(r.context)
+
 	if f := r.flagSet.Lookup("accept-forwarded-url"); f != nil {
 		if afv, ok := f.Value.(*AcceptForwardedValue); ok {
 			afv.settings = r.settings
@@ -134,6 +138,14 @@ func (r *Run) Execute(args Args, config *config.Couper, logEntry *logrus.Entry)
 		logEntry.Infof("configured with ca-certificate: %s", config.Settings.CAFile)
 	}
 
+	memStore := cache.New(logEntry, r.context.Done())
+	// logEntry has still the 'daemon' type which can be used for config related load errors.
+	srvConf, err := runtime.NewServerConfiguration(config, logEntry, memStore)
+	if err != nil {
+		return err
+	}
+	errors.SetLogger(logEntry)
+
 	telemetry.InitExporter(r.context, &telemetry.Options{
 		MetricsCollectPeriod: time.Second * 2,
 		Metrics:              r.settings.TelemetryMetrics,
@@ -143,14 +155,7 @@ func (r *Run) Execute(args Args, config *config.Couper, logEntry *logrus.Entry)
 		ServiceName:          r.settings.TelemetryServiceName,
 		Traces:               r.settings.TelemetryTraces,
 		TracesEndpoint:       r.settings.TelemetryTracesEndpoint,
-	}, logEntry)
-
-	// logEntry has still the 'daemon' type which can be used for config related load errors.
-	srvConf, err := runtime.NewServerConfiguration(config, logEntry, cache.New(logEntry, r.context.Done()))
-	if err != nil {
-		return err
-	}
-	errors.SetLogger(logEntry)
+	}, memStore, logEntry)
 
 	if limitFn != nil {
 		limitFn(logEntry)
diff --git a/config/backend.go b/config/backend.go
index 208fb470..5e418e26 100644
--- a/config/backend.go
+++ b/config/backend.go
@@ -53,6 +53,7 @@ func (b Backend) Inline() interface{} {
 		ResponseStatus *uint8                    `hcl:"set_response_status,optional"`
 		TTFBTimeout    string                    `hcl:"ttfb_timeout,optional"`
 		Timeout        string                    `hcl:"timeout,optional"`
+		UseUnhealthy   bool                      `hcl:"use_when_unhealthy,optional"`
 
 		// set by backend preparation
 		BackendURL string `hcl:"backend_url,optional"`
diff --git a/config/health_check.go b/config/health_check.go
index 0629ebd2..108f1cb8 100644
--- a/config/health_check.go
+++ b/config/health_check.go
@@ -1,6 +1,7 @@
 package config
 
 import (
+	"context"
 	"net/http"
 	"net/url"
 	"time"
@@ -19,19 +20,20 @@ var defaultHealthCheck = &HealthCheck{
 }
 
 type HealthCheck struct {
-	FailureThreshold uint
-	Interval         time.Duration
-	Timeout          time.Duration
+	Context          context.Context
 	ExpectedStatus   map[int]bool
 	ExpectedText     string
+	FailureThreshold uint
+	Interval         time.Duration
 	Request          *http.Request
 	RequestUIDFormat string
+	Timeout          time.Duration
 }
 
 type Headers map[string]string
 
 type Health struct {
-	FailureThreshold uint     `hcl:"failure_threshold,optional"`
+	FailureThreshold *uint    `hcl:"failure_threshold,optional"`
 	Interval         string   `hcl:"interval,optional"`
 	Timeout          string   `hcl:"timeout,optional"`
 	Path             string   `hcl:"path,optional"`
@@ -41,61 +43,66 @@ type Health struct {
 	Remain           hcl.Body `hcl:",remain"`
 }
 
-func NewHealthCheck(baseURL string, options *Health, settings *Settings) (*HealthCheck, error) {
+func NewHealthCheck(baseURL string, options *Health, conf *Couper) (*HealthCheck, error) {
 	healthCheck := *defaultHealthCheck
-	healthCheck.RequestUIDFormat = settings.RequestIDFormat
 
+	healthCheck.Context = conf.Context
+	healthCheck.RequestUIDFormat = conf.Settings.RequestIDFormat
+
+	if options == nil {
+		return &healthCheck, nil
+	}
 	var err error
-	if options != nil {
-		if options.Interval != "" {
-			healthCheck.Interval, err = time.ParseDuration(options.Interval)
-			if err != nil {
-				return nil, err
-			}
-			healthCheck.Timeout = healthCheck.Interval
-		}
-		if options.Timeout != "" {
-			healthCheck.Timeout, err = time.ParseDuration(options.Timeout)
-			if err != nil {
-				return nil, err
-			}
-		}
-		if healthCheck.Timeout > healthCheck.Interval {
-			healthCheck.Timeout = healthCheck.Interval
-		}
-		if options.FailureThreshold != 0 {
-			healthCheck.FailureThreshold = options.FailureThreshold
-		}
-		if len(options.ExpectedStatus) > 0 {
-			statusList := map[int]bool{}
-			for _, status := range options.ExpectedStatus {
-				statusList[status] = true
-			}
-			healthCheck.ExpectedStatus = statusList
-		}
-		healthCheck.ExpectedText = options.ExpectedText
 
-		request, err := http.NewRequest(http.MethodGet, baseURL, nil)
+	if options.Interval != "" {
+		healthCheck.Interval, err = time.ParseDuration(options.Interval)
 		if err != nil {
 			return nil, err
 		}
-
-		if options.Path != "" {
-			request.URL = request.URL.ResolveReference(createURL(options.Path))
+		healthCheck.Timeout = healthCheck.Interval
+	}
+	if options.Timeout != "" {
+		healthCheck.Timeout, err = time.ParseDuration(options.Timeout)
+		if err != nil {
+			return nil, err
 		}
-
-		if options.Headers != nil {
-			for key, value := range options.Headers {
-				request.Header.Add(key, value)
-			}
+	}
+	if healthCheck.Timeout > healthCheck.Interval {
+		healthCheck.Timeout = healthCheck.Interval
+	}
+	if options.FailureThreshold != nil {
+		healthCheck.FailureThreshold = *options.FailureThreshold
+	}
+	if len(options.ExpectedStatus) > 0 {
+		statusList := map[int]bool{}
+		for _, status := range options.ExpectedStatus {
+			statusList[status] = true
 		}
+		healthCheck.ExpectedStatus = statusList
+	}
+	healthCheck.ExpectedText = options.ExpectedText
 
-		if ua := request.Header.Get("User-Agent"); ua == "" {
-			request.Header.Set("User-Agent", "Couper / "+utils.VersionName+" health-check")
+	request, err := http.NewRequest(http.MethodGet, baseURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if options.Path != "" {
+		request.URL = request.URL.ResolveReference(createURL(options.Path))
+	}
+
+	if options.Headers != nil {
+		for key, value := range options.Headers {
+			request.Header.Add(key, value)
 		}
+	}
 
-		healthCheck.Request = request
+	if ua := request.Header.Get("User-Agent"); ua == "" {
+		request.Header.Set("User-Agent", "Couper / "+utils.VersionName+" health-check")
 	}
+
+	healthCheck.Request = request
+
 	return &healthCheck, err
 }
 
diff --git a/config/runtime/backend.go b/config/runtime/backend.go
index ef968cb2..46f69449 100644
--- a/config/runtime/backend.go
+++ b/config/runtime/backend.go
@@ -20,7 +20,7 @@ import (
 )
 
 func NewBackend(ctx *hcl.EvalContext, body hcl.Body, log *logrus.Entry,
-	settings *config.Settings, store *cache.MemoryStore) (http.RoundTripper, error) {
+	conf *config.Couper, store *cache.MemoryStore) (http.RoundTripper, error) {
 	const prefix = "backend_"
 	name, err := getBackendName(ctx, body)
 
@@ -35,7 +35,7 @@ func NewBackend(ctx *hcl.EvalContext, body hcl.Body, log *logrus.Entry,
 		return backend.NewContext(body, b.(http.RoundTripper)), nil
 	}
 
-	b, err = newBackend(ctx, body, log, settings, store)
+	b, err = newBackend(ctx, body, log, conf, store)
 	if err != nil {
 		return nil, err
 	}
@@ -47,7 +47,7 @@ func NewBackend(ctx *hcl.EvalContext, body hcl.Body, log *logrus.Entry,
 }
 
 func newBackend(evalCtx *hcl.EvalContext, backendCtx hcl.Body, log *logrus.Entry,
-	settings *config.Settings, memStore *cache.MemoryStore) (http.RoundTripper, error) {
+	conf *config.Couper, memStore *cache.MemoryStore) (http.RoundTripper, error) {
 	beConf := &config.Backend{}
 	if diags := gohcl.DecodeBody(backendCtx, evalCtx, beConf); diags.HasErrors() {
 		return nil, diags
@@ -63,11 +63,11 @@ func newBackend(evalCtx *hcl.EvalContext, backendCtx hcl.Body, log *logrus.Entry
 
 	tc := &transport.Config{
 		BackendName:            beConf.Name,
-		Certificate:            settings.Certificate,
+		Certificate:            conf.Settings.Certificate,
 		DisableCertValidation:  beConf.DisableCertValidation,
 		DisableConnectionReuse: beConf.DisableConnectionReuse,
 		HTTP2:                  beConf.HTTP2,
-		NoProxyFromEnv:         settings.NoProxyFromEnv,
+		NoProxyFromEnv:         conf.Settings.NoProxyFromEnv,
 		MaxConnections:         beConf.MaxConnections,
 	}
 
@@ -88,7 +88,7 @@ func newBackend(evalCtx *hcl.EvalContext, backendCtx hcl.Body, log *logrus.Entry
 			return nil, fmt.Errorf("missing origin for backend %q", beConf.Name)
 		}
 
-		options.HealthCheck, err = config.NewHealthCheck(origin.AsString(), beConf.Health, settings)
+		options.HealthCheck, err = config.NewHealthCheck(origin.AsString(), beConf.Health, conf)
 		if err != nil {
 			return nil, err
 		}
@@ -97,7 +97,7 @@ func newBackend(evalCtx *hcl.EvalContext, backendCtx hcl.Body, log *logrus.Entry
 	oauthContent, _, _ := backendCtx.PartialContent(config.OAuthBlockSchema)
 	if oauthContent != nil {
 		if blocks := oauthContent.Blocks.OfType("oauth2"); len(blocks) > 0 {
-			options.AuthBackend, err = newAuthBackend(evalCtx, beConf, blocks, log, settings, memStore)
+			options.AuthBackend, err = newAuthBackend(evalCtx, beConf, blocks, log, conf, memStore)
 			if err != nil {
 				return nil, err
 			}
@@ -109,7 +109,7 @@ func newBackend(evalCtx *hcl.EvalContext, backendCtx hcl.Body, log *logrus.Entry
 }
 
 func newAuthBackend(evalCtx *hcl.EvalContext, beConf *config.Backend, blocks hcl.Blocks, log *logrus.Entry,
-	settings *config.Settings, memStore *cache.MemoryStore) (transport.TokenRequest, error) {
+	conf *config.Couper, memStore *cache.MemoryStore) (transport.TokenRequest, error) {
 
 	beConf.OAuth2 = &config.OAuth2ReqAuth{}
 
@@ -123,7 +123,7 @@ func newAuthBackend(evalCtx *hcl.EvalContext, beConf *config.Backend, blocks hcl
 	}
 
 	innerBackend := innerContent.Blocks.OfType("backend")[0] // backend block is set by configload
-	authBackend, authErr := NewBackend(evalCtx, innerBackend.Body, log, settings, memStore)
+	authBackend, authErr := NewBackend(evalCtx, innerBackend.Body, log, conf, memStore)
 	if authErr != nil {
 		return nil, authErr
 	}
diff --git a/config/runtime/endpoint.go b/config/runtime/endpoint.go
index 5e495b76..05911563 100644
--- a/config/runtime/endpoint.go
+++ b/config/runtime/endpoint.go
@@ -71,7 +71,7 @@ func newEndpointMap(srvConf *config.Server, serverOptions *server.Options) (endp
 }
 
 func newEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint, apiConf *config.API,
-	serverOptions *server.Options, log *logrus.Entry, settings *config.Settings, memStore *cache.MemoryStore) (*handler.EndpointOptions, error) {
+	serverOptions *server.Options, log *logrus.Entry, conf *config.Couper, memStore *cache.MemoryStore) (*handler.EndpointOptions, error) {
 	var errTpl *errors.Template
 
 	if endpointConf.ErrorFile != "" {
@@ -101,7 +101,7 @@ func newEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 
 	allProxies := make(map[string]*producer.Proxy)
 	for _, proxyConf := range endpointConf.Proxies {
-		backend, berr := NewBackend(confCtx, proxyConf.Backend, log, settings, memStore)
+		backend, berr := NewBackend(confCtx, proxyConf.Backend, log, conf, memStore)
 		if berr != nil {
 			return nil, berr
 		}
@@ -118,7 +118,7 @@ func newEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 
 	allRequests := make(map[string]*producer.Request)
 	for _, requestConf := range endpointConf.Requests {
-		backend, berr := NewBackend(confCtx, requestConf.Backend, log, settings, memStore)
+		backend, berr := NewBackend(confCtx, requestConf.Backend, log, conf, memStore)
 		if berr != nil {
 			return nil, berr
 		}
diff --git a/config/runtime/error_handler.go b/config/runtime/error_handler.go
index 41344d9c..2b480431 100644
--- a/config/runtime/error_handler.go
+++ b/config/runtime/error_handler.go
@@ -13,7 +13,7 @@ import (
 	"github.com/avenga/couper/handler"
 )
 
-func newErrorHandler(ctx *hcl.EvalContext, opts *protectedOptions, log *logrus.Entry,
+func newErrorHandler(ctx *hcl.EvalContext, conf *config.Couper, opts *protectedOptions, log *logrus.Entry,
 	defs ACDefinitions, references ...string) (http.Handler, error) {
 	kindsHandler := map[string]http.Handler{}
 	for _, ref := range references {
@@ -62,7 +62,7 @@ func newErrorHandler(ctx *hcl.EvalContext, opts *protectedOptions, log *logrus.E
 				epConf.Response = &config.Response{Remain: emptyBody}
 			}
 
-			epOpts, err := newEndpointOptions(ctx, epConf, nil, opts.srvOpts, log, opts.settings, opts.memStore)
+			epOpts, err := newEndpointOptions(ctx, epConf, nil, opts.srvOpts, log, conf, opts.memStore)
 			if err != nil {
 				return nil, err
 			}
diff --git a/config/runtime/server.go b/config/runtime/server.go
index 7d0bd7a1..77f02465 100644
--- a/config/runtime/server.go
+++ b/config/runtime/server.go
@@ -87,7 +87,9 @@ func NewServerConfiguration(conf *config.Couper, log *logrus.Entry, memStore *ca
 	if ocErr != nil {
 		return nil, ocErr
 	}
-	conf.Context = evalContext.WithOidcConfig(oidcConfigs)
+	conf.Context = evalContext.
+		WithMemStore(memStore).
+		WithOidcConfig(oidcConfigs)
 
 	accessControls, acErr := configureAccessControls(conf, confCtx, log, memStore, oidcConfigs)
 	if acErr != nil {
@@ -104,7 +106,7 @@ func NewServerConfiguration(conf *config.Couper, log *logrus.Entry, memStore *ca
 	// Populate defined backends first...
 	if conf.Definitions != nil {
 		for _, backend := range conf.Definitions.Backend {
-			_, err := NewBackend(confCtx, backend.HCLBody(), log, conf.Settings, memStore)
+			_, err := NewBackend(confCtx, backend.HCLBody(), log, conf, memStore)
 			if err != nil {
 				return nil, err
 			}
@@ -156,14 +158,13 @@ func NewServerConfiguration(conf *config.Couper, log *logrus.Entry, memStore *ca
 			allowedMethodsHandler := middleware.NewAllowedMethodsHandler(nil, middleware.DefaultFileSpaAllowedMethods, spaHandler, notAllowedMethodsHandler)
 			spaHandler = allowedMethodsHandler
 
-			spaHandler, err = configureProtectedHandler(accessControls, confCtx,
+			spaHandler, err = configureProtectedHandler(accessControls, conf, confCtx,
 				config.NewAccessControl(srvConf.AccessControl, srvConf.DisableAccessControl),
 				config.NewAccessControl(spaConf.AccessControl, spaConf.DisableAccessControl),
 				&protectedOptions{
 					epOpts:   epOpts,
 					handler:  spaHandler,
 					memStore: memStore,
-					settings: conf.Settings,
 					srvOpts:  serverOptions,
 				}, log)
 			if err != nil {
@@ -207,14 +208,13 @@ func NewServerConfiguration(conf *config.Couper, log *logrus.Entry, memStore *ca
 			allowedMethodsHandler := middleware.NewAllowedMethodsHandler(nil, middleware.DefaultFileSpaAllowedMethods, fileHandler, notAllowedMethodsHandler)
 			fileHandler = allowedMethodsHandler
 
-			fileHandler, err = configureProtectedHandler(accessControls, confCtx,
+			fileHandler, err = configureProtectedHandler(accessControls, conf, confCtx,
 				config.NewAccessControl(srvConf.AccessControl, srvConf.DisableAccessControl),
 				config.NewAccessControl(srvConf.Files.AccessControl, srvConf.Files.DisableAccessControl),
 				&protectedOptions{
 					epOpts:   epOpts,
 					handler:  fileHandler,
 					memStore: memStore,
-					settings: conf.Settings,
 					srvOpts:  serverOptions,
 				}, log)
 			if err != nil {
@@ -263,7 +263,7 @@ func NewServerConfiguration(conf *config.Couper, log *logrus.Entry, memStore *ca
 			endpointPatterns[cleanPattern] = true
 
 			epOpts, err := newEndpointOptions(confCtx, endpointConf, parentAPI, serverOptions,
-				log, conf.Settings, memStore)
+				log, conf, memStore)
 			if err != nil {
 				return nil, err
 			}
@@ -295,10 +295,9 @@ func NewServerConfiguration(conf *config.Couper, log *logrus.Entry, memStore *ca
 			if parentAPI != nil && parentAPI.CatchAllEndpoint == endpointConf {
 				protectedHandler = epOpts.ErrorTemplate.WithError(errors.RouteNotFound)
 			} else {
-				epErrorHandler, err := newErrorHandler(confCtx, &protectedOptions{
+				epErrorHandler, err := newErrorHandler(confCtx, conf, &protectedOptions{
 					epOpts:   epOpts,
 					memStore: memStore,
-					settings: conf.Settings,
 					srvOpts:  serverOptions,
 				}, log, errorHandlerDefinitions, "api", "endpoint") // sequence of ref is important: api, endpoint (endpoint error_handler overrides api error_handler)
 				if err != nil {
@@ -318,10 +317,9 @@ func NewServerConfiguration(conf *config.Couper, log *logrus.Entry, memStore *ca
 					protectedHandler = epHandler
 				} else {
 					permissionsControl := ac.NewPermissionsControl(requiredPermissionExpr)
-					permissionsErrorHandler, err := newErrorHandler(confCtx, &protectedOptions{
+					permissionsErrorHandler, err := newErrorHandler(confCtx, conf, &protectedOptions{
 						epOpts:   epOpts,
 						memStore: memStore,
-						settings: conf.Settings,
 						srvOpts:  serverOptions,
 					}, log, errorHandlerDefinitions, "api", "endpoint") // sequence of ref is important: api, endpoint (endpoint error_handler overrides api error_handler)
 					if err != nil {
@@ -343,13 +341,12 @@ func NewServerConfiguration(conf *config.Couper, log *logrus.Entry, memStore *ca
 			allowedMethodsHandler := middleware.NewAllowedMethodsHandler(allowedMethods, middleware.DefaultEndpointAllowedMethods, protectedHandler, notAllowedMethodsHandler)
 			protectedHandler = allowedMethodsHandler
 
-			epHandler, err = configureProtectedHandler(accessControls, confCtx, accessControl,
+			epHandler, err = configureProtectedHandler(accessControls, conf, confCtx, accessControl,
 				config.NewAccessControl(endpointConf.AccessControl, endpointConf.DisableAccessControl),
 				&protectedOptions{
 					epOpts:   epOpts,
 					handler:  protectedHandler,
 					memStore: memStore,
-					settings: conf.Settings,
 					srvOpts:  serverOptions,
 				}, log)
 			if err != nil {
@@ -444,7 +441,7 @@ func configureOidcConfigs(conf *config.Couper, confCtx *hcl.EvalContext, log *lo
 			backends := map[string]http.RoundTripper{}
 			for k, backendBody := range oidcConf.Backends {
 				var err error
-				backends[k], err = NewBackend(confCtx, backendBody, log, conf.Settings, memStore)
+				backends[k], err = NewBackend(confCtx, backendBody, log, conf, memStore)
 				if err != nil {
 					return nil, confErr.With(err)
 				}
@@ -512,7 +509,7 @@ func configureAccessControls(conf *config.Couper, confCtx *hcl.EvalContext, log
 
 		for _, oauth2Conf := range conf.Definitions.OAuth2AC {
 			confErr := errors.Configuration.Label(oauth2Conf.Name)
-			backend, err := NewBackend(confCtx, oauth2Conf.Backend, log, conf.Settings, memStore)
+			backend, err := NewBackend(confCtx, oauth2Conf.Backend, log, conf, memStore)
 			if err != nil {
 				return nil, confErr.With(err)
 			}
@@ -602,7 +599,7 @@ func configureJWKS(jwtConf *config.JWT, confContext *hcl.EvalContext, log *logru
 	if jwtConf.Backends != nil {
 		backendBody, ok := jwtConf.Backends["backend"]
 		if ok {
-			b, err := NewBackend(confContext, backendBody, log, conf.Settings, memStore)
+			b, err := NewBackend(confContext, backendBody, log, conf, memStore)
 			if err != nil {
 				return nil, err
 			}
@@ -617,18 +614,17 @@ type protectedOptions struct {
 	epOpts   *handler.EndpointOptions
 	handler  http.Handler
 	memStore *cache.MemoryStore
-	settings *config.Settings
 	srvOpts  *server.Options
 }
 
-func configureProtectedHandler(m ACDefinitions, ctx *hcl.EvalContext, parentAC, handlerAC config.AccessControl,
+func configureProtectedHandler(m ACDefinitions, conf *config.Couper, ctx *hcl.EvalContext, parentAC, handlerAC config.AccessControl,
 	opts *protectedOptions, log *logrus.Entry) (http.Handler, error) {
 	var list ac.List
 	for _, acName := range parentAC.Merge(handlerAC).List() {
 		if e := m.MustExist(acName); e != nil {
 			return nil, e
 		}
-		eh, err := newErrorHandler(ctx, opts, log, m, acName)
+		eh, err := newErrorHandler(ctx, conf, opts, log, m, acName)
 		if err != nil {
 			return nil, err
 		}
diff --git a/docs/ERRORS.md b/docs/ERRORS.md
index 8c9a6227..eb9f5bd1 100644
--- a/docs/ERRORS.md
+++ b/docs/ERRORS.md
@@ -64,6 +64,7 @@ The following table documents error types that can be handled in `api` blocks:
 | `backend`                                          | All catchable backend related errors                                                                    | Send error template with status `502`.                                                                        |
 | `backend_openapi_validation` (`backend`)           | Backend request or response is invalid                                                                  | Send error template with status code `400` for invalid backend request or `502` for invalid backend response. |
 | `backend_timeout` (`backend`)                      | A backend request timed out                                                                             | Send error template with status `504`.                                                                        |
+| `backend_unhealthy` (`backend`)                    | A backend is unhealthy and will not send the request                                                    | Send error template with status `502`.                                                                        |
 | `access_control`                                   | Access control related errors                                                                           | Send error template with status `403`.                                                                        |
 | `beta_insufficient_permissions` (`access_control`) | The permission required for the requested operation is not in the permissions granted to the requester. | Send error template with status `403`.                                                                        |
 
@@ -76,6 +77,7 @@ The following table documents error types that can be handled in `endpoint` bloc
 | `backend`                                          | All catchable backend related errors                                                                    | Send error template with status `502`.                                                                        |
 | `backend_openapi_validation` (`backend`)           | Backend request or response is invalid                                                                  | Send error template with status code `400` for invalid backend request or `502` for invalid backend response. |
 | `backend_timeout` (`backend`)                      | A backend request timed out                                                                             | Send error template with status `504`.                                                                        |
+| `backend_unhealthy` (`backend`)                    | A backend is unhealthy and will not send the request                                                    | Send error template with status `502`.                                                                        |
 | `access_control`                                   | Access control related errors                                                                           | Send error template with status `403`.                                                                        |
 | `beta_insufficient_permissions` (`access_control`) | The permission required for the requested operation is not in the permissions granted to the requester. | Send error template with status `403`.                                                                        |
 | `endpoint`                                         | All catchable `endpoint` related errors                                                                 | Send error template with status `502`.                                                                        |
diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md
index 39f3b435..7fecfa42 100644
--- a/docs/REFERENCE.md
+++ b/docs/REFERENCE.md
@@ -217,32 +217,33 @@ The `backend` block defines the connection to a local/remote backend service.
 
 &#9888; Backends can be defined in the [Definitions Block](#definitions-block) and referenced by _label_.
 
-|Block name|Context|Label|Nested block(s)|
-| :----------| :-----------| :-----------| :-----------|
-|`backend`| [Definitions Block](#definitions-block), [Proxy Block](#proxy-block), [Request Block](#request-block), [OAuth2 CC Block](#oauth2-block), [JWT Block](#jwt-block), [OAuth2 AC Block (beta)](#beta-oauth2-block), [OIDC Block](#oidc-block)| &#9888; required, when defined in [Definitions Block](#definitions-block)| [OpenAPI Block](#openapi-block), [OAuth2 CC Block](#oauth2-block), [Health Block](#health-block)|
-
-| Attribute(s) | Type |Default|Description|Characteristic(s)| Example|
-| :------------------------------ | :--------------- | :--------------- | :--------------- | :--------------- | :--------------- |
-| `basic_auth`                    | string|-|Basic auth for the upstream request. | format is `"username:password"`|-|
-| `custom_log_fields`             | object           | -                | Defines log fields for [Custom Logging](LOGS.md#custom-logging). | - | - |
-| `hostname`                      | string |-|Value of the HTTP host header field for the origin request. |Since `hostname` replaces the request host the value will also be used for a server identity check during a TLS handshake with the origin.|-|
-| `origin`                        |string|-|URL to connect to for backend requests.|&#9888; required.  &#9888; Must start with one of the URI schemes `https` or `http`.|-|
-| `path`                          | string|-|Changeable part of upstream URL.|-|-|
-| `path_prefix`                   | string|-|Prefixes all backend request paths with the given prefix|-|-|
-| `connect_timeout`                | [duration](#duration) | `"10s"`    | The total timeout for dialing and connect to the origin.   |-                                   |-|
-| `disable_certificate_validation` | bool               | `false`       | Disables the peer certificate validation.                                              |      - |-|
-| `disable_connection_reuse`       | bool               | `false`        | Disables reusage of connections to the origin.                                          |    -  |-|
-| `http2`                          | bool               | `false`         | Enables the HTTP2 support.                                                               | -    |-|
-| `max_connections`                | integer                | `0` (unlimited) | The maximum number of concurrent connections in any state (_active_ or _idle_) to the origin. |-|-|
-| `proxy`                          | string             | -| A proxy URL for the related origin request.      |-   | `"http://SERVER-IP_OR_NAME:PORT"`|
-| `timeout`                        | [duration](#duration) | `"300s"`        | The total deadline duration a backend request has for write and read/pipe.               |-     |-|
-| `ttfb_timeout`                   | [duration](#duration) | `"60s"`         | The duration from writing the full request to the origin and receiving the answer.        |-    |-|
-| [Modifiers](#modifiers)           |- |-|All [Modifiers](#modifiers)|-|-|
+| Block name | Context                                                                                                                                                                                                                                   | Label                                                                     | Nested block(s)                                                                                  |
+|:-----------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------|
+| `backend`  | [Definitions Block](#definitions-block), [Proxy Block](#proxy-block), [Request Block](#request-block), [OAuth2 CC Block](#oauth2-block), [JWT Block](#jwt-block), [OAuth2 AC Block (beta)](#beta-oauth2-block), [OIDC Block](#oidc-block) | &#9888; required, when defined in [Definitions Block](#definitions-block) | [OpenAPI Block](#openapi-block), [OAuth2 CC Block](#oauth2-block), [Health Block](#health-block) |
+
+| Attribute(s)                     | Type                  | Default         | Description                                                                                   | Characteristic(s)                                                                                                                          | Example                           |
+|:---------------------------------|:----------------------|:----------------|:----------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------|
+| `basic_auth`                     | string                | -               | Basic auth for the upstream request.                                                          | format is `"username:password"`                                                                                                            | -                                 |
+| `custom_log_fields`              | object                | -               | Defines log fields for [Custom Logging](LOGS.md#custom-logging).                              | -                                                                                                                                          | -                                 |
+| `hostname`                       | string                | -               | Value of the HTTP host header field for the origin request.                                   | Since `hostname` replaces the request host the value will also be used for a server identity check during a TLS handshake with the origin. | -                                 |
+| `origin`                         | string                | -               | URL to connect to for backend requests.                                                       | &#9888; required.  &#9888; Must start with one of the URI schemes `https` or `http`.                                                       | -                                 |
+| `path`                           | string                | -               | Changeable part of upstream URL.                                                              | -                                                                                                                                          | -                                 |
+| `path_prefix`                    | string                | -               | Prefixes all backend request paths with the given prefix                                      | -                                                                                                                                          | -                                 |
+| `connect_timeout`                | [duration](#duration) | `"10s"`         | The total timeout for dialing and connect to the origin.                                      | -                                                                                                                                          | -                                 |
+| `disable_certificate_validation` | bool                  | `false`         | Disables the peer certificate validation.                                                     | -                                                                                                                                          | -                                 |
+| `disable_connection_reuse`       | bool                  | `false`         | Disables reusage of connections to the origin.                                                | -                                                                                                                                          | -                                 |
+| `http2`                          | bool                  | `false`         | Enables the HTTP2 support.                                                                    | -                                                                                                                                          | -                                 |
+| `max_connections`                | integer               | `0` (unlimited) | The maximum number of concurrent connections in any state (_active_ or _idle_) to the origin. | -                                                                                                                                          | -                                 |
+| `proxy`                          | string                | -               | A proxy URL for the related origin request.                                                   | -                                                                                                                                          | `"http://SERVER-IP_OR_NAME:PORT"` |
+| `timeout`                        | [duration](#duration) | `"300s"`        | The total deadline duration a backend request has for write and read/pipe.                    | -                                                                                                                                          | -                                 |
+| `ttfb_timeout`                   | [duration](#duration) | `"60s"`         | The duration from writing the full request to the origin and receiving the answer.            | -                                                                                                                                          | -                                 |
+| `use_when_unhealthy`             | bool                  | `false`         | Ignores the [health](#health-block) state and continues with the outgoing request             | -                                                                                                                                          | -                                 |
+| [Modifiers](#modifiers)          | -                     | -               | All [Modifiers](#modifiers)                                                                   | -                                                                                                                                          | -                                 |
 
 #### Duration
 
 | Duration units | Description  |
-| :------------- | :----------- |
+|:---------------|:-------------|
 | `ns`           | nanoseconds  |
 | `us` (or `µs`) | microseconds |
 | `ms`           | milliseconds |
@@ -543,21 +544,21 @@ Some information from the assertion consumed at the ACS endpoint is provided in
 
 Defines a recurring health check request for its backend. Results can be obtained via the [`backends.<label>.health` variables](#backends).
 Changes in health states and related requests will be logged. Default User-Agent will be `Couper / <version> health-check` if not provided
-via `headers` attribute.
+via `headers` attribute. An unhealthy backend will return with a [`backend_unhealthy`](ERRORS.md#api-error-types) error.
 
 | Block name    | Context                           | Label | Nested block |
 |:--------------|:----------------------------------|:------|:-------------|
 | `beta_health` | [`backend` block](#backend-block) | –     |              |
 
-| Attributes          | Type                  | Default             | Description                                        | Characteristics       | Example                            |
-|:--------------------|:----------------------|:--------------------|:---------------------------------------------------|:----------------------|:-----------------------------------|
-| `expected_status`   | tuple (number)        | `[200, 204, 301]`   | wanted response status code                        |                       | `expected_status = [418]`          |
-| `expected_text`     | string                | –                   | text response body must contain                    |                       | `expected_text = "alive"`          |
-| `failure_threshold` | number                | `2`                 | failed checks needed to consider backend unhealthy |                       | `failure_threshold = 3`            |
-| `headers`           | object                | –                   | request headers                                    |                       | `headers = {User-Agent = "health"}`|
-| `interval`          | [duration](#duration) | `"2s"`              | time interval for recheck                          |                       | `timeout = "5s"`                   |
-| `path`              | string                | –                   | URL path/query on backend host                     |                       | `path = "/health"`                 |
-| `timeout`           | [duration](#duration) | `"2s"`              | maximum allowed time limit                         | bounded by `interval` | `timeout = "3s"`                   |
+| Attributes          | Type                  | Default           | Description                                        | Characteristics       | Example                             |
+|:--------------------|:----------------------|:------------------|:---------------------------------------------------|:----------------------|:------------------------------------|
+| `expected_status`   | tuple (number)        | `[200, 204, 301]` | wanted response status code                        |                       | `expected_status = [418]`           |
+| `expected_text`     | string                | –                 | text response body must contain                    |                       | `expected_text = "alive"`           |
+| `failure_threshold` | number                | `2`               | failed checks needed to consider backend unhealthy |                       | `failure_threshold = 3`             |
+| `headers`           | object                | –                 | request headers                                    |                       | `headers = {User-Agent = "health"}` |
+| `interval`          | [duration](#duration) | `"2s"`            | time interval for recheck                          |                       | `timeout = "5s"`                    |
+| `path`              | string                | –                 | URL path/query on backend host                     |                       | `path = "/health"`                  |
+| `timeout`           | [duration](#duration) | `"2s"`            | maximum allowed time limit                         | bounded by `interval` | `timeout = "3s"`                    |
 
 ### Settings Block
 
diff --git a/errors/type_defintions.go b/errors/type_defintions.go
index f743145a..e14f80c0 100644
--- a/errors/type_defintions.go
+++ b/errors/type_defintions.go
@@ -25,6 +25,7 @@ var Definitions = []*Error{
 	Backend,
 	Backend.Kind("backend_openapi_validation").Status(http.StatusBadRequest),
 	Backend.Kind("backend_timeout").Status(http.StatusGatewayTimeout),
+	Backend.Kind("backend_unhealthy"),
 
 	Endpoint,
 	Endpoint.Kind("sequence"),
diff --git a/errors/types_generated.go b/errors/types_generated.go
index 06c07f6e..ebd89810 100644
--- a/errors/types_generated.go
+++ b/errors/types_generated.go
@@ -15,8 +15,9 @@ var (
 	BetaInsufficientPermissions = Definitions[10]
 	BackendOpenapiValidation    = Definitions[12]
 	BackendTimeout              = Definitions[13]
-	Sequence                    = Definitions[15]
-	UnexpectedStatus            = Definitions[16]
+	BackendUnhealthy            = Definitions[14]
+	Sequence                    = Definitions[16]
+	UnexpectedStatus            = Definitions[17]
 )
 
 // typeDefinitions holds all related error definitions which are
@@ -40,6 +41,7 @@ var types = typeDefinitions{
 	"backend":                        Backend,
 	"backend_openapi_validation":     BackendOpenapiValidation,
 	"backend_timeout":                BackendTimeout,
+	"backend_unhealthy":              BackendUnhealthy,
 	"endpoint":                       Endpoint,
 	"sequence":                       Sequence,
 	"unexpected_status":              UnexpectedStatus,
@@ -54,4 +56,4 @@ func IsKnown(errorType string) bool {
 
 // SuperTypesMapsByContext holds maps for error super-types to sub-types
 // by a given context block type (e.g. api or endpoint).
-var SuperTypesMapsByContext = map[string]map[string][]string{"api": map[string][]string{"*": []string{"beta_insufficient_permissions", "backend_openapi_validation", "backend_timeout"}, "access_control": []string{"beta_insufficient_permissions"}, "backend": []string{"backend_openapi_validation", "backend_timeout"}}, "endpoint": map[string][]string{"*": []string{"beta_insufficient_permissions", "backend_openapi_validation", "backend_timeout", "sequence", "unexpected_status"}, "access_control": []string{"beta_insufficient_permissions"}, "backend": []string{"backend_openapi_validation", "backend_timeout"}, "endpoint": []string{"sequence", "unexpected_status"}}}
+var SuperTypesMapsByContext = map[string]map[string][]string{"api": map[string][]string{"*": []string{"beta_insufficient_permissions", "backend_openapi_validation", "backend_timeout", "backend_unhealthy"}, "access_control": []string{"beta_insufficient_permissions"}, "backend": []string{"backend_openapi_validation", "backend_timeout", "backend_unhealthy"}}, "endpoint": map[string][]string{"*": []string{"beta_insufficient_permissions", "backend_openapi_validation", "backend_timeout", "backend_unhealthy", "sequence", "unexpected_status"}, "access_control": []string{"beta_insufficient_permissions"}, "backend": []string{"backend_openapi_validation", "backend_timeout", "backend_unhealthy"}, "endpoint": []string{"sequence", "unexpected_status"}}}
diff --git a/eval/context.go b/eval/context.go
index 305ba994..bd3e08bb 100644
--- a/eval/context.go
+++ b/eval/context.go
@@ -3,13 +3,13 @@ package eval
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"io"
 	"mime"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/hashicorp/hcl/v2"
@@ -20,11 +20,11 @@ import (
 	"github.com/zclconf/go-cty/cty/function/stdlib"
 	ctyjson "github.com/zclconf/go-cty/cty/json"
 
+	"github.com/avenga/couper/cache"
 	"github.com/avenga/couper/config"
 	"github.com/avenga/couper/config/env"
 	"github.com/avenga/couper/config/request"
 	"github.com/avenga/couper/eval/lib"
-	"github.com/avenga/couper/handler/transport/probe_map"
 	"github.com/avenga/couper/internal/seetie"
 	"github.com/avenga/couper/oauth2/oidc"
 	"github.com/avenga/couper/utils"
@@ -42,8 +42,11 @@ func (m ContextMap) Merge(other ContextMap) ContextMap {
 }
 
 type Context struct {
+	backends          []http.RoundTripper
+	backendsFn        sync.Once
 	eval              *hcl.EvalContext
 	inner             context.Context
+	memStore          *cache.MemoryStore
 	memorize          map[string]interface{}
 	oauth2            []config.OAuth2Authorization
 	jwtSigningConfigs map[string]*lib.JWTSigningConfig
@@ -79,6 +82,11 @@ func ContextFromRequest(req *http.Request) *Context {
 	return NewContext(nil, nil)
 }
 
+func (c *Context) WithContext(ctx context.Context) context.Context {
+	c.inner = ctx
+	return c
+}
+
 func (c *Context) Deadline() (deadline time.Time, ok bool) {
 	return c.inner.Deadline()
 }
@@ -99,9 +107,24 @@ func (c *Context) Value(key interface{}) interface{} {
 }
 
 func (c *Context) WithClientRequest(req *http.Request) *Context {
+	c.backendsFn.Do(func() {
+		if c.memStore == nil {
+			return
+		}
+
+		const prefix = "backend_"
+		for _, b := range c.memStore.GetAllWithPrefix(prefix) {
+			if rt, ok := b.(http.RoundTripper); ok {
+				c.backends = append(c.backends, rt)
+			}
+		}
+	})
+
 	ctx := &Context{
+		backends:          c.backends,
 		eval:              c.cloneEvalContext(),
 		inner:             c.inner,
+		memStore:          c.memStore,
 		memorize:          make(map[string]interface{}),
 		oauth2:            c.oauth2[:],
 		jwtSigningConfigs: c.jwtSigningConfigs,
@@ -160,20 +183,17 @@ func (c *Context) WithClientRequest(req *http.Request) *Context {
 	ctx.eval.Variables[BackendRequests] = cty.ObjectVal(make(map[string]cty.Value))
 	ctx.eval.Variables[BackendResponses] = cty.ObjectVal(make(map[string]cty.Value))
 
-	backendsVariable := map[string]interface{}{}
-	probe_map.BackendProbes.Range(func(backendName, value interface{}) bool {
-		health := value.(probe_map.HealthInfo)
-		backendsVariable[fmt.Sprint(backendName)] = map[string]interface{}{
-			Health: map[string]interface{}{
-				"healthy": health.Healthy,
-				"error":   health.Error,
-				"state":   health.State,
-			},
+	backendsVariable := map[string]cty.Value{}
+	for _, backend := range ctx.backends {
+		b, ok := backend.(seetie.Object)
+		if !ok {
+			continue
 		}
-		return true
-	})
-
-	ctx.eval.Variables[Backends] = seetie.MapToValue(backendsVariable)
+		v := b.Value()
+		vm := v.AsValueMap()
+		backendsVariable[vm["name"].AsString()] = v
+	}
+	ctx.eval.Variables[Backends] = cty.ObjectVal(backendsVariable)
 
 	ctx.updateRequestRelatedFunctions(origin)
 	ctx.updateFunctions()
@@ -183,8 +203,10 @@ func (c *Context) WithClientRequest(req *http.Request) *Context {
 
 func (c *Context) WithBeresp(beresp *http.Response, readBody bool) *Context {
 	ctx := &Context{
+		backends:          c.backends,
 		eval:              c.cloneEvalContext(),
 		inner:             c.inner,
+		memStore:          c.memStore,
 		memorize:          c.memorize,
 		oauth2:            c.oauth2[:],
 		jwtSigningConfigs: c.jwtSigningConfigs,
@@ -321,6 +343,11 @@ func (c *Context) WithOidcConfig(confs oidc.Configs) *Context {
 	return c
 }
 
+func (c *Context) WithMemStore(store *cache.MemoryStore) *Context {
+	c.memStore = store
+	return c
+}
+
 // WithSAML initially set up the saml configuration.
 func (c *Context) WithSAML(s []*config.SAML) *Context {
 	c.saml = s
diff --git a/handler/transport/backend.go b/handler/transport/backend.go
index 4b84df14..cf4f6a4e 100644
--- a/handler/transport/backend.go
+++ b/handler/transport/backend.go
@@ -36,19 +36,26 @@ import (
 	"github.com/avenga/couper/utils"
 )
 
-var _ http.RoundTripper = &Backend{}
+var (
+	_ http.RoundTripper = &Backend{}
+	_ ProbeStateChange  = &Backend{}
+	_ seetie.Object     = &Backend{}
+)
 
 // Backend represents the transport configuration.
 type Backend struct {
-	context          hcl.Body
-	logEntry         *logrus.Entry
-	name             string
-	openAPIValidator *validation.OpenAPI
-	tokenRequest     TokenRequest
-	transport        *http.Transport
-	transportOnce    sync.Once
-	transportConf    *Config
-	upstreamLog      *logging.UpstreamLog
+	context             hcl.Body
+	healthInfo          *HealthInfo
+	healthyMu           sync.RWMutex
+	logEntry            *logrus.Entry
+	name                string
+	openAPIValidator    *validation.OpenAPI
+	tokenRequest        TokenRequest
+	transport           *http.Transport
+	transportConf       *Config
+	transportConfResult Config
+	transportOnce       sync.Once
+	upstreamLog         *logging.UpstreamLog
 }
 
 // NewBackend creates a new <*Backend> object by the given <*Config>.
@@ -67,6 +74,7 @@ func NewBackend(ctx hcl.Body, tc *Config, opts *BackendOptions, log *logrus.Entr
 
 	backend := &Backend{
 		context:          ctx,
+		healthInfo:       &HealthInfo{Healthy: true, State: StateOk.String()},
 		logEntry:         log.WithField("backend", tc.BackendName),
 		name:             tc.BackendName,
 		openAPIValidator: openAPI,
@@ -76,9 +84,9 @@ func NewBackend(ctx hcl.Body, tc *Config, opts *BackendOptions, log *logrus.Entr
 
 	backend.upstreamLog = logging.NewUpstreamLog(backend.logEntry, backend, tc.NoProxyFromEnv)
 
-	distinct := !strings.HasPrefix(tc.Hostname, "anonymous_")
+	distinct := !strings.HasPrefix(tc.BackendName, "anonymous_")
 	if distinct && healthCheck != nil {
-		NewProbe(backend.logEntry, tc.BackendName, healthCheck)
+		NewProbe(backend.logEntry, tc, healthCheck, backend)
 	}
 
 	return backend.upstreamLog
@@ -87,17 +95,18 @@ func NewBackend(ctx hcl.Body, tc *Config, opts *BackendOptions, log *logrus.Entr
 // initOnce ensures synced transport configuration. First request will setup the origin, hostname and tls.
 func (b *Backend) initOnce(conf *Config) {
 	b.transport = NewTransport(conf, b.logEntry)
+	var healthy bool
+	b.healthyMu.Lock()
+	b.transportConfResult = *conf
+	healthy = b.healthInfo.Healthy
+	b.healthyMu.Unlock()
+
+	// race condition, update possible healthy backend with current origin and hostname
+	b.OnProbeChange(&HealthInfo{Healthy: healthy})
 }
 
 // RoundTrip implements the <http.RoundTripper> interface.
 func (b *Backend) RoundTrip(req *http.Request) (*http.Response, error) {
-	// for token-request retry purposes
-	originalReq := req.Clone(req.Context())
-
-	if err := b.withTokenRequest(req); err != nil {
-		return nil, err
-	}
-
 	ctxBody, _ := req.Context().Value(request.BackendParams).(hcl.Body)
 	if ctxBody == nil {
 		ctxBody = b.context
@@ -105,13 +114,26 @@ func (b *Backend) RoundTrip(req *http.Request) (*http.Response, error) {
 		ctxBody = hclbody.MergeBodies(b.context, ctxBody)
 	}
 
+	hclCtx := eval.ContextFromRequest(req).HCLContextSync()
+
+	if err := b.isUnhealthy(hclCtx, ctxBody); err != nil {
+		return &http.Response{
+			Request: req, // provide outreq (variable) on error cases
+		}, err
+	}
+
+	// for token-request retry purposes
+	originalReq := req.Clone(req.Context())
+
+	if err := b.withTokenRequest(req); err != nil {
+		return nil, err
+	}
+
 	logCh, _ := req.Context().Value(request.LogCustomUpstream).(chan hcl.Body)
 	if logCh != nil {
 		logCh <- ctxBody
 	}
 
-	hclCtx := eval.ContextFromRequest(req).HCLContextSync()
-
 	// Execute before <b.evalTransport()> due to right
 	// handling of query-params in the URL attribute.
 	if err := eval.ApplyRequestContext(hclCtx, ctxBody, req); err != nil {
@@ -244,7 +266,7 @@ func (b *Backend) innerRoundTrip(req *http.Request, tc *Config, deadlineErr <-ch
 		spanMsg += "." + b.name
 	}
 
-	meter := provider.Meter("couper/backend")
+	meter := provider.Meter(instrumentation.BackendInstrumentationName)
 	counter := metric.Must(meter).NewInt64Counter(instrumentation.BackendRequest, metric.WithDescription(string(unit.Dimensionless)))
 	duration := metric.Must(meter).
 		NewFloat64Histogram(instrumentation.BackendRequestDuration, metric.WithDescription(string(unit.Dimensionless)))
@@ -488,6 +510,56 @@ func (b *Backend) evalTransport(httpCtx *hcl.EvalContext, params hcl.Body, req *
 		WithTimings(connectTimeout, ttfbTimeout, timeout, log), nil
 }
 
+func (b *Backend) isUnhealthy(ctx *hcl.EvalContext, params hcl.Body) error {
+	paramsContent, _, diags := params.PartialContent(config.BackendInlineSchema)
+	if diags.HasErrors() {
+		return diags
+	}
+	val, err := eval.ValueFromAttribute(ctx, paramsContent, "use_when_unhealthy")
+	if err != nil {
+		return err
+	}
+
+	var useUnhealthy bool
+	if val.Type() == cty.Bool {
+		useUnhealthy = val.True()
+	} // else not set
+
+	b.healthyMu.RLock()
+	defer b.healthyMu.RUnlock()
+
+	if b.healthInfo.Healthy || useUnhealthy {
+		return nil
+	}
+
+	return errors.BackendUnhealthy
+}
+
+func (b *Backend) OnProbeChange(info *HealthInfo) {
+	b.healthyMu.Lock()
+	b.healthInfo = info
+	b.healthyMu.Unlock()
+}
+
+func (b *Backend) Value() cty.Value {
+	b.healthyMu.RLock()
+	defer b.healthyMu.RUnlock()
+
+	return seetie.GoToValue(map[string]interface{}{
+		"health": map[string]interface{}{
+			"healthy": b.healthInfo.Healthy,
+			"error":   b.healthInfo.Error,
+			"state":   b.healthInfo.State,
+		},
+		"hostname":        b.transportConfResult.Hostname,
+		"name":            b.name, // mandatory
+		"origin":          b.transportConfResult.Origin,
+		"connect_timeout": b.transportConfResult.ConnectTimeout.String(),
+		"ttfb_timeout":    b.transportConfResult.TTFBTimeout.String(),
+		"timeout":         b.transportConfResult.Timeout.String(),
+	})
+}
+
 // setUserAgent sets an empty one if none is present or empty
 // to prevent the go http defaultUA gets written.
 func setUserAgent(outreq *http.Request) {
diff --git a/handler/transport/probe.go b/handler/transport/probe.go
index 2311ee75..b906478a 100644
--- a/handler/transport/probe.go
+++ b/handler/transport/probe.go
@@ -17,7 +17,6 @@ import (
 	"github.com/avenga/couper/errors"
 	"github.com/avenga/couper/eval"
 	"github.com/avenga/couper/handler/middleware"
-	probe "github.com/avenga/couper/handler/transport/probe_map"
 	"github.com/avenga/couper/logging"
 )
 
@@ -43,6 +42,13 @@ func (s state) String() string {
 	return healthStateLabels[s]
 }
 
+type HealthInfo struct {
+	Error   string
+	Healthy bool
+	Origin  string
+	State   string
+}
+
 type Probe struct {
 	//configurable settings
 	backendName string
@@ -56,34 +62,55 @@ type Probe struct {
 	state   state
 	status  int
 
+	listener ProbeStateChange
+
 	uidFunc middleware.UIDFunc
 }
 
-func NewProbe(log *logrus.Entry, backendName string, opts *config.HealthCheck) {
+type ProbeStateChange interface {
+	OnProbeChange(info *HealthInfo)
+}
+
+func NewProbe(log *logrus.Entry, tc *Config, opts *config.HealthCheck, listener ProbeStateChange) {
 	client := &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
-		Transport: logging.NewUpstreamLog(log, http.DefaultTransport,
-			false), // always false due to defaultTransport
+		Transport: logging.NewUpstreamLog(log,
+			NewTransport(tc.
+				WithTarget(opts.Request.URL.Scheme, opts.Request.URL.Host, opts.Request.URL.Host, ""),
+				log),
+			tc.NoProxyFromEnv),
 	}
 
 	p := &Probe{
-		backendName: backendName,
-		log:         log,
+		backendName: tc.BackendName,
+		log:         log.WithField("url", opts.Request.URL.String()),
 		opts:        opts,
 
 		client: client,
 		state:  StateInvalid,
 
+		listener: listener,
+
 		uidFunc: middleware.NewUIDFunc(opts.RequestUIDFormat),
 	}
 
-	go p.probe()
+	// do not start go-routine on config check (-watch)
+	if _, exist := opts.Context.Value("config-dry-run").(bool); exist {
+		return
+	}
+	go p.probe(opts.Context)
 }
 
-func (p *Probe) probe() {
+func (p *Probe) probe(c context.Context) {
 	for {
+		select {
+		case <-c.Done():
+			p.log.Warn("shutdown health probe")
+			return
+		default:
+		}
 		ctx, cancel := context.WithTimeout(context.Background(), p.opts.Timeout)
 		ctx = context.WithValue(ctx, request.RoundTripName, "health-check")
 		uid := p.uidFunc()
@@ -131,11 +158,16 @@ func (p *Probe) probe() {
 
 		if prevState != p.state {
 			newState := p.state.String()
-			probe.BackendProbes.Store(p.backendName, probe.HealthInfo{
-				State:   newState,
+			info := &HealthInfo{
 				Error:   errorMessage,
 				Healthy: p.state != StateDown,
-			})
+				Origin:  p.opts.Request.URL.Host,
+				State:   newState,
+			}
+
+			if p.listener != nil {
+				p.listener.OnProbeChange(info)
+			}
 
 			message := fmt.Sprintf("new health state: %s", newState)
 
diff --git a/handler/transport/probe_map/probe_map.go b/handler/transport/probe_map/probe_map.go
deleted file mode 100644
index 47022f57..00000000
--- a/handler/transport/probe_map/probe_map.go
+++ /dev/null
@@ -1,11 +0,0 @@
-package probe_map
-
-import "sync"
-
-var BackendProbes sync.Map
-
-type HealthInfo struct {
-	State   string
-	Healthy bool
-	Error   string
-}
diff --git a/handler/transport/transport.go b/handler/transport/transport.go
index 3201c03f..45c78e0c 100644
--- a/handler/transport/transport.go
+++ b/handler/transport/transport.go
@@ -64,12 +64,12 @@ func NewTransport(conf *Config, log *logrus.Entry) *http.Transport {
 	var proxyFunc func(req *http.Request) (*url.URL, error)
 	if conf.Proxy != "" {
 		proxyFunc = func(req *http.Request) (*url.URL, error) {
-			config := &httpproxy.Config{
+			proxyConf := &httpproxy.Config{
 				HTTPProxy:  conf.Proxy,
 				HTTPSProxy: conf.Proxy,
 			}
 
-			return config.ProxyFunc()(req.URL)
+			return proxyConf.ProxyFunc()(req.URL)
 		}
 	} else if !conf.NoProxyFromEnv {
 		proxyFunc = http.ProxyFromEnvironment
diff --git a/internal/seetie/object.go b/internal/seetie/object.go
new file mode 100644
index 00000000..c9ec63fe
--- /dev/null
+++ b/internal/seetie/object.go
@@ -0,0 +1,7 @@
+package seetie
+
+import "github.com/zclconf/go-cty/cty"
+
+type Object interface {
+	Value() cty.Value
+}
diff --git a/logging/upstream_log.go b/logging/upstream_log.go
index 6ef8fff5..6e3888bc 100644
--- a/logging/upstream_log.go
+++ b/logging/upstream_log.go
@@ -10,15 +10,20 @@ import (
 	"time"
 
 	"github.com/sirupsen/logrus"
+	"github.com/zclconf/go-cty/cty"
 
 	"github.com/avenga/couper/config/env"
 	"github.com/avenga/couper/config/request"
 	"github.com/avenga/couper/errors"
 	"github.com/avenga/couper/handler/validation"
+	"github.com/avenga/couper/internal/seetie"
 	"github.com/hashicorp/hcl/v2"
 )
 
-var _ http.RoundTripper = &UpstreamLog{}
+var (
+	_ http.RoundTripper = &UpstreamLog{}
+	_ seetie.Object     = &UpstreamLog{}
+)
 
 type UpstreamLog struct {
 	config *Config
@@ -245,3 +250,12 @@ func (u *UpstreamLog) newTraceContext() (Fields, *sync.RWMutex, *httptrace.Clien
 
 	return timings, mapMu, trace
 }
+
+func (u *UpstreamLog) Value() cty.Value {
+	next, ok := u.next.(seetie.Object)
+	if !ok {
+		return cty.NilVal
+	}
+
+	return next.Value()
+}
diff --git a/main.go b/main.go
index 382d019d..0aeced15 100644
--- a/main.go
+++ b/main.go
@@ -27,6 +27,7 @@ import (
 	"github.com/avenga/couper/config/configload"
 	"github.com/avenga/couper/config/env"
 	"github.com/avenga/couper/config/runtime"
+	"github.com/avenga/couper/eval"
 	"github.com/avenga/couper/logging"
 	"github.com/avenga/couper/logging/hooks"
 	"github.com/avenga/couper/utils"
@@ -211,8 +212,15 @@ func realmain(arguments []string) int {
 			// dry run configuration
 			tmpStoreCh := make(chan struct{})
 			tmpMemStore := cache.New(logger, tmpStoreCh)
+
+			dryCtx, cancelDry := context.
+				WithCancel(context.WithValue(context.Background(), "config-dry-run", true))
+			cf.Context = cf.Context.(*eval.Context).WithContext(dryCtx)
+
 			_, reloadErr = runtime.NewServerConfiguration(cf, logger.WithFields(fields), tmpMemStore)
 			close(tmpStoreCh)
+			cancelDry()
+
 			if reloadErr != nil {
 				logger.WithError(reloadErr).Error("reload failed")
 				time.Sleep(flags.FileWatchRetryDelay)
diff --git a/telemetry/exporter.go b/telemetry/exporter.go
index 2a7efd67..7d4a00e2 100644
--- a/telemetry/exporter.go
+++ b/telemetry/exporter.go
@@ -27,6 +27,7 @@ import (
 	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 	"google.golang.org/grpc"
 
+	"github.com/avenga/couper/cache"
 	"github.com/avenga/couper/telemetry/provider"
 	"github.com/avenga/couper/utils"
 )
@@ -42,7 +43,7 @@ const (
 const otlpExporterEnvKey = "OTEL_EXPORTER_OTLP_ENDPOINT"
 
 // InitExporter initialises configured metrics and/or trace exporter.
-func InitExporter(ctx context.Context, opts *Options, logEntry *logrus.Entry) {
+func InitExporter(ctx context.Context, opts *Options, memStore *cache.MemoryStore, logEntry *logrus.Entry) {
 	log := logEntry.WithField("type", "couper_telemetry")
 	otel.SetErrorHandler(ErrorHandleFunc(func(e error) { // configure otel to use our logger for error handling
 		if e != nil {
@@ -54,6 +55,8 @@ func InitExporter(ctx context.Context, opts *Options, logEntry *logrus.Entry) {
 	if opts.Metrics {
 		wg.Add(1)
 		otel.Handle(initMetricExporter(ctx, opts, log, wg))
+
+		newBackendsObserver(memStore)
 	}
 
 	if opts.Traces {
diff --git a/telemetry/instrumentation/instrumentation.go b/telemetry/instrumentation/instrumentation.go
index 2725d295..545bb9bb 100644
--- a/telemetry/instrumentation/instrumentation.go
+++ b/telemetry/instrumentation/instrumentation.go
@@ -4,9 +4,12 @@ const (
 	Name   = "github.com/avenga/couper/telemetry"
 	Prefix = "couper_"
 
+	BackendInstrumentationName = "couper/backend"
+
 	BackendConnections         = Prefix + "backend_connections_count"
 	BackendConnectionsLifetime = Prefix + "backend_connections_lifetime_seconds"
 	BackendConnectionsTotal    = Prefix + "backend_connections_total"
+	BackendHealthState         = Prefix + "backend_up"
 	BackendRequest             = Prefix + "backend_request_total"
 	BackendRequestDuration     = Prefix + "backend_request_duration_seconds"
 	ClientConnections          = Prefix + "client_connections_count"
diff --git a/telemetry/observer.go b/telemetry/observer.go
new file mode 100644
index 00000000..715bdbfd
--- /dev/null
+++ b/telemetry/observer.go
@@ -0,0 +1,49 @@
+package telemetry
+
+import (
+	"context"
+
+	"github.com/zclconf/go-cty/cty"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+
+	"github.com/avenga/couper/cache"
+	"github.com/avenga/couper/telemetry/instrumentation"
+	"github.com/avenga/couper/telemetry/provider"
+)
+
+func newBackendsObserver(memStore *cache.MemoryStore) {
+	bs := memStore.GetAllWithPrefix("backend_")
+	var backends []interface{ Value() cty.Value }
+	for _, b := range bs {
+		if backend, ok := b.(interface{ Value() cty.Value }); ok {
+			backends = append(backends, backend)
+		}
+	}
+
+	onObserverFn := func(_ context.Context, result metric.Int64ObserverResult) {
+		backendsObserver(backends, result)
+	}
+
+	meter := provider.Meter(instrumentation.BackendInstrumentationName)
+	metric.Must(meter).
+		NewInt64GaugeObserver(instrumentation.BackendHealthState, onObserverFn)
+}
+
+func backendsObserver(backends []interface{ Value() cty.Value }, result metric.Int64ObserverResult) {
+	for _, backend := range backends {
+		v := backend.Value().AsValueMap()
+		attrs := []attribute.KeyValue{
+			attribute.String("backend_name", v["name"].AsString()),
+			attribute.String("hostname", v["hostname"].AsString()),
+			attribute.String("origin", v["origin"].AsString()),
+		}
+		var value int64 = 1 // default healthy due to anonymous ones
+		health := v["health"].AsValueMap()
+		if health["healthy"].False() {
+			value = 0
+		}
+
+		result.Observe(value, attrs...)
+	}
+}


Test patch:
diff --git a/command/run_test.go b/command/run_test.go
index b035894f..86f3a2ee 100644
--- a/command/run_test.go
+++ b/command/run_test.go
@@ -11,6 +11,8 @@ import (
 	"reflect"
 	"runtime"
 	"strconv"
+	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -253,7 +255,6 @@ func TestArgs_CAFile(t *testing.T) {
 	}
 
 	expiresIn := time.Minute
-	//expires := time.After(expiresIn)
 	selfSigned, err := server.NewCertificate(expiresIn, nil, nil)
 	helper.Must(err)
 
@@ -264,7 +265,12 @@ func TestArgs_CAFile(t *testing.T) {
 	helper.Must(tmpFile.Close())
 	defer os.Remove(tmpFile.Name())
 
+	var healthCheckSeen uint32
+
 	srv := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if strings.HasSuffix(request.Header.Get("User-Agent"), "health-check") {
+			atomic.StoreUint32(&healthCheckSeen, 1)
+		}
 		writer.WriteHeader(http.StatusNoContent)
 
 		// force close to trigger a new handshake
@@ -286,7 +292,16 @@ func TestArgs_CAFile(t *testing.T) {
 	couperHCL := `server {
 	endpoint "/" {
 		request {
-			url = "` + srv.URL + `"
+			backend = "tls"
+		}
+	}
+}
+
+definitions {
+	backend "tls" {
+		origin = "` + srv.URL + `"
+		beta_health {
+			failure_threshold = 0
 		}
 	}
 }`
@@ -318,16 +333,9 @@ func TestArgs_CAFile(t *testing.T) {
 		t.Error("unexpected status code")
 	}
 
-	// ca after
-	//<-expires
-
-	// handshake error
-	//res, err = client.Do(req)
-	//helper.Must(err)
-	//
-	//if res.StatusCode != http.StatusBadGateway {
-	//	t.Error("unexpected status code")
-	//}
+	if atomic.LoadUint32(&healthCheckSeen) != 1 {
+		t.Error("expected a successful tls health check")
+	}
 }
 
 func TestCAFile_Run(t *testing.T) {
diff --git a/handler/transport/backend_test.go b/handler/transport/backend_test.go
index 1ab1e8d2..1ef894c4 100644
--- a/handler/transport/backend_test.go
+++ b/handler/transport/backend_test.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
-	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -13,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
 	"github.com/hashicorp/hcl/v2/hcltest"
@@ -358,29 +358,21 @@ func TestBackend_director(t *testing.T) {
 }
 
 func TestBackend_HealthCheck(t *testing.T) {
-	type expectation struct {
-		FailureThreshold uint
-		Interval         time.Duration
-		Timeout          time.Duration
-		ExpectedStatus   map[int]bool
-		ExpectedText     string
-		URL              *url.URL
-		RequestUIDFormat string
-	}
-
 	type testCase struct {
 		name        string
 		health      *config.Health
-		expectation expectation
+		expectation config.HealthCheck
 	}
 
 	defaultExpectedStatus := map[int]bool{200: true, 204: true, 301: true}
 
+	toPtr := func(n uint) *uint { return &n }
+
 	for _, tc := range []testCase{
 		{
 			name:   "health check with default values",
 			health: &config.Health{},
-			expectation: expectation{
+			expectation: config.HealthCheck{
 				FailureThreshold: 2,
 				Interval:         time.Second,
 				Timeout:          time.Second,
@@ -392,32 +384,32 @@ func TestBackend_HealthCheck(t *testing.T) {
 		{
 			name: "health check with configured values",
 			health: &config.Health{
-				FailureThreshold: 42,
+				FailureThreshold: toPtr(42),
 				Interval:         "1h",
 				Timeout:          "9m",
 				Path:             "/gsund??",
 				ExpectedStatus:   []int{418},
 				ExpectedText:     "roger roger",
 			},
-			expectation: expectation{
+			expectation: config.HealthCheck{
 				FailureThreshold: 42,
 				Interval:         time.Hour,
 				Timeout:          9 * time.Minute,
 				ExpectedStatus:   map[int]bool{418: true},
 				ExpectedText:     "roger roger",
-				URL: &url.URL{
+				Request: &http.Request{URL: &url.URL{
 					Scheme:   "http",
 					Host:     "origin:8080",
 					Path:     "/gsund",
 					RawQuery: "?",
-				},
+				}},
 				RequestUIDFormat: "common",
 			},
 		},
 		{
 			name:   "uninitialised health check",
 			health: nil,
-			expectation: expectation{
+			expectation: config.HealthCheck{
 				FailureThreshold: 2,
 				Interval:         time.Second,
 				Timeout:          time.Second,
@@ -431,7 +423,7 @@ func TestBackend_HealthCheck(t *testing.T) {
 			health: &config.Health{
 				Interval: "10s",
 			},
-			expectation: expectation{
+			expectation: config.HealthCheck{
 				FailureThreshold: 2,
 				Interval:         10 * time.Second,
 				Timeout:          10 * time.Second,
@@ -446,7 +438,7 @@ func TestBackend_HealthCheck(t *testing.T) {
 				Interval: "5s",
 				Timeout:  "10s",
 			},
-			expectation: expectation{
+			expectation: config.HealthCheck{
 				FailureThreshold: 2,
 				Interval:         5 * time.Second,
 				Timeout:          5 * time.Second,
@@ -458,10 +450,10 @@ func TestBackend_HealthCheck(t *testing.T) {
 		{
 			name: "zero threshold",
 			health: &config.Health{
-				FailureThreshold: 0,
+				FailureThreshold: toPtr(0),
 			},
-			expectation: expectation{
-				FailureThreshold: 2,
+			expectation: config.HealthCheck{
+				FailureThreshold: 0,
 				Interval:         time.Second,
 				Timeout:          time.Second,
 				ExpectedStatus:   defaultExpectedStatus,
@@ -473,19 +465,23 @@ func TestBackend_HealthCheck(t *testing.T) {
 		t.Run(tc.name, func(subT *testing.T) {
 			h := test.New(subT)
 
-			health, err := config.NewHealthCheck("http://origin:8080/foo", tc.health, &config.DefaultSettings)
+			health, err := config.
+				NewHealthCheck("http://origin:8080/foo", tc.health, &config.Couper{
+					Settings: &config.DefaultSettings,
+				})
 			h.Must(err)
 
-			if tc.expectation.URL != nil {
-				if *tc.expectation.URL != *health.Request.URL {
-					t.Errorf("Unexpected health check URI:\n\tWant: %#v\n\tGot:  %#v", tc.expectation.URL, health.Request.URL)
+			if tc.expectation.Request != nil && tc.expectation.Request.URL != nil {
+				if *tc.expectation.Request.URL != *health.Request.URL {
+					t.Errorf("Unexpected health check URI:\n\tWant: %#v\n\tGot:  %#v", tc.expectation.Request.URL, health.Request.URL)
 				}
-				tc.expectation.URL = nil
+				tc.expectation.Request = nil
 			}
+
 			health.Request = nil
 
-			if fmt.Sprint(tc.expectation) != fmt.Sprint(*health) {
-				t.Errorf("Unexpected health options:\n\tWant: %v\n\tGot:  %v", tc.expectation, *health)
+			if diff := cmp.Diff(tc.expectation, *health); diff != "" {
+				t.Errorf("Unexpected health options:\n\n%s", diff)
 			}
 		})
 	}
diff --git a/server/http_backend_test.go b/server/http_backend_test.go
index bf3b343b..a902f18e 100644
--- a/server/http_backend_test.go
+++ b/server/http_backend_test.go
@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -272,3 +273,61 @@ func TestBackend_LogResponseBytes(t *testing.T) {
 		}
 	}
 }
+
+func TestBackend_Unhealthy(t *testing.T) {
+	helper := test.New(t)
+
+	var unhealthy int64
+	origin := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		if counter := r.Header.Get("Counter"); counter != "" {
+			c, _ := strconv.Atoi(counter)
+			if c > 2 {
+				atomic.StoreInt64(&unhealthy, 1)
+			}
+		}
+		if atomic.LoadInt64(&unhealthy) == 1 {
+			rw.WriteHeader(http.StatusConflict)
+		} else {
+			rw.WriteHeader(http.StatusNoContent)
+			time.Sleep(time.Second / 3)
+		}
+	}))
+
+	defer origin.Close()
+
+	shutdown, _ := newCouperWithTemplate("testdata/integration/backends/06_couper.hcl", helper,
+		map[string]interface{}{
+			"origin": origin.URL,
+		})
+	defer shutdown()
+
+	client := test.NewHTTPClient()
+
+	type testcase struct {
+		path      string
+		expStatus int
+	}
+
+	for i, tc := range []testcase{
+		{"/anon", http.StatusNoContent},
+		{"/ref", http.StatusNoContent},
+		{"/catch", http.StatusNoContent},
+		// server switched resp status-code -> unhealthy
+		{"/anon", http.StatusConflict}, // always healthy
+		{"/ref", http.StatusBadGateway},
+		{"/catch", http.StatusTeapot},
+	} {
+		t.Run(tc.path, func(st *testing.T) {
+			h := test.New(st)
+			req, err := http.NewRequest(http.MethodGet, "http://couper.dev:8080"+tc.path, nil)
+			h.Must(err)
+			req.Header.Set("Counter", strconv.Itoa(i))
+			res, err := client.Do(req)
+			h.Must(err)
+
+			if res.StatusCode != tc.expStatus {
+				st.Errorf("want status %d, got: %d", tc.expStatus, res.StatusCode)
+			}
+		})
+	}
+}
diff --git a/server/http_integration_test.go b/server/http_integration_test.go
index d80b5252..30cf5c81 100644
--- a/server/http_integration_test.go
+++ b/server/http_integration_test.go
@@ -2703,7 +2703,7 @@ func TestHTTPServer_BackendProbes(t *testing.T) {
 		{
 			"unhealthy backend: timeout",
 			"/unhealthy/timeout",
-			`{"error":"backend error: proxyconnect tcp: dial tcp 127.0.0.1:9999: connect: connection refused","healthy":false,"state":"unhealthy"}`,
+			`{"error":"backend error: context deadline exceeded","healthy":false,"state":"unhealthy"}`,
 		},
 		{
 			"unhealthy backend: unexpected status code",
@@ -2743,7 +2743,7 @@ func TestHTTPServer_BackendProbes(t *testing.T) {
 		{
 			"backend error: timeout but threshold not reached",
 			"/failing",
-			`{"error":"backend error: proxyconnect tcp: dial tcp 127.0.0.1:9999: connect: connection refused","healthy":true,"state":"failing"}`,
+			`{"error":"backend error: context deadline exceeded","healthy":true,"state":"failing"}`,
 		},
 	} {
 		t.Run(tc.name, func(subT *testing.T) {
diff --git a/server/http_telemetry_test.go b/server/http_telemetry_test.go
index 95972c57..c1795fcc 100644
--- a/server/http_telemetry_test.go
+++ b/server/http_telemetry_test.go
@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/avenga/couper/config"
 	"github.com/avenga/couper/internal/test"
@@ -22,19 +23,19 @@ func TestServeMetrics(t *testing.T) {
 		fmt.Sprintf("http://localhost:%d/metrics", config.DefaultSettings.TelemetryMetricsPort), nil)
 	helper.Must(err)
 
-	clientReq, err := http.NewRequest(http.MethodGet, "http://localhost:8080/", nil)
-	helper.Must(err)
+	paths := []string{"/", "/down", "/notfound"}
+	for _, path := range paths {
+		clientReq, perr := http.NewRequest(http.MethodGet, "http://localhost:8080"+path, nil)
+		helper.Must(perr)
 
-	res, err := client.Do(clientReq)
-	helper.Must(err)
-	helper.Must(res.Body.Close())
+		res, rerr := client.Do(clientReq)
+		helper.Must(rerr)
+		helper.Must(res.Body.Close())
+	}
 
-	clientReq.URL.Path = "/notfound"
-	res, err = client.Do(clientReq)
-	helper.Must(err)
-	helper.Must(res.Body.Close())
+	time.Sleep(time.Second * 2)
 
-	res, err = client.Do(mreq)
+	res, err := client.Do(mreq)
 	if err != nil {
 		t.Fatalf("metrics endpoint could not be reached: %v", err)
 	}
@@ -44,21 +45,18 @@ func TestServeMetrics(t *testing.T) {
 
 	// due to random client remote port
 	result := regexp.MustCompile(`127\.0\.0\.1:\d{5}`).ReplaceAll(b, []byte("127.0.0.1"))
-	defer func() {
-		if t.Failed() {
-			println(string(result))
-		}
-	}()
 
 	expMetrics := []string{
 		`couper_backend_request_duration_seconds_count{backend_name="anything",code="200",hostname="127.0.0.1",method="GET",origin="127.0.0.1",service_name="my-service",service_version="0"} 1`,
 		`couper_backend_request_total{backend_name="anything",code="200",hostname="127.0.0.1",method="GET",origin="127.0.0.1",service_name="my-service",service_version="0"} 1`,
+		`couper_backend_up{backend_name="anything",hostname="127.0.0.1",origin="127.0.0.1",service_name="my-service",service_version="0"} 1`,
+		`couper_backend_up{backend_name="not_healthy",hostname="1.2.3.4",origin="1.2.3.4:80",service_name="my-service",service_version="0"} 0`,
 		`couper_client_request_duration_seconds_count{code="200",host="localhost:8080",method="GET",service_name="my-service",service_version="0"} 1`,
 		`couper_client_request_duration_seconds_count{code="404",host="localhost:8080",method="GET",service_name="my-service",service_version="0"} 1`,
 		`couper_client_request_total{code="200",host="localhost:8080",method="GET",service_name="my-service",service_version="0"} 1`,
 		`couper_client_request_total{code="404",host="localhost:8080",method="GET",service_name="my-service",service_version="0"} 1`,
 		`couper_client_request_error_types_total{error="route_not_found_error",service_name="my-service",service_version="0"} 1`,
-		`couper_client_connections_total{service_name="my-service",service_version="0"} 2`,
+		`couper_client_connections_total{service_name="my-service",service_version="0"} 3`,
 		`go_goroutines{service_name="my-service"}`,
 	}
 
@@ -67,4 +65,8 @@ func TestServeMetrics(t *testing.T) {
 			t.Errorf("missing metric: %s", expMetric)
 		}
 	}
+
+	if t.Failed() {
+		println(string(result))
+	}
 }
diff --git a/server/testdata/integration/backends/06_couper.hcl b/server/testdata/integration/backends/06_couper.hcl
new file mode 100644
index 00000000..23f14330
--- /dev/null
+++ b/server/testdata/integration/backends/06_couper.hcl
@@ -0,0 +1,35 @@
+server {
+  endpoint "/anon" {
+    proxy {
+      url = "{{ .origin }}/"
+    }
+  }
+
+  endpoint "/ref" {
+    proxy {
+      backend = "be"
+    }
+  }
+
+  endpoint "/catch" {
+    proxy {
+      backend = "be"
+    }
+
+    error_handler "backend_unhealthy" {
+      response {
+        status = 418
+      }
+    }
+  }
+}
+
+definitions {
+  backend "be" {
+    origin = "{{ .origin }}"
+    beta_health {
+      expected_status = [204]
+      interval = "250ms"
+    }
+  }
+}
diff --git a/server/testdata/integration/config/14_couper.hcl b/server/testdata/integration/config/14_couper.hcl
index 944b4b58..0b3fd927 100644
--- a/server/testdata/integration/config/14_couper.hcl
+++ b/server/testdata/integration/config/14_couper.hcl
@@ -180,7 +180,10 @@ definitions {
   }
   backend "unhealthy_timeout" {
     origin = "http://1.2.3.4"
-    beta_health {}
+    beta_health {
+      failure_threshold = 0
+      interval = "250ms"
+    }
   }
   backend "unhealthy_bad_status" {
     origin = "${env.COUPER_TEST_BACKEND_ADDR}"
@@ -221,7 +224,12 @@ definitions {
   backend "failing" {
     origin = "http://1.2.3.4"
     beta_health {
-      failure_threshold = 4
+      failure_threshold = 10
+      interval = "250ms"
     }
   }
 }
+
+settings {
+  no_proxy_from_env = true
+}
diff --git a/server/testdata/integration/telemetry/01_couper.hcl b/server/testdata/integration/telemetry/01_couper.hcl
index aee0a57f..10e4fda8 100644
--- a/server/testdata/integration/telemetry/01_couper.hcl
+++ b/server/testdata/integration/telemetry/01_couper.hcl
@@ -4,6 +4,12 @@ server "metrics" {
       backend = "anything"
     }
   }
+
+  endpoint "/down" {
+    proxy {
+      backend = "not_healthy"
+    }
+  }
 }
 
 
@@ -12,10 +18,22 @@ definitions {
   backend "anything" {
     path = "/anything"
     origin = env.COUPER_TEST_BACKEND_ADDR
+    beta_health {
+      path = "/small"
+    }
+  }
+
+  backend "not_healthy" {
+    origin = "http://1.2.3.4"
+    timeout = "2s"
+    beta_health {
+      timeout = "250ms"
+    }
   }
 }
 
 settings {
   beta_metrics = true
   beta_service_name = "my-service"
+  no_proxy_from_env = true
 }