Repository: cilium/cilium (go)

Issue:
Metrics are reporting unreachable nodes when Cilium starts up
The connectivity health checking metrics are reporting unreachable nodes in the beginning of the bootstrapping process. This should be avoided to avoid spikes in the metric when new nodes are being added.

Core issue: We don’t have a difference between “unknown” / “unreachable”.

Pull request:
health: Differentiate between unknown and unreachable state
Before this change, adding a new node with Cilium to the Kubernetes
cluster resulted in the new node being reported as unreachable in
cilium-health. That's because before Cilium gets fully provisions, the
node status is going to contain empty status (for HTTP probe).

That behavior was increasing the unreachable nodes Prometheus metric,
thus false alarming Cilium users and suggesting, that the provisioning
of the new node failed.

This change differentiates between unreachable and unhealthy status. In
order to be considered as unreachable, the status has to contain an
explicit error message, which is always set by the prober when it really
has problems with reaching already provisioned Cilium endpoint.
Otherwise its status is considered to be uknkown, which usually should
mean that the node is being provisioned.

Fixes #11874

Signed-off-by: Michal Rostecki <mrostecki@opensuse.org>

```release-note
health: Differentiate between unknown and unreachable state
```

Outputs **Before the fix** when the 2nd node is provisioning:

```
vagrant@k8s1:~/go/src/github.com/cilium/cilium$ cilium metrics list | grep nodes
cilium_nodes_all_datapath_validations_total                                         37.000000
cilium_nodes_all_events_received_total        eventType="add" source="kvstore"      1.000000
cilium_nodes_all_events_received_total        eventType="add" source="local"        1.000000
cilium_nodes_all_events_received_total        source="kvstore" eventType="update"   4.000000
cilium_nodes_all_num                                                                2.000000
cilium_unreachable_nodes                                                            1.000000
vagrant@k8s1:~/go/src/github.com/cilium/cilium$ cilium-health status --succinct
Cluster health:      1/2 reachable   (2020-07-16T11:12:45Z)
  Name               IP              Reachable   Endpoints reachable
  k8s1 (localhost)   192.168.33.11   true        true
  k8s2               192.168.33.12   false       false
```

Outputs **after the fix** when the 2nd node is provisioning:

```
vagrant@k8s1:~/go/src/github.com/cilium/cilium$ cilium metrics list | grep nodes
cilium_nodes_all_datapath_validations_total                                         43.000000
cilium_nodes_all_events_received_total        eventType="add" source="kvstore"      1.000000
cilium_nodes_all_events_received_total        eventType="add" source="local"        1.000000
cilium_nodes_all_events_received_total        source="kvstore" eventType="update"   4.000000
cilium_nodes_all_num                                                                2.000000
cilium_unreachable_nodes                                                            0.000000
vagrant@k8s1:~/go/src/github.com/cilium/cilium$ cilium-health status --succinct
Cluster health:      2/2 reachable   (2020-07-16T11:48:37Z)
  Name               IP              Node        Endpoints
  k8s1 (localhost)   192.168.33.11   reachable   reachable
  k8s2               192.168.33.12   unknown     unknown
```

Patch:
diff --git a/Documentation/cmdref/cilium-agent.md b/Documentation/cmdref/cilium-agent.md
index 0946c91301..9375a72377 100644
--- a/Documentation/cmdref/cilium-agent.md
+++ b/Documentation/cmdref/cilium-agent.md
@@ -52,7 +52,7 @@ cilium-agent [flags]
       --debug-verbose strings                         List of enabled verbose debug groups
       --devices strings                               List of devices facing cluster/external network (used for BPF NodePort, BPF masquerading and host firewall)
       --direct-routing-device string                  Device name used to connect nodes in direct routing mode (required only by BPF NodePort; if empty, automatically set to a device with k8s InternalIP/ExternalIP or with a default route)
-      --disable-cnp-status-updates                    Do not send CNP NodeStatus updates to the Kubernetes api-server (recommended to run with "cnp-node-status-gc=false" in cilium-operator)
+      --disable-cnp-status-updates                    Do not send CNP NodeStatus updates to the Kubernetes api-server (recommended to run with "cnp-node-status-gc-interval=0" in cilium-operator)
       --disable-conntrack                             Disable connection tracking
       --disable-endpoint-crd                          Disable use of CiliumEndpoint CRD
       --disable-iptables-feeder-rules strings         Chains to ignore when installing feeder rules.
diff --git a/Documentation/configuration/metrics.rst b/Documentation/configuration/metrics.rst
index 9f949d9b5a..de49b3b1a8 100644
--- a/Documentation/configuration/metrics.rst
+++ b/Documentation/configuration/metrics.rst
@@ -352,7 +352,8 @@ Configuration
 
 ``cilium-operator`` can be configured to serve metrics by running with the
 option ``--enable-metrics``.  By default, the operator will expose metrics on
-port 6942, the port can be changed with the option ``--metrics-address``.
+port 6942, the port can be changed with the option
+``--operator-prometheus-serve-addr``.
 
 Exported Metrics
 ^^^^^^^^^^^^^^^^
diff --git a/Documentation/install/upgrade.rst b/Documentation/install/upgrade.rst
index ace398a860..cb42bdc5cc 100644
--- a/Documentation/install/upgrade.rst
+++ b/Documentation/install/upgrade.rst
@@ -339,8 +339,6 @@ New Metrics
   * ``cilium_kvstore_quorum_errors_total`` counts the number of kvstore quorum
     loss errors. The label ``error`` indicates the type of error.
 
-.. _1.8_upgrade_notes:
-
 Removed options
 ~~~~~~~~~~~~~~~
 
@@ -349,6 +347,32 @@ Removed options
 * The ``prometheus-serve-addr-deprecated`` option is now removed. Please use
   ``prometheus-serve-addr`` instead.
 
+Removed cilium-operator options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* The options ``cnp-node-status-gc`` and ``ccnp-node-status-gc`` are now
+  removed. Please use ``cnp-node-status-gc-interval=0`` instead.
+
+* The ``cilium-endpoint-gc`` option is now removed. Please use
+  ``cilium-endpoint-gc-interval=0`` instead.
+
+* The ``eni-parallel-workers`` option is now removed. Please use
+  ``parallel-alloc-workers`` instead.
+
+* The ``aws-client-burst`` option is now removed. Please use
+  ``limit-ipam-api-burst`` instead.
+
+* The ``aws-client-qps`` option is now removed. Please use
+  ``limit-ipam-api-qps`` instead.
+
+* The ``api-server-port`` option is now removed. Please use
+  ``operator-api-serve-addr`` instead.
+
+* The ``metrics-address`` option is now removed. Please use
+  ``operator-prometheus-serve-addr`` instead.
+
+.. _1.8_upgrade_notes:
+
 1.8 Upgrade Notes
 -----------------
 
diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go
index 3fb0c3cd0e..5ef7f5c65b 100644
--- a/daemon/cmd/daemon_main.go
+++ b/daemon/cmd/daemon_main.go
@@ -772,7 +772,7 @@ func init() {
 	flags.MarkHidden(option.PolicyTriggerInterval)
 	option.BindEnv(option.PolicyTriggerInterval)
 
-	flags.Bool(option.DisableCNPStatusUpdates, false, `Do not send CNP NodeStatus updates to the Kubernetes api-server (recommended to run with "cnp-node-status-gc=false" in cilium-operator)`)
+	flags.Bool(option.DisableCNPStatusUpdates, false, `Do not send CNP NodeStatus updates to the Kubernetes api-server (recommended to run with "cnp-node-status-gc-interval=0" in cilium-operator)`)
 	option.BindEnv(option.DisableCNPStatusUpdates)
 
 	flags.Bool(option.PolicyAuditModeArg, false, "Enable policy audit (non-drop) mode")
diff --git a/operator/flags.go b/operator/flags.go
index 2b3020f85b..8882cc59e5 100644
--- a/operator/flags.go
+++ b/operator/flags.go
@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"time"
 
-	operatorMetrics "github.com/cilium/cilium/operator/metrics"
 	operatorOption "github.com/cilium/cilium/operator/option"
 	"github.com/cilium/cilium/pkg/defaults"
 	ipamOption "github.com/cilium/cilium/pkg/ipam/option"
@@ -33,15 +32,9 @@ func init() {
 
 	flags := rootCmd.Flags()
 
-	flags.Int(operatorOption.AWSClientBurstDeprecated, defaults.IPAMAPIBurst, "")
-	flags.MarkDeprecated(operatorOption.AWSClientBurstDeprecated, fmt.Sprintf("please use --%s", operatorOption.IPAMAPIBurst))
-
 	flags.Int(operatorOption.IPAMAPIBurst, defaults.IPAMAPIBurst, "Upper burst limit when accessing external APIs")
 	option.BindEnv(operatorOption.IPAMAPIBurst)
 
-	flags.Float64(operatorOption.AWSClientQPSLimitDeprecated, defaults.IPAMAPIQPSLimit, "")
-	flags.MarkDeprecated(operatorOption.AWSClientQPSLimitDeprecated, fmt.Sprintf("please use --%s", operatorOption.IPAMAPIQPSLimit))
-
 	flags.Float64(operatorOption.IPAMAPIQPSLimit, defaults.IPAMAPIQPSLimit, "Queries per second limit when accessing external IPAM APIs")
 	option.BindEnv(operatorOption.IPAMAPIQPSLimit)
 
@@ -53,10 +46,6 @@ func init() {
 		"Subnets IDs (separated by commas)")
 	option.BindEnv(operatorOption.IPAMSubnetsIDs)
 
-	flags.Int64(operatorOption.ENIParallelWorkersDeprecated, defaults.ParallelAllocWorkers, "")
-	flags.MarkDeprecated(operatorOption.ENIParallelWorkersDeprecated, fmt.Sprintf("please use --%s", operatorOption.ParallelAllocWorkers))
-	option.BindEnv(operatorOption.ENIParallelWorkersDeprecated)
-
 	flags.Int64(operatorOption.ParallelAllocWorkers, defaults.ParallelAllocWorkers, "Maximum number of parallel IPAM workers")
 	option.BindEnv(operatorOption.ParallelAllocWorkers)
 
@@ -74,16 +63,6 @@ func init() {
 	flags.String(option.ConfigDir, "", `Configuration directory that contains a file for each option`)
 	option.BindEnv(option.ConfigDir)
 
-	// Deprecated, remove in 1.9
-	flags.Bool(operatorOption.EnableCCNPNodeStatusGC, true, "Enable CiliumClusterwideNetworkPolicy Status garbage collection for nodes which have been removed from the cluster")
-	option.BindEnv(operatorOption.EnableCCNPNodeStatusGC)
-	flags.MarkDeprecated(operatorOption.EnableCCNPNodeStatusGC, fmt.Sprintf("Please use %s=0 to disable CCNP Status GC", operatorOption.CNPNodeStatusGCInterval))
-
-	// Deprecated, remove in 1.9
-	flags.Bool(operatorOption.EnableCNPNodeStatusGC, true, "Enable CiliumNetworkPolicy Status garbage collection for nodes which have been removed from the cluster")
-	option.BindEnv(operatorOption.EnableCNPNodeStatusGC)
-	flags.MarkDeprecated(operatorOption.EnableCNPNodeStatusGC, fmt.Sprintf("Please use %s=0 to disable CNP Status GC", operatorOption.CNPNodeStatusGCInterval))
-
 	flags.Duration(operatorOption.CNPNodeStatusGCInterval, 2*time.Minute, "GC interval for nodes which have been removed from the cluster in CiliumNetworkPolicy Status")
 	option.BindEnv(operatorOption.CNPNodeStatusGCInterval)
 
@@ -101,11 +80,6 @@ func init() {
 	flags.MarkHidden(option.DisableCiliumEndpointCRDName)
 	option.BindEnv(option.DisableCiliumEndpointCRDName)
 
-	// Deprecated, remove in 1.9
-	flags.Bool(operatorOption.EnableCEPGC, true, "Enable CiliumEndpoint garbage collector")
-	option.BindEnv(operatorOption.EnableCEPGC)
-	flags.MarkDeprecated(operatorOption.EnableCEPGC, fmt.Sprintf("Please use %s=0 to disable CEP GC", operatorOption.EndpointGCInterval))
-
 	flags.Duration(operatorOption.EndpointGCInterval, operatorOption.EndpointGCIntervalDefault, "GC interval for cilium endpoints")
 	option.BindEnv(operatorOption.EndpointGCInterval)
 
@@ -283,14 +257,6 @@ func init() {
 	flags.Bool(option.Version, false, "Print version information")
 	option.BindEnv(option.Version)
 
-	// Deprecated, remove in 1.9
-	flags.Uint16Var(&apiServerPort, "api-server-port", 9234, "Port on which the operator should serve API requests")
-	flags.MarkDeprecated("api-server-port", fmt.Sprintf("Please use %s instead", operatorOption.OperatorAPIServeAddr))
-
-	// Deprecated, remove in 1.9
-	flags.StringVar(&operatorMetrics.Address, "metrics-address", ":6942", "Address to serve Prometheus metrics")
-	flags.MarkDeprecated("metrics-address", fmt.Sprintf("Please use %s instead", operatorOption.OperatorPrometheusServeAddr))
-
 	flags.String(option.CMDRef, "", "Path to cmdref output directory")
 	flags.MarkHidden(option.CMDRef)
 	option.BindEnv(option.CMDRef)
diff --git a/operator/k8s_node.go b/operator/k8s_node.go
index c2e3d08fce..22e6fd4a98 100644
--- a/operator/k8s_node.go
+++ b/operator/k8s_node.go
@@ -133,11 +133,8 @@ func runNodeWatcher(nodeManager *allocator.NodeEventHandler) error {
 
 	}()
 
-	if operatorOption.Config.EnableCNPNodeStatusGC && operatorOption.Config.CNPNodeStatusGCInterval != 0 {
+	if operatorOption.Config.CNPNodeStatusGCInterval != 0 {
 		go runCNPNodeStatusGC("cnp-node-gc", false, ciliumNodeStore)
-	}
-
-	if operatorOption.Config.EnableCCNPNodeStatusGC && operatorOption.Config.CNPNodeStatusGCInterval != 0 {
 		go runCNPNodeStatusGC("ccnp-node-gc", true, ciliumNodeStore)
 	}
 
diff --git a/operator/main.go b/operator/main.go
index 2f45dd162b..db8027ad12 100644
--- a/operator/main.go
+++ b/operator/main.go
@@ -342,7 +342,7 @@ func runOperator(cmd *cobra.Command) {
 		}
 	}
 
-	if operatorOption.Config.EnableCEPGC && operatorOption.Config.EndpointGCInterval != 0 {
+	if operatorOption.Config.EndpointGCInterval != 0 {
 		enableCiliumEndpointSyncGC()
 	}
 
diff --git a/operator/metrics/metrics.go b/operator/metrics/metrics.go
index c0ffd7bc13..49af9b68d4 100644
--- a/operator/metrics/metrics.go
+++ b/operator/metrics/metrics.go
@@ -28,8 +28,6 @@ const Namespace = "cilium_operator"
 
 var (
 	Registry *prometheus.Registry
-
-	Address string
 )
 
 func Register() {
@@ -39,13 +37,6 @@ func Register() {
 		// The Handler function provides a default handler to expose metrics
 		// via an HTTP server. "/metrics" is the usual endpoint for that.
 		http.Handle("/metrics", promhttp.HandlerFor(Registry, promhttp.HandlerOpts{}))
-		log.Fatal(http.ListenAndServe(getPrometheusServerAddr(), nil))
+		log.Fatal(http.ListenAndServe(operatorOption.Config.OperatorPrometheusServeAddr, nil))
 	}()
 }
-
-func getPrometheusServerAddr() string {
-	if operatorOption.Config.OperatorPrometheusServeAddr == "" {
-		return Address
-	}
-	return operatorOption.Config.OperatorPrometheusServeAddr
-}
diff --git a/operator/option/config.go b/operator/option/config.go
index fccd4f465a..afc4834758 100644
--- a/operator/option/config.go
+++ b/operator/option/config.go
@@ -35,20 +35,6 @@ const (
 	// being sent to the K8s apiserver for a given CNP.
 	CNPStatusUpdateInterval = "cnp-status-update-interval"
 
-	// EnableCEPGC enables CiliumEndpoint garbage collector
-	// Deprecated: use EndpointGCInterval and remove in 1.9
-	EnableCEPGC = "cilium-endpoint-gc"
-
-	// EnableCCNPNodeStatusGC enables CiliumClusterwideNetworkPolicy Status
-	// garbage collection for nodes which have been removed from the cluster
-	// Deprecated: use CNPNodeStatusGCInterval and remove in 1.9
-	EnableCCNPNodeStatusGC = "ccnp-node-status-gc"
-
-	// EnableCNPNodeStatusGC enables CiliumNetworkPolicy Status garbage
-	// collection for nodes which have been removed from the cluster
-	// Deprecated: use CNPNodeStatusGCInterval and remove in 1.9
-	EnableCNPNodeStatusGC = "cnp-node-status-gc"
-
 	// EnableMetrics enables prometheus metrics.
 	EnableMetrics = "enable-metrics"
 
@@ -95,12 +81,6 @@ const (
 
 	// IPAM options
 
-	// AWSClientBurstDeprecated is the deprecated version of IPAMAPIBurst and will be rewmoved in v1.9
-	AWSClientBurstDeprecated = "aws-client-burst"
-
-	// AWSClientQPSLimitDeprecated is the deprecated version of IPAMAPIQPSLimit and will be removed in v1.9
-	AWSClientQPSLimitDeprecated = "aws-client-qps"
-
 	// IPAMAPIBurst is the burst value allowed when accessing external IPAM APIs
 	IPAMAPIBurst = "limit-ipam-api-burst"
 
@@ -145,10 +125,6 @@ const (
 	// AWS ENI IPAM.
 	ENITags = "eni-tags"
 
-	// ENIParallelWorkersDeprecated is the deprecated name of the option
-	// ParallelAllocWorkers that can be removed in Cilium 1.9
-	ENIParallelWorkersDeprecated = "eni-parallel-workers"
-
 	// ParallelAllocWorkers specifies the number of parallel workers to be used for IPAM allocation
 	ParallelAllocWorkers = "parallel-alloc-workers"
 
@@ -179,23 +155,9 @@ type OperatorConfig struct {
 	// being sent to the K8s apiserver for a given CNP.
 	CNPStatusUpdateInterval time.Duration
 
-	// EnableCEPGC enables CiliumEndpoint garbage collector
-	// Deprecated: use EndpointGCInterval and remove in 1.9
-	EnableCEPGC bool
-
-	// EnableCNPNodeStatusGC enables CiliumNetworkPolicy Status garbage collection
-	// for nodes which have been removed from the cluster
-	// Deprecated: use CNPNodeStatusGCInterval and remove in 1.9
-	EnableCNPNodeStatusGC bool
-
 	// EnableMetrics enables prometheus metrics.
 	EnableMetrics bool
 
-	// EnableCCNPNodeStatusGC enables CiliumClusterwideNetworkPolicy Status
-	// garbage collection for nodes which have been removed from the cluster
-	// Deprecated: use CNPNodeStatusGCInterval and remove in 1.9
-	EnableCCNPNodeStatusGC bool
-
 	// EndpointGCInterval is the interval between attempts of the CEP GC
 	// controller.
 	// Note that only one node per cluster should run this, and most iterations
@@ -300,9 +262,6 @@ type OperatorConfig struct {
 func (c *OperatorConfig) Populate() {
 	c.CNPNodeStatusGCInterval = viper.GetDuration(CNPNodeStatusGCInterval)
 	c.CNPStatusUpdateInterval = viper.GetDuration(CNPStatusUpdateInterval)
-	c.EnableCEPGC = viper.GetBool(EnableCEPGC)
-	c.EnableCNPNodeStatusGC = viper.GetBool(EnableCNPNodeStatusGC)
-	c.EnableCCNPNodeStatusGC = viper.GetBool(EnableCCNPNodeStatusGC)
 	c.EnableMetrics = viper.GetBool(EnableMetrics)
 	c.EndpointGCInterval = viper.GetDuration(EndpointGCInterval)
 	c.IdentityGCInterval = viper.GetDuration(IdentityGCInterval)
@@ -332,26 +291,6 @@ func (c *OperatorConfig) Populate() {
 	c.AzureSubscriptionID = viper.GetString(AzureSubscriptionID)
 	c.AzureResourceGroup = viper.GetString(AzureResourceGroup)
 
-	// Deprecated options
-
-	if val := viper.GetInt(AWSClientBurstDeprecated); val != 0 {
-		c.IPAMAPIBurst = val
-	} else {
-		c.IPAMAPIBurst = viper.GetInt(IPAMAPIBurst)
-	}
-
-	if val := viper.GetFloat64(AWSClientQPSLimitDeprecated); val != 0 {
-		c.IPAMAPIQPSLimit = val
-	} else {
-		c.IPAMAPIQPSLimit = viper.GetFloat64(IPAMAPIQPSLimit)
-	}
-
-	if val := viper.GetInt64(ENIParallelWorkersDeprecated); val != 0 {
-		c.ParallelAllocWorkers = val
-	} else {
-		c.ParallelAllocWorkers = viper.GetInt64(ParallelAllocWorkers)
-	}
-
 	// Option maps and slices
 
 	if m := viper.GetStringSlice(IPAMSubnetsIDs); len(m) != 0 {
diff --git a/pkg/endpoint/endpoint.go b/pkg/endpoint/endpoint.go
index 12b9f1efd0..316884e186 100644
--- a/pkg/endpoint/endpoint.go
+++ b/pkg/endpoint/endpoint.go
@@ -377,18 +377,6 @@ func (e *Endpoint) HasIpvlanDataPath() bool {
 	return false
 }
 
-// GetIngressPolicyEnabledLocked returns whether ingress policy enforcement is
-// enabled for endpoint or not. The endpoint's mutex must be held.
-func (e *Endpoint) GetIngressPolicyEnabledLocked() bool {
-	return e.desiredPolicy.IngressPolicyEnabled
-}
-
-// GetEgressPolicyEnabledLocked returns whether egress policy enforcement is
-// enabled for endpoint or not. The endpoint's mutex must be held.
-func (e *Endpoint) GetEgressPolicyEnabledLocked() bool {
-	return e.desiredPolicy.EgressPolicyEnabled
-}
-
 // waitForProxyCompletions blocks until all proxy changes have been completed.
 // Called with buildMutex held.
 func (e *Endpoint) waitForProxyCompletions(proxyWaitGroup *completion.WaitGroup) error {
diff --git a/pkg/endpoint/regeneration/owner.go b/pkg/endpoint/regeneration/owner.go
index 86d9673176..561461aad4 100644
--- a/pkg/endpoint/regeneration/owner.go
+++ b/pkg/endpoint/regeneration/owner.go
@@ -1,4 +1,4 @@
-// Copyright 2016-2019 Authors of Cilium
+// Copyright 2016-2020 Authors of Cilium
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -60,8 +60,6 @@ type EndpointInfoSource interface {
 	HasSidecarProxy() bool
 	ConntrackName() string
 	ConntrackNameLocked() string
-	GetIngressPolicyEnabledLocked() bool
-	GetEgressPolicyEnabledLocked() bool
 	ProxyID(npMap policy.NamedPortsMap, l4 *policy.L4Filter) (string, error)
 	GetProxyInfoByFields() (uint64, string, string, []string, string, uint64, error)
 }
diff --git a/pkg/health/client/client.go b/pkg/health/client/client.go
index 790eee2652..132a3bfc38 100644
--- a/pkg/health/client/client.go
+++ b/pkg/health/client/client.go
@@ -33,10 +33,27 @@ import (
 	"github.com/go-openapi/strfmt"
 )
 
+type ConnectivityStatusType int
+
 const (
 	ipUnavailable = "Unavailable"
+
+	ConnStatusReachable   ConnectivityStatusType = 0
+	ConnStatusUnreachable ConnectivityStatusType = 1
+	ConnStatusUnknown     ConnectivityStatusType = 2
 )
 
+func (c ConnectivityStatusType) String() string {
+	switch c {
+	case ConnStatusReachable:
+		return "reachable"
+	case ConnStatusUnreachable:
+		return "unreachable"
+	default:
+		return "unknown"
+	}
+}
+
 // Client is a client for cilium health
 type Client struct {
 	clientapi.CiliumHealthAPI
@@ -111,13 +128,51 @@ func Hint(err error) error {
 	return fmt.Errorf("%s", e)
 }
 
-func connectivityStatusHealthy(cs *models.ConnectivityStatus) bool {
-	return cs != nil && cs.Status == ""
+func GetConnectivityStatusType(cs *models.ConnectivityStatus) ConnectivityStatusType {
+	// If the connecticity status is nil, it means that there was no
+	// successful probe, but also no failed probe with a concrete reason. In
+	// that case, the status is unknown and it usually means that the new
+	// is still in the beginning of the bootstraping process.
+	if cs == nil {
+		return ConnStatusUnknown
+	}
+	// Empty status means successful probe.
+	if cs.Status == "" {
+		return ConnStatusReachable
+	}
+	// Non-empty status means that there was an explicit reason of failure.
+	return ConnStatusUnreachable
+}
+
+func GetPathConnectivityStatusType(cp *models.PathStatus) ConnectivityStatusType {
+	if cp == nil {
+		return ConnStatusUnreachable
+	}
+	statuses := []*models.ConnectivityStatus{
+		cp.Icmp,
+		cp.HTTP,
+	}
+	// Initially assume healthy status.
+	status := ConnStatusReachable
+	for _, cs := range statuses {
+		switch GetConnectivityStatusType(cs) {
+		case ConnStatusUnreachable:
+			// If any status is unreachable, return it immediately.
+			return ConnStatusUnreachable
+		case ConnStatusUnknown:
+			// If the status is unknown, prepare to return it. It's
+			// going to be returned if there is no unreachable
+			// status in next iterations.
+			status = ConnStatusUnknown
+		}
+	}
+	return status
 }
 
 func formatConnectivityStatus(w io.Writer, cs *models.ConnectivityStatus, path, indent string) {
 	status := cs.Status
-	if connectivityStatusHealthy(cs) {
+	switch GetConnectivityStatusType(cs) {
+	case ConnStatusReachable:
 		latency := time.Duration(cs.Latency)
 		status = fmt.Sprintf("OK, RTT=%s", latency)
 	}
@@ -142,9 +197,10 @@ func formatPathStatus(w io.Writer, name string, cp *models.PathStatus, indent st
 	}
 }
 
-// PathIsHealthy checks whether ICMP and TCP(HTTP) connectivity to the given
-// path is available.
-func PathIsHealthy(cp *models.PathStatus) bool {
+// pathIsHealthyOrUnknown checks whether ICMP and TCP(HTTP) connectivity to the
+// given path is available or had no explicit error status (which usually is the
+// case when the new node is provisioned).
+func pathIsHealthyOrUnknown(cp *models.PathStatus) bool {
 	if cp == nil {
 		return false
 	}
@@ -154,7 +210,8 @@ func PathIsHealthy(cp *models.PathStatus) bool {
 		cp.HTTP,
 	}
 	for _, status := range statuses {
-		if !connectivityStatusHealthy(status) {
+		switch GetConnectivityStatusType(status) {
+		case ConnStatusUnreachable:
 			return false
 		}
 	}
@@ -162,8 +219,8 @@ func PathIsHealthy(cp *models.PathStatus) bool {
 }
 
 func nodeIsHealthy(node *models.NodeStatus) bool {
-	return PathIsHealthy(GetHostPrimaryAddress(node)) &&
-		(node.Endpoint == nil || PathIsHealthy(node.Endpoint))
+	return pathIsHealthyOrUnknown(GetHostPrimaryAddress(node)) &&
+		(node.Endpoint == nil || pathIsHealthyOrUnknown(node.Endpoint))
 }
 
 func nodeIsLocalhost(node *models.NodeStatus, self *models.SelfStatus) bool {
@@ -196,10 +253,10 @@ func formatNodeStatus(w io.Writer, node *models.NodeStatus, printAll, succinct,
 	if succinct {
 		if printAll || !nodeIsHealthy(node) {
 
-			fmt.Fprintf(w, "  %s%s\t%s\t%t\t%t\n", node.Name,
+			fmt.Fprintf(w, "  %s%s\t%s\t%s\t%s\n", node.Name,
 				localStr, getPrimaryAddressIP(node),
-				PathIsHealthy(GetHostPrimaryAddress(node)),
-				PathIsHealthy(node.Endpoint))
+				GetPathConnectivityStatusType(GetHostPrimaryAddress(node)).String(),
+				GetPathConnectivityStatusType(node.Endpoint).String())
 		}
 	} else {
 		fmt.Fprintf(w, "  %s%s:\n", node.Name, localStr)
@@ -237,7 +294,7 @@ func FormatHealthStatusResponse(w io.Writer, sr *models.HealthStatusResponse, pr
 		fmt.Fprintf(w, "Cluster health:\t%d/%d reachable\t(%s)\n",
 			healthy, len(sr.Nodes), sr.Timestamp)
 		if printAll || healthy < len(sr.Nodes) {
-			fmt.Fprintf(w, "  Name\tIP\tReachable\tEndpoints reachable\n")
+			fmt.Fprintf(w, "  Name\tIP\tNode\tEndpoints\n")
 		}
 	} else {
 		fmt.Fprintf(w, "Probe time:\t%s\n", sr.Timestamp)
diff --git a/pkg/hubble/relay/observer/server.go b/pkg/hubble/relay/observer/server.go
index 016437e12e..8d478132a9 100644
--- a/pkg/hubble/relay/observer/server.go
+++ b/pkg/hubble/relay/observer/server.go
@@ -169,12 +169,13 @@ func (s *Server) ServerStatus(ctx context.Context, req *observerpb.ServerStatusR
 	g, ctx := errgroup.WithContext(ctx)
 
 	peers := s.peers.List()
-	var numConnectedNodes uint32
+	var numConnectedNodes, numUnavailableNodes uint32
 	var unavailableNodes []string
 	statuses := make(chan *observerpb.ServerStatusResponse, len(peers))
 	for _, p := range peers {
 		p := p
 		if !isAvailable(p.Conn) {
+			numUnavailableNodes++
 			s.opts.log.WithField("address", p.Address.String()).Infof(
 				"No connection to peer %s, skipping", p.Name,
 			)
@@ -211,7 +212,7 @@ func (s *Server) ServerStatus(ctx context.Context, req *observerpb.ServerStatusR
 			Value: numConnectedNodes,
 		},
 		NumUnavailableNodes: &wrappers.UInt32Value{
-			Value: uint32(len(unavailableNodes)),
+			Value: numUnavailableNodes,
 		},
 		UnavailableNodes: unavailableNodes,
 	}
diff --git a/pkg/metrics/status.go b/pkg/metrics/status.go
index c7a9844d00..ce531df714 100644
--- a/pkg/metrics/status.go
+++ b/pkg/metrics/status.go
@@ -139,11 +139,15 @@ func (s *statusCollector) Collect(ch chan<- prometheus.Metric) {
 	)
 
 	for _, nodeStatus := range healthStatusResponse.Payload.Nodes {
-		if !healthClientPkg.PathIsHealthy(healthClientPkg.GetHostPrimaryAddress(nodeStatus)) {
+		switch healthClientPkg.GetPathConnectivityStatusType(healthClientPkg.GetHostPrimaryAddress(nodeStatus)) {
+		case healthClientPkg.ConnStatusUnreachable:
 			unreachableNodes++
 		}
-		if nodeStatus.Endpoint != nil && !healthClientPkg.PathIsHealthy(nodeStatus.Endpoint) {
-			unreachableEndpoints++
+		if nodeStatus.Endpoint != nil {
+			switch healthClientPkg.GetPathConnectivityStatusType(nodeStatus.Endpoint) {
+			case healthClientPkg.ConnStatusUnreachable:
+				unreachableEndpoints++
+			}
 		}
 	}
 
diff --git a/pkg/policy/distillery.go b/pkg/policy/distillery.go
index 8ff1caddb7..558337c8e7 100644
--- a/pkg/policy/distillery.go
+++ b/pkg/policy/distillery.go
@@ -1,4 +1,4 @@
-// Copyright 2019 Authors of Cilium
+// Copyright 2019-2020 Authors of Cilium
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -62,9 +62,8 @@ func (cache *PolicyCache) GetSelectorCache() *SelectorCache {
 }
 
 // lookupOrCreate adds the specified Identity to the policy cache, with a reference
-// from the specified Endpoint, then returns the threadsafe copy of the policy
-// and whether policy has been computed for this identity.
-func (cache *PolicyCache) lookupOrCreate(identity *identityPkg.Identity, create bool) (SelectorPolicy, bool) {
+// from the specified Endpoint, then returns the threadsafe copy of the policy.
+func (cache *PolicyCache) lookupOrCreate(identity *identityPkg.Identity, create bool) SelectorPolicy {
 	cache.Lock()
 	defer cache.Unlock()
 	cip, ok := cache.policies[identity.ID]
@@ -72,16 +71,12 @@ func (cache *PolicyCache) lookupOrCreate(identity *identityPkg.Identity, create
 		cip = newCachedSelectorPolicy(identity, cache.repo.GetSelectorCache())
 		cache.policies[identity.ID] = cip
 	}
-	if cip != nil {
-		return cip, cip.getPolicy().Revision > 0
-	}
-	return nil, false
+	return cip
 }
 
 // insert adds the specified Identity to the policy cache, with a reference
-// from the specified Endpoint, then returns the threadsafe copy of the policy
-// and whether policy has been computed for this identity.
-func (cache *PolicyCache) insert(identity *identityPkg.Identity) (SelectorPolicy, bool) {
+// from the specified Endpoint, then returns the threadsafe copy of the policy.
+func (cache *PolicyCache) insert(identity *identityPkg.Identity) SelectorPolicy {
 	return cache.lookupOrCreate(identity, true)
 }
 
@@ -157,8 +152,7 @@ func (cache *PolicyCache) LocalEndpointIdentityRemoved(identity *identityPkg.Ide
 // Lookup attempts to locate the SelectorPolicy corresponding to the specified
 // identity. If policy is not cached for the identity, it returns nil.
 func (cache *PolicyCache) Lookup(identity *identityPkg.Identity) SelectorPolicy {
-	cip, _ := cache.lookupOrCreate(identity, false)
-	return cip
+	return cache.lookupOrCreate(identity, false)
 }
 
 // UpdatePolicy resolves the policy for the security identity of the specified
diff --git a/pkg/proxy/logger/epinfo.go b/pkg/proxy/logger/epinfo.go
index 1e6d49731e..a41d43aea5 100644
--- a/pkg/proxy/logger/epinfo.go
+++ b/pkg/proxy/logger/epinfo.go
@@ -1,4 +1,4 @@
-// Copyright 2018 Authors of Cilium
+// Copyright 2018-2020 Authors of Cilium
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -39,8 +39,6 @@ type EndpointInfoSource interface {
 	// implementation.
 	ConntrackName() string
 	ConntrackNameLocked() string
-	GetIngressPolicyEnabledLocked() bool
-	GetEgressPolicyEnabledLocked() bool
 	ProxyID(npMap policy.NamedPortsMap, l4 *policy.L4Filter) (string, error)
 	GetProxyInfoByFields() (uint64, string, string, []string, string, uint64, error)
 }


Test patch:
diff --git a/pkg/health/client/client_test.go b/pkg/health/client/client_test.go
index adb5024508..3ae62c6301 100644
--- a/pkg/health/client/client_test.go
+++ b/pkg/health/client/client_test.go
@@ -34,6 +34,130 @@ type ClientTestSuite struct{}
 
 var _ = Suite(&ClientTestSuite{})
 
+func (s *ClientTestSuite) TestConnectivityStatusType(c *C) {
+	tests := []struct {
+		cst         ConnectivityStatusType
+		expectedStr string
+	}{
+		{
+			cst:         ConnStatusReachable,
+			expectedStr: "reachable",
+		},
+		{
+			cst:         ConnStatusUnreachable,
+			expectedStr: "unreachable",
+		},
+		{
+			cst:         ConnStatusUnknown,
+			expectedStr: "unknown",
+		},
+		{
+			cst:         10,
+			expectedStr: "unknown",
+		},
+	}
+	for _, tc := range tests {
+		c.Assert(tc.cst.String(), Equals, tc.expectedStr)
+	}
+}
+
+func (s *ClientTestSuite) TestGetConnectivityStatusType(c *C) {
+	tests := []struct {
+		cs                 *models.ConnectivityStatus
+		expectedStatusType ConnectivityStatusType
+	}{
+		{
+			cs:                 &models.ConnectivityStatus{Status: ""},
+			expectedStatusType: ConnStatusReachable,
+		},
+		{
+			cs:                 &models.ConnectivityStatus{Status: "failed"},
+			expectedStatusType: ConnStatusUnreachable,
+		},
+		{
+			cs:                 nil,
+			expectedStatusType: ConnStatusUnknown,
+		},
+	}
+	for _, tc := range tests {
+		c.Assert(GetConnectivityStatusType(tc.cs), Equals, tc.expectedStatusType)
+	}
+}
+
+func (s *ClientTestSuite) TestGetPathConnectivityStatusType(c *C) {
+	tests := []struct {
+		cp                 *models.PathStatus
+		expectedStatusType ConnectivityStatusType
+	}{
+		{
+			cp: &models.PathStatus{
+				Icmp: &models.ConnectivityStatus{Status: ""},
+				HTTP: &models.ConnectivityStatus{Status: ""},
+			},
+			expectedStatusType: ConnStatusReachable,
+		},
+		{
+			cp: &models.PathStatus{
+				Icmp: &models.ConnectivityStatus{Status: "failed"},
+				HTTP: &models.ConnectivityStatus{Status: "failed"},
+			},
+			expectedStatusType: ConnStatusUnreachable,
+		},
+		{
+			cp: &models.PathStatus{
+				Icmp: &models.ConnectivityStatus{Status: ""},
+				HTTP: &models.ConnectivityStatus{Status: "failed"},
+			},
+			expectedStatusType: ConnStatusUnreachable,
+		},
+		{
+			cp: &models.PathStatus{
+				Icmp: &models.ConnectivityStatus{Status: "failed"},
+				HTTP: &models.ConnectivityStatus{Status: ""},
+			},
+			expectedStatusType: ConnStatusUnreachable,
+		},
+		{
+			cp: &models.PathStatus{
+				Icmp: &models.ConnectivityStatus{Status: "failed"},
+				HTTP: nil,
+			},
+			expectedStatusType: ConnStatusUnreachable,
+		},
+		{
+			cp: &models.PathStatus{
+				Icmp: nil,
+				HTTP: &models.ConnectivityStatus{Status: "failed"},
+			},
+			expectedStatusType: ConnStatusUnreachable,
+		},
+		{
+			cp: &models.PathStatus{
+				Icmp: nil,
+				HTTP: nil,
+			},
+			expectedStatusType: ConnStatusUnknown,
+		},
+		{
+			cp: &models.PathStatus{
+				Icmp: &models.ConnectivityStatus{Status: ""},
+				HTTP: nil,
+			},
+			expectedStatusType: ConnStatusUnknown,
+		},
+		{
+			cp: &models.PathStatus{
+				Icmp: nil,
+				HTTP: &models.ConnectivityStatus{Status: ""},
+			},
+			expectedStatusType: ConnStatusUnknown,
+		},
+	}
+	for _, tc := range tests {
+		c.Assert(GetPathConnectivityStatusType(tc.cp), Equals, tc.expectedStatusType)
+	}
+}
+
 func (s *ClientTestSuite) TestFormatNodeStatus(c *C) {
 	// This test generates permutations of models.NodeStatus and sees whether
 	// the calls to formatNodeStatus panic; the result of this test being
diff --git a/pkg/policy/distillery_test.go b/pkg/policy/distillery_test.go
index b020aa6e8e..72c4596268 100644
--- a/pkg/policy/distillery_test.go
+++ b/pkg/policy/distillery_test.go
@@ -1,4 +1,4 @@
-// Copyright 2019 Authors of Cilium
+// Copyright 2019-2020 Authors of Cilium
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -58,8 +58,8 @@ func (s *DistilleryTestSuite) TestCacheManagement(c *C) {
 	c.Assert(deleted, Equals, false)
 
 	// Insert identity twice. Should be the same policy.
-	policy1, _ := cache.insert(identity)
-	policy2, _ := cache.insert(identity)
+	policy1 := cache.insert(identity)
+	policy2 := cache.insert(identity)
 	c.Assert(policy1, Equals, policy2)
 
 	// Despite two insert calls, there is no reference tracking; any delete
@@ -75,11 +75,11 @@ func (s *DistilleryTestSuite) TestCacheManagement(c *C) {
 	ep3.SetIdentity(1234, true)
 	identity3 := ep3.GetSecurityIdentity()
 	c.Assert(identity3, Not(Equals), identity)
-	policy1, _ = cache.insert(identity)
-	policy3, _ := cache.insert(identity3)
+	policy1 = cache.insert(identity)
+	policy3 := cache.insert(identity3)
 	c.Assert(policy1, Not(Equals), policy3)
 	_ = cache.delete(identity)
-	policy3, _ = cache.lookupOrCreate(identity3, false)
+	policy3 = cache.lookupOrCreate(identity3, false)
 	c.Assert(policy3, NotNil)
 }
 
@@ -90,8 +90,7 @@ func (s *DistilleryTestSuite) TestCachePopulation(c *C) {
 
 	identity1 := ep1.GetSecurityIdentity()
 	c.Assert(ep2.GetSecurityIdentity(), Equals, identity1)
-	policy1, computed := cache.insert(identity1)
-	c.Assert(computed, Equals, false)
+	policy1 := cache.insert(identity1)
 
 	// Calculate the policy and observe that it's cached
 	updated, err := cache.updateSelectorPolicy(identity1)
@@ -100,8 +99,7 @@ func (s *DistilleryTestSuite) TestCachePopulation(c *C) {
 	updated, err = cache.updateSelectorPolicy(identity1)
 	c.Assert(err, IsNil)
 	c.Assert(updated, Equals, false)
-	policy2, computed := cache.insert(identity1)
-	c.Assert(computed, Equals, true)
+	policy2 := cache.insert(identity1)
 	idp1 := policy1.(*cachedSelectorPolicy).getPolicy()
 	idp2 := policy2.(*cachedSelectorPolicy).getPolicy()
 	c.Assert(idp1, Equals, idp2)
@@ -121,14 +119,12 @@ func (s *DistilleryTestSuite) TestCachePopulation(c *C) {
 
 	// Insert endpoint with different identity and observe that the cache
 	// is different from ep1, ep2
-	policy1, computed = cache.insert(identity1)
-	c.Assert(computed, Equals, false)
+	policy1 = cache.insert(identity1)
 	idp1 = policy1.(*cachedSelectorPolicy).getPolicy()
 	c.Assert(idp1, NotNil)
 	identity3 := ep3.GetSecurityIdentity()
-	policy3, computed := cache.insert(identity3)
+	policy3 := cache.insert(identity3)
 	c.Assert(policy3, Not(Equals), policy1)
-	c.Assert(computed, Equals, false)
 	updated, err = cache.updateSelectorPolicy(identity3)
 	c.Assert(err, IsNil)
 	c.Assert(updated, Equals, true)
diff --git a/pkg/proxy/mock_test.go b/pkg/proxy/mock_test.go
index a7341a24ad..4e5fb345bc 100644
--- a/pkg/proxy/mock_test.go
+++ b/pkg/proxy/mock_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017-2019 Authors of Cilium
+// Copyright 2017-2020 Authors of Cilium
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -45,8 +45,6 @@ func (m *proxyUpdaterMock) GetID() uint64                               { return
 func (m *proxyUpdaterMock) GetIPv4Address() string                      { return m.ipv4 }
 func (m *proxyUpdaterMock) GetIPv6Address() string                      { return m.ipv6 }
 func (m *proxyUpdaterMock) GetLabels() []string                         { return m.labels }
-func (m *proxyUpdaterMock) GetEgressPolicyEnabledLocked() bool          { return true }
-func (m *proxyUpdaterMock) GetIngressPolicyEnabledLocked() bool         { return true }
 func (m *proxyUpdaterMock) GetIdentityLocked() identity.NumericIdentity { return m.identity }
 func (m *proxyUpdaterMock) ProxyID(npMap policy.NamedPortsMap, l4 *policy.L4Filter) (string, error) {
 	return "", nil