Repository: VictoriaMetrics/VictoriaMetrics
Pull request #1575 (2021-08-28 06:41:21)

lib/protoparser/opentsdb: accept multiple spaces between fields as a deliminator
For #1574,
I changed OpenTSDB put message parser to accept  some spaces between fields and at an end of a row as a deliminator.


Patch:
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index ff4226f27..59daf63d8 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -8,6 +8,7 @@ sort: 15
 
 * FEATURE: vmagent: add ability to read scrape configs from multiple files specified in `scrape_config_files` section. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1559).
 * FEATURE: vmagent: reduce memory usage and CPU usage when Prometheus staleness tracking is enabled for metrics exported from the deleted or disappeared scrape targets.
+* FEATURE: vmagent: discover `role: ingress` and `role: endpointslice` in [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) via v1 API instead of v1beta1 API if Kubernetes supports it. This fixes service discovery in Kubernetes v1.22 and newer versions. See [these docs](https://kubernetes.io/docs/reference/using-api/deprecation-guide/#ingress-v122).
 * FEATURE: take into account failed queries in `vm_request_duration_seconds` summary at `/metrics`. Previously only successful queries were taken into account. This could result in skewed summary. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1537).
 * FEATURE: vmalert: add `-disableAlertgroupLabel` command-line flag for disabling the label with alert group name. This may be needed for proper deduplication in Alertmanager. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1532).
 * FEATURE: update Go builder from v1.16.7 to v1.17.0. This improves data ingestion and query performance by up to 5% according to benchmarks. See [the release post for Go1.17](https://go.dev/blog/go1.17).
@@ -16,6 +17,7 @@ sort: 15
 * FEATURE: add `quantiles("quantileLabel", phi1, ..., phiN, q)` aggregate function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html), which calculates the given `phi*` quantiles over time series returned by `q`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1573).
 
 * BUGFIX: rename `sign` function to `sgn` in order to be consistent with PromQL. See [this pull request from Prometheus](https://github.com/prometheus/prometheus/pull/8457).
+* BUGFIX: vmagent: rename `role: endpointslices` to `role: endpointslice` in [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) in order to be consistent with Prometheus. See [the corresponding code in Prometheus](https://github.com/prometheus/prometheus/blob/2ec6c7dbb82b72834021e01f1773eb90a67a371f/discovery/kubernetes/kubernetes.go#L99).
 * BUGFIX: improve the detection of the needed free space for background merge operation. This should prevent from possible out of disk space crashes during big merges. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1560).
 * BUGFIX: vmauth: remove trailing slash from the full url before requesting it from the backend. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1554).
 * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix timeout error when snapshot takes longer than 10 seconds. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1571).
diff --git a/lib/promscrape/discovery/kubernetes/api.go b/lib/promscrape/discovery/kubernetes/api.go
index ff7db4a45..96b2a5a41 100644
--- a/lib/promscrape/discovery/kubernetes/api.go
+++ b/lib/promscrape/discovery/kubernetes/api.go
@@ -16,9 +16,9 @@ type apiConfig struct {
 
 func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFunc) (*apiConfig, error) {
 	switch sdc.Role {
-	case "node", "pod", "service", "endpoints", "endpointslices", "ingress":
+	case "node", "pod", "service", "endpoints", "endpointslice", "ingress":
 	default:
-		return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `pod`, `service`, `endpoints`, `endpointslices` or `ingress`", sdc.Role)
+		return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `pod`, `service`, `endpoints`, `endpointslice` or `ingress`", sdc.Role)
 	}
 	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
 	if err != nil {
diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go
index 9171ab856..6d3bf8d4e 100644
--- a/lib/promscrape/discovery/kubernetes/api_watcher.go
+++ b/lib/promscrape/discovery/kubernetes/api_watcher.go
@@ -13,6 +13,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -159,6 +160,14 @@ func (aw *apiWatcher) getScrapeWorkObjects() []interface{} {
 // groupWatcher watches for Kubernetes objects on the given apiServer with the given namespaces,
 // selectors using the given client.
 type groupWatcher struct {
+	// Old Kubernetes doesn't support /apis/networking.k8s.io/v1/, so /apis/networking.k8s.io/v1beta1/ must be used instead.
+	// This flag is used for automatic substitution of v1 API path with v1beta1 API path during requests to apiServer.
+	useNetworkingV1Beta1 uint32
+
+	// Old Kubernetes doesn't support /apis/discovery.k8s.io/v1/, so discovery.k8s.io/v1beta1/ must be used instead.
+	// This flag is used for automatic substitution of v1 API path with v1beta1 API path during requests to apiServer.
+	useDiscoveryV1Beta1 uint32
+
 	apiServer     string
 	namespaces    []string
 	selectors     []Selector
@@ -254,8 +263,8 @@ func (gw *groupWatcher) getObjectByRoleLocked(role, namespace, name string) obje
 }
 
 func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) {
-	if role == "endpoints" || role == "endpointslices" {
-		// endpoints and endpointslices watchers query pod and service objects. So start watchers for these roles as well.
+	if role == "endpoints" || role == "endpointslice" {
+		// endpoints and endpointslice watchers query pod and service objects. So start watchers for these roles as well.
 		gw.startWatchersForRole("pod", nil)
 		gw.startWatchersForRole("service", nil)
 	}
@@ -276,7 +285,7 @@ func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) {
 		if needStart {
 			uw.reloadObjects()
 			go uw.watchForUpdates()
-			if role == "endpoints" || role == "endpointslices" {
+			if role == "endpoints" || role == "endpointslice" {
 				// Refresh endpoints and enpointslices targets in background, since they depend on other object types such as pod and service.
 				// This should fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240 .
 				go func() {
@@ -294,6 +303,14 @@ func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) {
 
 // doRequest performs http request to the given requestURL.
 func (gw *groupWatcher) doRequest(requestURL string) (*http.Response, error) {
+	if strings.Contains(requestURL, "/apis/networking.k8s.io/v1/") && atomic.LoadUint32(&gw.useNetworkingV1Beta1) == 1 {
+		// Update networking URL for old Kubernetes API, which supports only v1beta1 path.
+		requestURL = strings.Replace(requestURL, "/apis/networking.k8s.io/v1/", "/apis/networking.k8s.io/v1beta1/", 1)
+	}
+	if strings.Contains(requestURL, "/apis/discovery.k8s.io/v1/") && atomic.LoadUint32(&gw.useDiscoveryV1Beta1) == 1 {
+		// Update discovery URL for old Kuberentes API, which supports only v1beta1 path.
+		requestURL = strings.Replace(requestURL, "/apis/discovery.k8s.io/v1/", "/apis/discovery.k8s.io/v1beta1/", 1)
+	}
 	req, err := http.NewRequest("GET", requestURL, nil)
 	if err != nil {
 		logger.Fatalf("cannot create a request for %q: %s", requestURL, err)
@@ -301,7 +318,21 @@ func (gw *groupWatcher) doRequest(requestURL string) (*http.Response, error) {
 	if ah := gw.getAuthHeader(); ah != "" {
 		req.Header.Set("Authorization", ah)
 	}
-	return gw.client.Do(req)
+	resp, err := gw.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode == http.StatusNotFound {
+		if strings.Contains(requestURL, "/apis/networking.k8s.io/v1/") && atomic.LoadUint32(&gw.useNetworkingV1Beta1) == 0 {
+			atomic.StoreUint32(&gw.useNetworkingV1Beta1, 1)
+			return gw.doRequest(requestURL)
+		}
+		if strings.Contains(requestURL, "/apis/discovery.k8s.io/v1/") && atomic.LoadUint32(&gw.useDiscoveryV1Beta1) == 0 {
+			atomic.StoreUint32(&gw.useDiscoveryV1Beta1, 1)
+			return gw.doRequest(requestURL)
+		}
+	}
+	return resp, nil
 }
 
 func (gw *groupWatcher) registerPendingAPIWatchers() {
@@ -682,10 +713,10 @@ func getAPIPath(objectType, namespace, query string) string {
 		suffix += "?" + query
 	}
 	if objectType == "ingresses" {
-		return "/apis/networking.k8s.io/v1beta1/" + suffix
+		return "/apis/networking.k8s.io/v1/" + suffix
 	}
 	if objectType == "endpointslices" {
-		return "/apis/discovery.k8s.io/v1beta1/" + suffix
+		return "/apis/discovery.k8s.io/v1/" + suffix
 	}
 	return "/api/v1/" + suffix
 }
@@ -723,7 +754,7 @@ func getObjectTypeByRole(role string) string {
 		return "services"
 	case "endpoints":
 		return "endpoints"
-	case "endpointslices":
+	case "endpointslice":
 		return "endpointslices"
 	case "ingress":
 		return "ingresses"
@@ -743,7 +774,7 @@ func getObjectParsersForRole(role string) (parseObjectFunc, parseObjectListFunc)
 		return parseService, parseServiceList
 	case "endpoints":
 		return parseEndpoints, parseEndpointsList
-	case "endpointslices":
+	case "endpointslice":
 		return parseEndpointSlice, parseEndpointSliceList
 	case "ingress":
 		return parseIngress, parseIngressList
diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go
index 68b5c3c3d..a4a9b21b9 100644
--- a/lib/promscrape/discovery/kubernetes/endpoints.go
+++ b/lib/promscrape/discovery/kubernetes/endpoints.go
@@ -79,7 +79,7 @@ type ObjectReference struct {
 
 // EndpointPort implements k8s endpoint port.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointport-v1beta1-discovery-k8s-io
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointport-v1-discovery-k8s-io
 type EndpointPort struct {
 	AppProtocol string
 	Name        string
diff --git a/lib/promscrape/discovery/kubernetes/endpointslices.go b/lib/promscrape/discovery/kubernetes/endpointslices.go
index 4ac24b475..37053de79 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslices.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslices.go
@@ -146,16 +146,17 @@ func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp En
 	return m
 }
 
-// EndpointSliceList - implements kubernetes endpoint slice list object,
-// that groups service endpoints slices.
-// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslice-v1beta1-discovery-k8s-io
+// EndpointSliceList - implements kubernetes endpoint slice list object, that groups service endpoints slices.
+//
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointslicelist-v1-discovery-k8s-io
 type EndpointSliceList struct {
 	Metadata ListMeta
 	Items    []*EndpointSlice
 }
 
 // EndpointSlice - implements kubernetes endpoint slice.
-// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslice-v1beta1-discovery-k8s-io
+//
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointslice-v1-discovery-k8s-io
 type EndpointSlice struct {
 	Metadata    ObjectMeta
 	Endpoints   []Endpoint
@@ -164,7 +165,8 @@ type EndpointSlice struct {
 }
 
 // Endpoint implements kubernetes object endpoint for endpoint slice.
-// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpoint-v1beta1-discovery-k8s-io
+//
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpoint-v1-discovery-k8s-io
 type Endpoint struct {
 	Addresses  []string
 	Conditions EndpointConditions
@@ -174,7 +176,8 @@ type Endpoint struct {
 }
 
 // EndpointConditions implements kubernetes endpoint condition.
-// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointconditions-v1beta1-discovery-k8s-io
+//
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointconditions-v1-discovery-k8s-io
 type EndpointConditions struct {
 	Ready bool
 }
diff --git a/lib/promscrape/discovery/kubernetes/ingress.go b/lib/promscrape/discovery/kubernetes/ingress.go
index cca6bdb24..deedd32f2 100644
--- a/lib/promscrape/discovery/kubernetes/ingress.go
+++ b/lib/promscrape/discovery/kubernetes/ingress.go
@@ -33,7 +33,7 @@ func parseIngress(data []byte) (object, error) {
 
 // IngressList represents ingress list in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingresslist-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingresslist-v1-networking-k8s-io
 type IngressList struct {
 	Metadata ListMeta
 	Items    []*Ingress
@@ -41,7 +41,7 @@ type IngressList struct {
 
 // Ingress represents ingress in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingress-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingress-v1-networking-k8s-io
 type Ingress struct {
 	Metadata ObjectMeta
 	Spec     IngressSpec
@@ -49,7 +49,7 @@ type Ingress struct {
 
 // IngressSpec represents ingress spec in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingressspec-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingressspec-v1-networking-k8s-io
 type IngressSpec struct {
 	TLS   []IngressTLS `json:"tls"`
 	Rules []IngressRule
@@ -57,14 +57,14 @@ type IngressSpec struct {
 
 // IngressTLS represents ingress TLS spec in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingresstls-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingresstls-v1-networking-k8s-io
 type IngressTLS struct {
 	Hosts []string
 }
 
 // IngressRule represents ingress rule in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingressrule-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingressrule-v1-networking-k8s-io
 type IngressRule struct {
 	Host string
 	HTTP HTTPIngressRuleValue `json:"http"`
@@ -72,14 +72,14 @@ type IngressRule struct {
 
 // HTTPIngressRuleValue represents HTTP ingress rule value in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#httpingressrulevalue-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#httpingressrulevalue-v1-networking-k8s-io
 type HTTPIngressRuleValue struct {
 	Paths []HTTPIngressPath
 }
 
 // HTTPIngressPath represents HTTP ingress path in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#httpingresspath-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#httpingresspath-v1-networking-k8s-io
 type HTTPIngressPath struct {
 	Path string
 }
diff --git a/lib/protoparser/opentsdb/parser.go b/lib/protoparser/opentsdb/parser.go
index a1fd9a624..54d0d9da9 100644
--- a/lib/protoparser/opentsdb/parser.go
+++ b/lib/protoparser/opentsdb/parser.go
@@ -57,10 +57,11 @@ func (r *Row) reset() {
 
 func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) {
 	r.reset()
+	s = strings.TrimSpace(s)
 	if !strings.HasPrefix(s, "put ") {
 		return tagsPool, fmt.Errorf("missing `put ` prefix in %q", s)
 	}
-	s = s[len("put "):]
+  s = strings.TrimSpace(s[len("put "):])
 	n := strings.IndexByte(s, ' ')
 	if n < 0 {
 		return tagsPool, fmt.Errorf("cannot find whitespace between metric and timestamp in %q", s)
@@ -69,7 +70,7 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) {
 	if len(r.Metric) == 0 {
 		return tagsPool, fmt.Errorf("metric cannot be empty")
 	}
-	tail := s[n+1:]
+	tail := strings.TrimSpace(s[n+1:])
 	n = strings.IndexByte(tail, ' ')
 	if n < 0 {
 		return tagsPool, fmt.Errorf("cannot find whitespace between timestamp and value in %q", s)
@@ -79,7 +80,7 @@ func (r *Row) unmarshal(s string, tagsPool []Tag) ([]Tag, error) {
 		return tagsPool, fmt.Errorf("cannot parse timestamp from %q: %w", tail[:n], err)
 	}
 	r.Timestamp = int64(timestamp)
-	tail = tail[n+1:]
+	tail = strings.TrimSpace(tail[n+1:])
 	n = strings.IndexByte(tail, ' ')
 	if n < 0 {
 		return tagsPool, fmt.Errorf("cannot find whitespace between value and the first tag in %q", s)
@@ -148,6 +149,7 @@ func unmarshalTags(dst []Tag, s string) ([]Tag, error) {
 		}
 		tag := &dst[len(dst)-1]
 
+		s = strings.TrimSpace(s)
 		n := strings.IndexByte(s, ' ')
 		if n < 0 {
 			// The last tag found


Test patch:
diff --git a/lib/promscrape/discovery/kubernetes/api_watcher_test.go b/lib/promscrape/discovery/kubernetes/api_watcher_test.go
index 35b2b181c..f59e84833 100644
--- a/lib/promscrape/discovery/kubernetes/api_watcher_test.go
+++ b/lib/promscrape/discovery/kubernetes/api_watcher_test.go
@@ -128,21 +128,21 @@ func TestGetAPIPathsWithNamespaces(t *testing.T) {
 		"/api/v1/namespaces/y/endpoints?labelSelector=bbb",
 	})
 
-	// role=endpointslices
-	f("endpointslices", nil, nil, []string{"/apis/discovery.k8s.io/v1beta1/endpointslices"})
-	f("endpointslices", []string{"x", "y"}, []Selector{
+	// role=endpointslice
+	f("endpointslice", nil, nil, []string{"/apis/discovery.k8s.io/v1/endpointslices"})
+	f("endpointslice", []string{"x", "y"}, []Selector{
 		{
-			Role:  "endpointslices",
+			Role:  "endpointslice",
 			Field: "field",
 			Label: "label",
 		},
 	}, []string{
-		"/apis/discovery.k8s.io/v1beta1/namespaces/x/endpointslices?labelSelector=label&fieldSelector=field",
-		"/apis/discovery.k8s.io/v1beta1/namespaces/y/endpointslices?labelSelector=label&fieldSelector=field",
+		"/apis/discovery.k8s.io/v1/namespaces/x/endpointslices?labelSelector=label&fieldSelector=field",
+		"/apis/discovery.k8s.io/v1/namespaces/y/endpointslices?labelSelector=label&fieldSelector=field",
 	})
 
 	// role=ingress
-	f("ingress", nil, nil, []string{"/apis/networking.k8s.io/v1beta1/ingresses"})
+	f("ingress", nil, nil, []string{"/apis/networking.k8s.io/v1/ingresses"})
 	f("ingress", []string{"x", "y"}, []Selector{
 		{
 			Role:  "node",
@@ -161,8 +161,8 @@ func TestGetAPIPathsWithNamespaces(t *testing.T) {
 			Label: "baaa",
 		},
 	}, []string{
-		"/apis/networking.k8s.io/v1beta1/namespaces/x/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc",
-		"/apis/networking.k8s.io/v1beta1/namespaces/y/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc",
+		"/apis/networking.k8s.io/v1/namespaces/x/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc",
+		"/apis/networking.k8s.io/v1/namespaces/y/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc",
 	})
 }
 
@@ -577,9 +577,9 @@ func TestGetScrapeWorkObjects(t *testing.T) {
 			initAPIObjectsByRole: map[string][]byte{
 				"ingress": []byte(`{
   "kind": "IngressList",
-  "apiVersion": "extensions/v1beta1",
+  "apiVersion": "extensions/v1",
   "metadata": {
-    "selfLink": "/apis/extensions/v1beta1/ingresses",
+    "selfLink": "/apis/extensions/v1/ingresses",
     "resourceVersion": "351452"
   },
   "items": [
@@ -672,15 +672,15 @@ func TestGetScrapeWorkObjects(t *testing.T) {
 		{
 			name: "7 endpointslices slice with 1 service update",
 			sdc: &SDConfig{
-				Role: "endpointslices",
+				Role: "endpointslice",
 			},
 			expectedTargetsLen: 7,
 			initAPIObjectsByRole: map[string][]byte{
-				"endpointslices": []byte(`{
+				"endpointslice": []byte(`{
   "kind": "EndpointSliceList",
-  "apiVersion": "discovery.k8s.io/v1beta1",
+  "apiVersion": "discovery.k8s.io/v1",
   "metadata": {
-    "selfLink": "/apis/discovery.k8s.io/v1beta1/endpointslices",
+    "selfLink": "/apis/discovery.k8s.io/v1/endpointslices",
     "resourceVersion": "1177"
   },
   "items": [
diff --git a/lib/promscrape/discovery/kubernetes/endpointslices_test.go b/lib/promscrape/discovery/kubernetes/endpointslices_test.go
index 4114e0cff..f647695f4 100644
--- a/lib/promscrape/discovery/kubernetes/endpointslices_test.go
+++ b/lib/promscrape/discovery/kubernetes/endpointslices_test.go
@@ -32,9 +32,9 @@ func TestParseEndpointSliceListFail(t *testing.T) {
 func TestParseEndpointSliceListSuccess(t *testing.T) {
 	data := `{
   "kind": "EndpointSliceList",
-  "apiVersion": "discovery.k8s.io/v1beta1",
+  "apiVersion": "discovery.k8s.io/v1",
   "metadata": {
-    "selfLink": "/apis/discovery.k8s.io/v1beta1/endpointslices",
+    "selfLink": "/apis/discovery.k8s.io/v1/endpointslices",
     "resourceVersion": "1177"
   },
   "items": [
@@ -42,7 +42,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) {
       "metadata": {
         "name": "kubernetes",
         "namespace": "default",
-        "selfLink": "/apis/discovery.k8s.io/v1beta1/namespaces/default/endpointslices/kubernetes",
+        "selfLink": "/apis/discovery.k8s.io/v1/namespaces/default/endpointslices/kubernetes",
         "uid": "a60d9173-5fe4-4bc3-87a6-269daee71f8a",
         "resourceVersion": "159",
         "generation": 1,
@@ -54,7 +54,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) {
           {
             "manager": "kube-apiserver",
             "operation": "Update",
-            "apiVersion": "discovery.k8s.io/v1beta1",
+            "apiVersion": "discovery.k8s.io/v1",
             "time": "2020-09-07T14:27:22Z",
             "fieldsType": "FieldsV1",
             "fieldsV1": {"f:addressType":{},"f:endpoints":{},"f:metadata":{"f:labels":{".":{},"f:kubernetes.io/service-name":{}}},"f:ports":{}}
@@ -85,7 +85,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) {
         "name": "kube-dns-22mvb",
         "generateName": "kube-dns-",
         "namespace": "kube-system",
-        "selfLink": "/apis/discovery.k8s.io/v1beta1/namespaces/kube-system/endpointslices/kube-dns-22mvb",
+        "selfLink": "/apis/discovery.k8s.io/v1/namespaces/kube-system/endpointslices/kube-dns-22mvb",
         "uid": "7c95c854-f34c-48e1-86f5-bb8269113c11",
         "resourceVersion": "604",
         "generation": 5,
@@ -111,7 +111,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) {
           {
             "manager": "kube-controller-manager",
             "operation": "Update",
-            "apiVersion": "discovery.k8s.io/v1beta1",
+            "apiVersion": "discovery.k8s.io/v1",
             "time": "2020-09-07T14:28:35Z",
             "fieldsType": "FieldsV1",
             "fieldsV1": {"f:addressType":{},"f:endpoints":{},"f:metadata":{"f:annotations":{".":{},"f:endpoints.kubernetes.io/last-change-trigger-time":{}},"f:generateName":{},"f:labels":{".":{},"f:endpointslice.kubernetes.io/managed-by":{},"f:kubernetes.io/service-name":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"509e80d8-6d05-487b-bfff-74f5768f1024\"}":{".":{},"f:apiVersion":{},"f:blockOwnerDeletion":{},"f:controller":{},"f:kind":{},"f:name":{},"f:uid":{}}}},"f:ports":{}}
diff --git a/lib/promscrape/discovery/kubernetes/ingress_test.go b/lib/promscrape/discovery/kubernetes/ingress_test.go
index f3440ac23..149660b80 100644
--- a/lib/promscrape/discovery/kubernetes/ingress_test.go
+++ b/lib/promscrape/discovery/kubernetes/ingress_test.go
@@ -30,9 +30,9 @@ func TestParseIngressListSuccess(t *testing.T) {
 	data := `
 {
   "kind": "IngressList",
-  "apiVersion": "extensions/v1beta1",
+  "apiVersion": "extensions/v1",
   "metadata": {
-    "selfLink": "/apis/extensions/v1beta1/ingresses",
+    "selfLink": "/apis/extensions/v1/ingresses",
     "resourceVersion": "351452"
   },
   "items": [
@@ -40,13 +40,13 @@ func TestParseIngressListSuccess(t *testing.T) {
       "metadata": {
         "name": "test-ingress",
         "namespace": "default",
-        "selfLink": "/apis/extensions/v1beta1/namespaces/default/ingresses/test-ingress",
+        "selfLink": "/apis/extensions/v1/namespaces/default/ingresses/test-ingress",
         "uid": "6d3f38f9-de89-4bc9-b273-c8faf74e8a27",
         "resourceVersion": "351445",
         "generation": 1,
         "creationTimestamp": "2020-04-13T16:43:52Z",
         "annotations": {
-          "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"networking.k8s.io/v1beta1\",\"kind\":\"Ingress\",\"metadata\":{\"annotations\":{},\"name\":\"test-ingress\",\"namespace\":\"default\"},\"spec\":{\"backend\":{\"serviceName\":\"testsvc\",\"servicePort\":80}}}\n"
+          "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"networking.k8s.io/v1\",\"kind\":\"Ingress\",\"metadata\":{\"annotations\":{},\"name\":\"test-ingress\",\"namespace\":\"default\"},\"spec\":{\"backend\":{\"serviceName\":\"testsvc\",\"servicePort\":80}}}\n"
         }
       },
       "spec": {
@@ -85,7 +85,7 @@ func TestParseIngressListSuccess(t *testing.T) {
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__": "foobar",
-			"__meta_kubernetes_ingress_annotation_kubectl_kubernetes_io_last_applied_configuration":        `{"apiVersion":"networking.k8s.io/v1beta1","kind":"Ingress","metadata":{"annotations":{},"name":"test-ingress","namespace":"default"},"spec":{"backend":{"serviceName":"testsvc","servicePort":80}}}` + "\n",
+			"__meta_kubernetes_ingress_annotation_kubectl_kubernetes_io_last_applied_configuration":        `{"apiVersion":"networking.k8s.io/v1","kind":"Ingress","metadata":{"annotations":{},"name":"test-ingress","namespace":"default"},"spec":{"backend":{"serviceName":"testsvc","servicePort":80}}}` + "\n",
 			"__meta_kubernetes_ingress_annotationpresent_kubectl_kubernetes_io_last_applied_configuration": "true",
 			"__meta_kubernetes_ingress_host":   "foobar",
 			"__meta_kubernetes_ingress_name":   "test-ingress",
diff --git a/lib/protoparser/opentsdb/parser_test.go b/lib/protoparser/opentsdb/parser_test.go
index bc7fc63a9..d2e739f90 100644
--- a/lib/protoparser/opentsdb/parser_test.go
+++ b/lib/protoparser/opentsdb/parser_test.go
@@ -221,4 +221,90 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 			},
 		},
 	})
+
+	// Multi spaces
+	f("put  foobar 789 -123.456 a=b", &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 789,
+			Tags: []Tag{
+				{
+					Key:   "a",
+					Value: "b",
+				},
+			},
+		}},
+	})
+	f("put foobar  789 -123.456 a=b", &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 789,
+			Tags: []Tag{
+				{
+					Key:   "a",
+					Value: "b",
+				},
+			},
+		}},
+	})
+	f("put foobar 789  -123.456 a=b", &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 789,
+			Tags: []Tag{
+				{
+					Key:   "a",
+					Value: "b",
+				},
+			},
+		}},
+	})
+	f("put foobar 789 -123.456  a=b", &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 789,
+			Tags: []Tag{
+				{
+					Key:   "a",
+					Value: "b",
+				},
+			},
+		}},
+	})
+	f("put foobar 789 -123.456 a=b  c=d", &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 789,
+			Tags: []Tag{
+				{
+					Key:   "a",
+					Value: "b",
+				},
+				{
+					Key:   "c",
+					Value: "d",
+				},
+			},
+		}},
+	})
+	// Soace after tags
+	f("put foobar 789 -123.456 a=b ", &Rows{
+		Rows: []Row{{
+			Metric:    "foobar",
+			Value:     -123.456,
+			Timestamp: 789,
+			Tags: []Tag{
+				{
+					Key:   "a",
+					Value: "b",
+				},
+			},
+		}},
+	})
+
 }