Repository: VictoriaMetrics/VictoriaMetrics
Pull request #7251 (2024-10-14 07:30:36)

vmagent: Add Marathon service discovery support
### Describe Your Changes

https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6642

Added support for [Marathon](https://mesosphere.github.io/marathon/) service discovery.

The following flag is introduced:
```
  -promscrape.marathonSDCheckInterval duration
          Interval for checking for changes in Marathon service discovery. This works only if marathon_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#marathon_sd_configs for details  (default 30s)
```

The service discovery could be config like:
```yaml
scrape_configs:          
- job_name: marathon_job 
  marathon_sd_configs:   
      servers:
        - "..."
        - "..."
```
See: [https://docs.victoriametrics.com/sd_configs/#marathon_sd_configs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/b555d94d1a861925759669daa86e4a9f635b0746/docs/sd_configs.md#marathon_sd_configs)

### Checklist

The following checks are **mandatory**:

- [x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).
- [x] Tested by developer.
- [ ] Tested by user. 



DuraCHYo: @jiekun 
Also sd_configs.md, line 1281
" job_name: nomad "
Not nomad but marathon_sd
jiekun: @DuraCHYo Thanks for the review. Typo fixed.

Patch:
diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go
index b62aa8794..f07357c28 100644
--- a/app/vmauth/auth_config.go
+++ b/app/vmauth/auth_config.go
@@ -7,11 +7,13 @@ import (
 	"flag"
 	"fmt"
 	"math"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -348,6 +350,7 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 	hostToAddrs := make(map[string][]string)
 	for _, bu := range up.busOriginal {
 		host := bu.Hostname()
+		port := bu.Port()
 		if hostToAddrs[host] != nil {
 			// ips for the given host have been already discovered
 			continue
@@ -364,7 +367,11 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 			} else {
 				resolvedAddrs = make([]string, len(addrs))
 				for i, addr := range addrs {
-					resolvedAddrs[i] = fmt.Sprintf("%s:%d", addr.Target, addr.Port)
+					hostPort := port
+					if hostPort == "" && addr.Port > 0 {
+						hostPort = strconv.FormatUint(uint64(addr.Port), 10)
+					}
+					resolvedAddrs[i] = net.JoinHostPort(addr.Target, hostPort)
 				}
 			}
 		} else {
@@ -375,7 +382,7 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 			} else {
 				resolvedAddrs = make([]string, len(addrs))
 				for i, addr := range addrs {
-					resolvedAddrs[i] = addr.String()
+					resolvedAddrs[i] = net.JoinHostPort(addr.String(), port)
 				}
 			}
 		}
@@ -389,17 +396,9 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 	var busNew []*backendURL
 	for _, bu := range up.busOriginal {
 		host := bu.Hostname()
-		port := bu.Port()
 		for _, addr := range hostToAddrs[host] {
 			buCopy := *bu
 			buCopy.Host = addr
-			if port != "" {
-				if n := strings.IndexByte(buCopy.Host, ':'); n >= 0 {
-					// Drop the discovered port and substitute it the port specified in bu.
-					buCopy.Host = buCopy.Host[:n]
-				}
-				buCopy.Host += ":" + port
-			}
 			busNew = append(busNew, &backendURL{
 				url: &buCopy,
 			})
diff --git a/docs/VictoriaLogs/FAQ.md b/docs/VictoriaLogs/FAQ.md
index 98f8df3b0..376cf2e91 100644
--- a/docs/VictoriaLogs/FAQ.md
+++ b/docs/VictoriaLogs/FAQ.md
@@ -175,3 +175,20 @@ And for the following log, its `_msg` will be `foo bar in body`:
   "body": "foo bar in body"
 }
 ```
+
+## What length a log record is expected to have?
+
+VictoriaLogs works optimally with log records of up to `10KB`. It works OK with
+log records of up to `100KB`. It works not so optimal with log records exceeding
+`100KB`.
+
+The max size of a log record VictoriaLogs can handle is `2MB`. This is
+because VictoriaLogs stores log records in blocks and `2MB` is the max size of a
+block. Blocks of this size fit the L2 cache of a typical CPU, which gives an
+optimal processing performance.
+
+However, log records whose size is close to `2MB` aren't handled efficiently by
+VictoriaLogs because per-block overhead translates to a single log record, and
+this overhead is big. 
+
+The `2MB` limit is hadrcoded and is unlikely to change.
diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md
index fd6c020dd..1ad8d4d0b 100644
--- a/docs/changelog/CHANGELOG.md
+++ b/docs/changelog/CHANGELOG.md
@@ -22,14 +22,16 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add export data functionality for the `Raw Query` page and the ability to import exported data into the `Query Analyzer` page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7628).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add `markdown` support for comments during data export. [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7828).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [Single-node VictoriaMetrics](https://docs.victoriametrics.com/): added `min` and `max` metrics for Datadog Sketches API metrics, changed `_` metric name separator to `.` if metrics are not sanitized for consistency.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [Single-node VictoriaMetrics](https://docs.victoriametrics.com/): add service discovery support for [Marathon](https://mesosphere.github.io/marathon/). See [these docs](https://docs.victoriametrics.com/sd_configs/#marathon_sd_configs) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6642).
 * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/): support `-maxIngestionRate` cmd-line flag to ratelimit samples/sec ingested. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7377) for details.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): improve query performance on systems with high number of CPU cores. See [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7416) for details.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add command-line flag `-search.maxBinaryOpPushdownLabelValues` to allow using labels with more candidate values as push down filter in binary operation. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7243). Thanks to @tydhot for implementation.
 
 * BUGFIX: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards): consistently use `vmagent_remotewrite_pending_data_bytes` on vmagent dashboard to represent persistent queue size.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): fix the auto-generated metrics `ALERTS` and `ALERTS_FOR_STATE` for alerting rules. Previously, metrics might have incorrect labels and affect the restore process. See this [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7796).
-* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly set `host` field at debug information formatted with `dump_request_on_errors: true` setting.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victorialogs/vmalert/): do not append tenant info to VictoriaLogs datasource request path in [clusterMode](https://docs.victoriametrics.com/vmalert/#multitenancy). See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/#how-to-use-multitenancy-in-rules) for how to use multitenancy in VictoriaLogs.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly set `host` field at debug information formatted with `dump_request_on_errors: true` setting.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly handle discovery for ipv6 addresses. Thanks to @badie for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7955).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): fix support for migrating influx series without any tag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7921). Thanks to @bitbidu for reporting.
 * BUGFIX: [vminsert](https://docs.victoriametrics.com/vminsert/): storage nodes defined in `-storageNode` are now sorted, ensuring that varying node orders across different vminsert instances do not result in inconsistent replication.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly ingest `influx` line protocol metrics with empty tags. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7933) for details.
@@ -79,6 +81,7 @@ Released at 2024-12-13
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix query interval display in the instant query info. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7401).
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): respect `-search.skipSlowReplicas` when `-globalReplicationFactor` > 1. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6924).
 
+
 ## [v1.107.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.107.0)
 
 Released at 2024-11-29
diff --git a/docs/sd_configs.md b/docs/sd_configs.md
index a48efe25c..8b44d7f66 100644
--- a/docs/sd_configs.md
+++ b/docs/sd_configs.md
@@ -28,6 +28,7 @@ supports the following Prometheus-compatible service discovery options for Prome
 * `http_sd_configs` is for discovering and scraping targets provided by external http-based service discovery. See [these docs](#http_sd_configs).
 * `kubernetes_sd_configs` is for discovering and scraping [Kubernetes](https://kubernetes.io/) targets. See [these docs](#kubernetes_sd_configs).
 * `kuma_sd_configs` is for discovering and scraping [Kuma](https://kuma.io) targets. See [these docs](#kuma_sd_configs).
+* `marathon_sd_configs` is for discovering and scraping [Marathon](https://mesosphere.github.io/marathon/) targets. See [these docs](#marathon_sd_configs).
 * `nomad_sd_configs` is for discovering and scraping targets registered in [HashiCorp Nomad](https://www.nomadproject.io/). See [these docs](#nomad_sd_configs).
 * `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [these docs](#openstack_sd_configs).
 * `ovhcloud_sd_configs` is for discovering and scraping OVH Cloud VPS and dedicated server targets. See [these docs](#ovhcloud_sd_configs).
@@ -1273,6 +1274,43 @@ The following meta labels are available on discovered targets during [relabeling
 
 The list of discovered Kuma targets is refreshed at the interval, which can be configured via `-promscrape.kumaSDCheckInterval` command-line flag.
 
+## marathon_sd_configs
+
+_Available from [CHANGEME](https://docs.victoriametrics.com/changelog/#vCHANGEME) version._
+
+Marathon SD configuration allows retrieving scrape targets from [Marathon](https://mesosphere.github.io/marathon/) REST API.
+
+Configuration example:
+
+```yaml
+scrape_configs:
+- job_name: marathon
+  marathon_sd_configs:
+    # List of URLs to be used to contact Marathon servers.
+    # You need to provide at least one server URL, but should provide URLs for
+    # all masters you have running.
+    #
+    servers:
+      - "host1:port1"
+      - "host2:port2"
+      - "..."
+    
+    # Additional HTTP API client options can be specified here.
+    # See https://docs.victoriametrics.com/sd_configs/#http-api-client-options
+```
+
+The following meta labels are available on discovered targets during [relabeling](https://docs.victoriametrics.com/vmagent/#relabeling):
+
+* `__meta_marathon_app`: the name of the app (with slashes replaced by dashes)
+* `__meta_marathon_image`: the name of the Docker image used (if available)
+* `__meta_marathon_task`: the ID of the Mesos task
+* `__meta_marathon_app_label_<labelname>`: any Marathon labels attached to the app
+* `__meta_marathon_port_definition_label_<labelname>`: the port definition labels
+* `__meta_marathon_port_mapping_label_<labelname>`: the port mapping labels
+* `__meta_marathon_port_index`: the port index number (e.g. `1` for `PORT1`)
+
+The list of discovered Marathon targets is refreshed at the interval, which can be configured via `-promscrape.marathonSDCheckInterval` command-line flag.
+
 ## nomad_sd_configs
 
 Nomad SD configuration allows retrieving scrape targets from [HashiCorp Nomad Services](https://www.hashicorp.com/blog/nomad-service-discovery).
diff --git a/docs/vmagent.md b/docs/vmagent.md
index 157502b55..3e23fa39b 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -2010,6 +2010,8 @@ See the docs at https://docs.victoriametrics.com/vmagent/ .
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs/#kubernetes_sd_configs for details (default 30s)
   -promscrape.kumaSDCheckInterval duration
      Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs/#kuma_sd_configs for details (default 30s)
+  -promscrape.marathonSDCheckInterval duration
+     Interval for checking for changes in Marathon service discovery. This works only if marathon_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#marathon_sd_configs for details  (default 30s)
   -promscrape.maxDroppedTargets int
      The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 10000)
   -promscrape.maxResponseHeadersSize size
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index c212f634b..c751a3015 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -12,6 +12,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
@@ -35,6 +39,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/http"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kuma"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/marathon"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/nomad"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ovhcloud"
@@ -43,9 +48,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/yandexcloud"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/cespare/xxhash/v2"
-	"gopkg.in/yaml.v2"
 )
 
 var (
@@ -314,6 +316,7 @@ type ScrapeConfig struct {
 	HTTPSDConfigs         []http.SDConfig         `yaml:"http_sd_configs,omitempty"`
 	KubernetesSDConfigs   []kubernetes.SDConfig   `yaml:"kubernetes_sd_configs,omitempty"`
 	KumaSDConfigs         []kuma.SDConfig         `yaml:"kuma_sd_configs,omitempty"`
+	MarathonSDConfigs     []marathon.SDConfig     `yaml:"marathon_sd_configs,omitempty"`
 	NomadSDConfigs        []nomad.SDConfig        `yaml:"nomad_sd_configs,omitempty"`
 	OpenStackSDConfigs    []openstack.SDConfig    `yaml:"openstack_sd_configs,omitempty"`
 	OVHCloudSDConfigs     []ovhcloud.SDConfig     `yaml:"ovhcloud_sd_configs,omitempty"`
@@ -749,6 +752,16 @@ func (cfg *Config) getKumaSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 	return cfg.getScrapeWorkGeneric(visitConfigs, "kuma_sd_config", prev)
 }
 
+// getMarathonSDScrapeWork returns `marathon_sd_configs` ScrapeWork from cfg.
+func (cfg *Config) getMarathonSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
+	visitConfigs := func(sc *ScrapeConfig, visitor func(sdc targetLabelsGetter)) {
+		for i := range sc.MarathonSDConfigs {
+			visitor(&sc.MarathonSDConfigs[i])
+		}
+	}
+	return cfg.getScrapeWorkGeneric(visitConfigs, "marathon_sd_config", prev)
+}
+
 // getNomadSDScrapeWork returns `nomad_sd_configs` ScrapeWork from cfg.
 func (cfg *Config) getNomadSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 	visitConfigs := func(sc *ScrapeConfig, visitor func(sdc targetLabelsGetter)) {
diff --git a/lib/promscrape/discovery/marathon/api.go b/lib/promscrape/discovery/marathon/api.go
new file mode 100644
index 000000000..4e30f30b4
--- /dev/null
+++ b/lib/promscrape/discovery/marathon/api.go
@@ -0,0 +1,47 @@
+package marathon
+
+import (
+	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+)
+
+// apiConfig contains config for API server.
+type apiConfig struct {
+	cs []*discoveryutils.Client
+}
+
+// getAPIConfig get or create API config from configMap.
+func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
+	v, err := configMap.Get(sdc, func() (any, error) { return newAPIConfig(sdc, baseDir) })
+	if err != nil {
+		return nil, err
+	}
+	return v.(*apiConfig), nil
+}
+
+// newAPIConfig create API Config.
+func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
+	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse auth config: %w", err)
+	}
+	proxyAC, err := sdc.ProxyClientConfig.NewConfig(baseDir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
+	}
+
+	cs := make([]*discoveryutils.Client, 0, len(sdc.Servers))
+	for i := range sdc.Servers {
+		c, e := discoveryutils.NewClient(sdc.Servers[i], ac, sdc.ProxyURL, proxyAC, &sdc.HTTPClientConfig)
+		if e != nil {
+			return nil, fmt.Errorf("cannot create client for %q: %w", sdc.Servers[i], e)
+		}
+		cs = append(cs, c)
+	}
+
+	cfg := &apiConfig{
+		cs: cs,
+	}
+	return cfg, nil
+}
diff --git a/lib/promscrape/discovery/marathon/apps.go b/lib/promscrape/discovery/marathon/apps.go
new file mode 100644
index 000000000..e1d7f6024
--- /dev/null
+++ b/lib/promscrape/discovery/marathon/apps.go
@@ -0,0 +1,96 @@
+package marathon
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand"
+)
+
+// AppList is a list of Marathon apps.
+type AppList struct {
+	Apps []app `json:"apps"`
+}
+
+// App describes a service running on Marathon.
+type app struct {
+	ID              string            `json:"id"`
+	Tasks           []task            `json:"tasks"`
+	TasksRunning    int               `json:"tasksRunning"`
+	Labels          map[string]string `json:"labels"`
+	Container       container         `json:"container"`
+	PortDefinitions []portDefinition  `json:"portDefinitions"`
+	Networks        []network         `json:"networks"`
+	RequirePorts    bool              `json:"requirePorts"`
+}
+
+// task describes one instance of a service running on Marathon.
+type task struct {
+	ID          string   `json:"id"`
+	Host        string   `json:"host"`
+	Ports       []uint32 `json:"ports"`
+	IPAddresses []ipAddr `json:"ipAddresses"`
+}
+
+// ipAddress describes the address and protocol the container's network interface is bound to.
+type ipAddr struct {
+	IPAddress string `json:"ipAddress"`
+	Protocol  string `json:"protocol"`
+}
+
+// Container describes the runtime an app in running in.
+type container struct {
+	Docker       dockerContainer `json:"docker"`
+	PortMappings []portMapping   `json:"portMappings"`
+}
+
+// DockerContainer describes a container which uses the docker runtime.
+type dockerContainer struct {
+	Image        string        `json:"image"`
+	PortMappings []portMapping `json:"portMappings"`
+}
+
+// PortMapping describes in which port the process are binding inside the docker container.
+type portMapping struct {
+	Labels        map[string]string `json:"labels"`
+	ContainerPort uint32            `json:"containerPort"`
+	HostPort      uint32            `json:"hostPort"`
+	ServicePort   uint32            `json:"servicePort"`
+}
+
+// PortDefinition describes which load balancer port you should access to access the service.
+type portDefinition struct {
+	Labels map[string]string `json:"labels"`
+	Port   uint32            `json:"port"`
+}
+
+// Network describes the name and type of network the container is attached to.
+type network struct {
+	Name string `json:"name"`
+	Mode string `json:"mode"`
+}
+
+// isContainerNet checks if the app's first network is set to mode 'container'.
+func (app app) isContainerNet() bool {
+	return len(app.Networks) > 0 && app.Networks[0].Mode == "container"
+}
+
+// GetAppsList randomly choose a server from the `servers` and get the list of running applications.
+func GetAppsList(ac *apiConfig) (*AppList, error) {
+	c := ac.cs[rand.Intn(len(ac.cs))]
+
+	// /v2/apps API: get the list of running applications.
+	// https://mesosphere.github.io/marathon/api-console/index.html
+	path := "/v2/apps/?embed=apps.tasks"
+	resp, err := c.GetAPIResponse(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get Marathon response from %s: %w", path, err)
+	}
+
+	var apps AppList
+	err = json.Unmarshal(resp, &apps)
+	if err != nil {
+		return nil, fmt.Errorf("cannot unmarshal AppList obtained from %q: %w; response=%q", path, err, resp)
+	}
+
+	return &apps, nil
+}
diff --git a/lib/promscrape/discovery/marathon/marathon.go b/lib/promscrape/discovery/marathon/marathon.go
new file mode 100644
index 000000000..3f558b422
--- /dev/null
+++ b/lib/promscrape/discovery/marathon/marathon.go
@@ -0,0 +1,184 @@
+package marathon
+
+import (
+	"flag"
+	"fmt"
+	"net"
+	"strconv"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
+)
+
+// SDCheckInterval defines interval for targets refresh.
+var SDCheckInterval = flag.Duration("promscrape.marathonSDCheckInterval", 30*time.Second, "Interval for checking for changes in Marathon REST API. "+
+	"This works only if marathon_sd_configs is configured in '-promscrape.config' file. "+
+	"See https://docs.victoriametrics.com/sd_configs/#marathon_sd_configs for details")
+
+// SDConfig is the configuration for Marathon service discovery.
+type SDConfig struct {
+	Servers []string `yaml:"servers"`
+
+	HTTPClientConfig  promauth.HTTPClientConfig  `yaml:",inline"`
+	ProxyURL          *proxy.URL                 `yaml:"proxy_url,omitempty"`
+	ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`
+}
+
+var configMap = discoveryutils.NewConfigMap()
+
+// GetLabels returns Marathon labels according to sdc.
+func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
+	ac, err := getAPIConfig(sdc, baseDir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get API config: %w", err)
+	}
+
+	apps, err := GetAppsList(ac)
+	if err != nil {
+		return nil, err
+	}
+	return getAppsLabels(apps), nil
+}
+
+// MustStop stops further usage for sdc.
+func (sdc *SDConfig) MustStop() {
+	_ = configMap.Delete(sdc)
+}
+
+// getAppsLabels takes an array of Marathon apps and converts them into labels.
+func getAppsLabels(apps *AppList) []*promutils.Labels {
+	ms := make([]*promutils.Labels, 0, len(apps.Apps))
+	for _, a := range apps.Apps {
+		ms = append(ms, getAppLabels(&a)...)
+	}
+	return ms
+}
+
+func getAppLabels(app *app) []*promutils.Labels {
+	m := promutils.NewLabels(5)
+
+	m.Add("__meta_marathon_app", app.ID)
+	m.Add("__meta_marathon_image", app.Container.Docker.Image)
+
+	var ports []uint32
+	var labels []map[string]string
+	var prefix string
+
+	switch {
+	case len(app.Container.PortMappings) != 0:
+		// In Marathon 1.5.x the "container.docker.portMappings" object was moved
+		// to "container.portMappings".
+		ports, labels = extractPortMapping(app.Container.PortMappings, app.isContainerNet())
+		prefix = "__meta_marathon_port_mapping_label_"
+
+	case len(app.Container.Docker.PortMappings) != 0:
+		// Prior to Marathon 1.5 the port mappings could be found at the path
+		// "container.docker.portMappings".
+		ports, labels = extractPortMapping(app.Container.Docker.PortMappings, app.isContainerNet())
+		prefix = "__meta_marathon_port_mapping_label_"
+
+	case len(app.PortDefinitions) != 0:
+		// PortDefinitions deprecates the "ports" array and can be used to specify
+		// a list of ports with metadata in case a mapping is not required.
+		ports = make([]uint32, len(app.PortDefinitions))
+		labels = make([]map[string]string, len(app.PortDefinitions))
+
+		for i := 0; i < len(app.PortDefinitions); i++ {
+			labels[i] = app.PortDefinitions[i].Labels
+			// When requirePorts is false, this port becomes the 'servicePort', not the listen port.
+			// In this case, the port needs to be taken from the task instead of the app.
+			if app.RequirePorts {
+				ports[i] = app.PortDefinitions[i].Port
+			}
+		}
+
+		prefix = "__meta_marathon_port_definition_label_"
+	}
+
+	for ln, lv := range app.Labels {
+		m.Add("__meta_marathon_app_label_"+discoveryutils.SanitizeLabelName(ln), lv)
+	}
+
+	labelss := make([]*promutils.Labels, 0, len(app.Tasks))
+
+	// Gather info about the app's 'tasks'. Each instance (container) is considered a task
+	// and can be reachable at one or more host:port endpoints.
+	for _, t := range app.Tasks {
+		mm := m.Clone()
+
+		// There are no labels to gather if only Ports is defined. (eg. with host networking)
+		// Ports can only be gathered from the Task (not from the app) and are guaranteed
+		// to be the same across all tasks. If we haven't gathered any ports by now,
+		// use the task's ports as the port list.
+		if len(ports) == 0 && len(t.Ports) != 0 {
+			ports = t.Ports
+		}
+
+		// Iterate over the ports we gathered using one of the methods above.
+		for i, port := range ports {
+			// A zero port here means that either the portMapping has a zero port defined,
+			// or there is a portDefinition with requirePorts set to false. This means the port
+			// is auto-generated by Mesos and needs to be looked up in the task.
+			if port == 0 && len(t.Ports) == len(ports) {
+				port = t.Ports[i]
+			}
+
+			// Each port represents a possible Prometheus target.
+			targetAddress := targetEndpoint(&t, port, app.isContainerNet())
+			mm.Add("__address__", targetAddress)
+			mm.Add("__meta_marathon_task", t.ID)
+			mm.Add("__meta_marathon_port_index", strconv.Itoa(i))
+
+			// Gather all port labels and set them on the current target, skip if the port has no Marathon labels.
+			// This will happen in the host networking case with only `ports` defined, where
+			// it is inefficient to allocate a list of possibly hundreds of empty label maps per host port.
+			if len(labels) > 0 {
+				for ln, lv := range labels[i] {
+					mm.Add(prefix+discoveryutils.SanitizeLabelName(ln), lv)
+				}
+			}
+		}
+		labelss = append(labelss, mm)
+	}
+
+	return labelss
+}
+
+// targetEndpoint Generate a target endpoint string in host:port format.
+func targetEndpoint(task *task, port uint32, containerNet bool) string {
+	var host string
+
+	// Use the task's ipAddress field when it's in a container network
+	if containerNet && len(task.IPAddresses) > 0 {
+		host = task.IPAddresses[0].IPAddress
+	} else {
+		host = task.Host
+	}
+
+	return net.JoinHostPort(host, strconv.Itoa(int(port)))
+}
+
+// extractPortMapping Get a list of ports and a list of labels from a PortMapping.
+func extractPortMapping(portMappings []portMapping, containerNet bool) ([]uint32, []map[string]string) {
+	ports := make([]uint32, len(portMappings))
+	labels := make([]map[string]string, len(portMappings))
+
+	for i := 0; i < len(portMappings); i++ {
+		labels[i] = portMappings[i].Labels
+
+		if containerNet {
+			// If the app is in a container network, connect directly to the container port.
+			ports[i] = portMappings[i].ContainerPort
+		} else {
+			// Otherwise, connect to the allocated host port for the container.
+			// Note that this host port is likely set to 0 in the app definition, which means it is
+			// automatically generated and needs to be extracted from the task's 'ports' array at a later stage.
+			ports[i] = portMappings[i].HostPort
+		}
+	}
+
+	return ports, labels
+}
diff --git a/lib/promscrape/discovery/ovhcloud/dedicated_server.go b/lib/promscrape/discovery/ovhcloud/dedicated_server.go
index 7ed860418..c4f01f741 100644
--- a/lib/promscrape/discovery/ovhcloud/dedicated_server.go
+++ b/lib/promscrape/discovery/ovhcloud/dedicated_server.go
@@ -88,7 +88,7 @@ func getDedicatedServerLabels(cfg *apiConfig) ([]*promutils.Labels, error) {
 	return ms, nil
 }
 
-// getVPSDetails get properties of a dedicated server.
+// getDedicatedServerDetails get properties of a dedicated server.
 // Also see: https://eu.api.ovh.com/console/#/dedicated/server/%7BserviceName%7D~GET
 func getDedicatedServerDetails(cfg *apiConfig, dedicatedServerName string) (*dedicatedServer, error) {
 	// get properties.
diff --git a/lib/promscrape/discovery/vultr/vultr.go b/lib/promscrape/discovery/vultr/vultr.go
index 13de311db..dfb317089 100644
--- a/lib/promscrape/discovery/vultr/vultr.go
+++ b/lib/promscrape/discovery/vultr/vultr.go
@@ -44,7 +44,7 @@ type SDConfig struct {
 
 var configMap = discoveryutils.NewConfigMap()
 
-// GetLabels returns gce labels according to sdc.
+// GetLabels returns Vultr instances' labels according to sdc.
 func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) {
 	ac, err := getAPIConfig(sdc, baseDir)
 	if err != nil {
@@ -62,7 +62,7 @@ func (sdc *SDConfig) MustStop() {
 	_ = configMap.Delete(sdc)
 }
 
-// getInstanceLabels returns labels for vultr instances obtained from the given cfg
+// getInstanceLabels returns labels for Vultr instances obtained from the given cfg
 func getInstanceLabels(instances []Instance, port int) []*promutils.Labels {
 	ms := make([]*promutils.Labels, 0, len(instances))
 
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index d8ad2707e..65c6cea53 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -9,6 +9,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -28,6 +30,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/http"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kuma"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/marathon"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/nomad"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ovhcloud"
@@ -35,7 +38,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/vultr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/yandexcloud"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -141,6 +143,7 @@ func runScraper(configFile string, pushData func(at *auth.Token, wr *prompbmarsh
 	scs.add("http_sd_configs", *http.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getHTTPDScrapeWork(swsPrev) })
 	scs.add("kubernetes_sd_configs", *kubernetes.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getKubernetesSDScrapeWork(swsPrev) })
 	scs.add("kuma_sd_configs", *kuma.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getKumaSDScrapeWork(swsPrev) })
+	scs.add("marathon_sd_configs", *marathon.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getMarathonSDScrapeWork(swsPrev) })
 	scs.add("nomad_sd_configs", *nomad.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getNomadSDScrapeWork(swsPrev) })
 	scs.add("openstack_sd_configs", *openstack.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getOpenStackSDScrapeWork(swsPrev) })
 	scs.add("ovhcloud_sd_configs", *ovhcloud.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getOVHCloudSDScrapeWork(swsPrev) })
diff --git a/lib/promutils/time.go b/lib/promutils/time.go
index a201dd19f..bdef5545c 100644
--- a/lib/promutils/time.go
+++ b/lib/promutils/time.go
@@ -146,7 +146,7 @@ func ParseTimeAt(s string, currentTimestamp int64) (int64, error) {
 	return t.UnixNano(), nil
 }
 
-// parseNumericTimestamps parses timestamp at s in seconds, milliseconds, microseconds or nanoseconds.
+// parseNumericTimestamp parses timestamp at s in seconds, milliseconds, microseconds or nanoseconds.
 //
 // It returns nanoseconds for the parsed timestamp.
 func parseNumericTimestamp(s string) (int64, error) {


Test patch:
diff --git a/app/vmauth/auth_config_test.go b/app/vmauth/auth_config_test.go
index 96a8c65f5..71b1b818e 100644
--- a/app/vmauth/auth_config_test.go
+++ b/app/vmauth/auth_config_test.go
@@ -3,12 +3,14 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"net"
 	"net/url"
 	"testing"
 
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 func TestParseAuthConfigFailure(t *testing.T) {
@@ -799,6 +801,75 @@ func TestBrokenBackend(t *testing.T) {
 	}
 }
 
+func TestDiscoverBackendIPsWithIPV6(t *testing.T) {
+	f := func(actualUrl, expectedUrl string) {
+		t.Helper()
+		up := mustParseURL(actualUrl)
+		up.discoverBackendIPs = true
+		up.loadBalancingPolicy = "least_loaded"
+
+		up.discoverBackendAddrsIfNeeded()
+		pbus := up.bus.Load()
+		bus := *pbus
+
+		if len(bus) != 1 {
+			t.Fatalf("expected url list to be of size 1; got %d instead", len(bus))
+		}
+
+		got := bus[0].url.Host
+		if got != expectedUrl {
+			t.Fatalf(`expected url to be %q; got %q instead`, expectedUrl, bus[0].url.Host)
+		}
+	}
+
+	// Discover backendURL with SRV hostnames
+	customResolver := &fakeResolver{
+		Resolver: &net.Resolver{},
+		// SRV records must return hostname
+		// not an IP address
+		lookupSRVResults: map[string][]*net.SRV{
+			"_vmselect._tcp.selectwithport.": {
+				{
+					Target: "vmselect.local",
+					Port:   8481,
+				},
+			},
+			"_vmselect._tcp.selectwoport.": {
+				{
+					Target: "vmselect.local",
+				},
+			},
+		},
+		lookupIPAddrResults: map[string][]net.IPAddr{
+			"vminsert.local": {
+				{
+					IP: net.ParseIP("10.0.10.13"),
+				},
+			},
+			"ipv6.vminsert.local": {
+				{
+					IP: net.ParseIP("2607:f8b0:400a:80b::200e"),
+				},
+			},
+		},
+	}
+	origResolver := netutil.Resolver
+	netutil.Resolver = customResolver
+	defer func() {
+		netutil.Resolver = origResolver
+	}()
+	f("http://srv+_vmselect._tcp.selectwithport.:8080", "vmselect.local:8080")
+	f("http://srv+_vmselect._tcp.selectwithport.:", "vmselect.local:8481")
+	f("http://srv+_vmselect._tcp.selectwoport.:8080", "vmselect.local:8080")
+	f("http://srv+_vmselect._tcp.selectwoport.", "vmselect.local:")
+
+	f("http://vminsert.local:8080", "10.0.10.13:8080")
+	f("http://vminsert.local", "10.0.10.13:")
+	f("http://ipv6.vminsert.local:8080", "[2607:f8b0:400a:80b::200e]:8080")
+	f("http://ipv6.vminsert.local", "[2607:f8b0:400a:80b::200e]:")
+
+}
+
 func getRegexs(paths []string) []*Regex {
 	var sps []*Regex
 	for _, path := range paths {
diff --git a/apptest/tests/key_concepts_test.go b/apptest/tests/key_concepts_test.go
index 24f519e25..8233c883a 100644
--- a/apptest/tests/key_concepts_test.go
+++ b/apptest/tests/key_concepts_test.go
@@ -50,7 +50,7 @@ func TestClusterKeyConceptsQueryData(t *testing.T) {
 	testKeyConceptsQueryData(t, sut)
 }
 
-// testClusterKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
+// testKeyConceptsQueryData verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
 func testKeyConceptsQueryData(t *testing.T, sut at.PrometheusWriteQuerier) {
 
 	// Insert example data from documentation.
diff --git a/lib/promscrape/discovery/marathon/app_test.go b/lib/promscrape/discovery/marathon/app_test.go
new file mode 100644
index 000000000..fd548f040
--- /dev/null
+++ b/lib/promscrape/discovery/marathon/app_test.go
@@ -0,0 +1,111 @@
+package marathon
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+)
+
+func TestGetAppsList_Success(t *testing.T) {
+	s := newMockMarathonServer(func() []byte {
+		return []byte(`{
+  "apps": [
+    {
+      "id": "/myapp",
+      "cmd": "env && sleep 60",
+      "args": null,
+      "user": null,
+      "env": {
+        "LD_LIBRARY_PATH": "/usr/local/lib/myLib"
+      },
+      "instances": 3,
+      "cpus": 0.1,
+      "mem": 5,
+      "disk": 0,
+      "executor": "",
+      "constraints": [
+        [
+          "hostname",
+          "UNIQUE",
+          ""
+        ]
+      ],
+      "uris": [
+        "https://raw.github.com/mesosphere/marathon/master/README.md"
+      ],
+      "ports": [
+        10013,
+        10015
+      ],
+      "portDefinitions": [
+         {
+            "labels": {"pdl1":"pdl1", "pdl2":"pdl2"},
+            "port": 1999
+         }
+      ],
+      "requirePorts": false,
+      "backoffSeconds": 1,
+      "backoffFactor": 1.15,
+      "maxLaunchDelaySeconds": 3600,
+      "container": null,
+      "healthChecks": [],
+      "dependencies": [],
+      "upgradeStrategy": {
+        "minimumHealthCapacity": 1,
+        "maximumOverCapacity": 1
+      },
+      "labels": {},
+      "acceptedResourceRoles": null,
+      "version": "2015-09-25T15:13:48.343Z",
+      "versionInfo": {
+        "lastScalingAt": "2015-09-25T15:13:48.343Z",
+        "lastConfigChangeAt": "2015-09-25T15:13:48.343Z"
+      },
+      "tasksStaged": 0,
+      "tasksRunning": 0,
+      "tasksHealthy": 0,
+      "tasksUnhealthy": 0,
+      "deployments": [
+        {
+          "id": "9538079c-3898-4e32-aa31-799bf9097f74"
+        }
+      ]
+    }
+  ]
+}`)
+	})
+
+	// Prepare a discovery HTTP client who calls mock server.
+	client, err := discoveryutils.NewClient(s.URL, nil, nil, nil, &promauth.HTTPClientConfig{})
+	if err != nil {
+		t.Fatalf("unexpected error wen creating http client: %s", err)
+	}
+	ac := &apiConfig{
+		cs: []*discoveryutils.Client{client},
+	}
+
+	apps, err := GetAppsList(ac)
+	if err != nil {
+		t.Fatalf("unexpected error in GetAppsList(): %s", err)
+	}
+
+	expect := &AppList{
+		Apps: []app{
+			{
+				ID: "/myapp",
+				PortDefinitions: []portDefinition{
+					{
+						Labels: map[string]string{"pdl1": "pdl1", "pdl2": "pdl2"},
+						Port:   1999,
+					},
+				},
+				Labels: map[string]string{},
+			},
+		},
+	}
+	if !reflect.DeepEqual(apps, expect) {
+		t.Fatalf("unexpected result, got: %v, expect: %v", apps, expect)
+	}
+}
diff --git a/lib/promscrape/discovery/marathon/marathon_test.go b/lib/promscrape/discovery/marathon/marathon_test.go
new file mode 100644
index 000000000..d2254ec0c
--- /dev/null
+++ b/lib/promscrape/discovery/marathon/marathon_test.go
@@ -0,0 +1,230 @@
+package marathon
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+func TestGetAppLabels(t *testing.T) {
+	jsonResponse := `{
+    "apps": [
+        {
+            "id": "/app-test",
+            "tasks": [
+                {
+                    "id": "app-test.b44e0f85-a586-11ef-b308-02429c08177d",
+                    "host": "pre2",
+                    "ports": [
+                        20651
+                    ],
+                    "ipAddresses": [
+                        {
+                            "ipAddress": "172.17.0.12",
+                            "protocol": "IPv4"
+                        }
+                    ]
+                },
+                {
+                    "id": "app-test.7cbfcce5-a586-11ef-b308-02429c08177d",
+                    "host": "pre3",
+                    "ports": [
+                        20681
+                    ],
+                    "ipAddresses": [
+                        {
+                            "ipAddress": "172.17.0.19",
+                            "protocol": "IPv4"
+                        }
+                    ]
+                },
+                {
+                    "id": "app-test.b0a7c3d4-a586-11ef-b308-02429c08177d",
+                    "host": "pre1",
+                    "ports": [
+                        20337
+                    ],
+                    "ipAddresses": [
+                        {
+                            "ipAddress": "172.17.0.13",
+                            "protocol": "IPv4"
+                        }
+                    ]
+                },
+                {
+                    "id": "app-test.7c26c12c-a586-11ef-b308-02429c08177d",
+                    "host": "pre2",
+                    "ports": [
+                        20668
+                    ],
+                    "ipAddresses": [
+                        {
+                            "ipAddress": "172.17.0.9",
+                            "protocol": "IPv4"
+                        }
+                    ]
+                }
+            ],
+            "tasksRunning": 4,
+            "labels": {
+                "HAPROXY_0_HTTP_BACKEND_PROXYPASS_GLUE": "  reqirep  \"^([^ :]*)\\ {proxypath}/?(.*)\" \"\\1\\ /\\2\"\n",
+                "HAPROXY_0_HTTP_BACKEND_PROXYPASS_PATH": "/app-test",
+                "HAPROXY_0_PATH": "-i /app-test",
+                "HAPROXY_0_VHOST": "pre1,pre2,pre3,pre4,pre1:9000,pre",
+                "HAPROXY_GROUP": "local",
+                "micrometer_prometheus": "/actuator/prometheus"
+            },
+            "container": {
+                "docker": {
+                    "image": "docker.local/app-test:1.0.0",
+                    "portMappings": null
+                },
+                "portMappings": [
+                    {
+                        "labels": {
+							"portMappingsLabel1": "portMappingsValue1"
+						},
+                        "containerPort": 8080,
+                        "hostPort": 0,
+                        "servicePort": 11002
+                    }
+                ]
+            },
+            "portDefinitions": [],
+            "networks": [
+                {
+                    "name": "",
+                    "mode": "container/bridge"
+                }
+            ],
+            "requirePorts": false
+        },
+{
+            "id": "/app-test-for-port-definition",
+            "tasks": [
+                {
+                    "id": "app-test.b44e0f85-a586-11ef-b308-02429c08177d",
+                    "host": "pre2",
+                    "ports": [
+                        20651
+                    ],
+                    "ipAddresses": [
+                        {
+                            "ipAddress": "172.17.0.12",
+                            "protocol": "IPv4"
+                        }
+                    ]
+                }
+            ],
+            "tasksRunning": 1,
+            "labels": {
+                "HAPROXY_0_HTTP_BACKEND_PROXYPASS_GLUE": "  reqirep  \"^([^ :]*)\\ {proxypath}/?(.*)\" \"\\1\\ /\\2\"\n",
+                "HAPROXY_0_HTTP_BACKEND_PROXYPASS_PATH": "/app-test",
+                "HAPROXY_0_PATH": "-i /app-test",
+                "HAPROXY_0_VHOST": "pre1,pre2,pre3,pre4,pre1:9000,pre",
+                "HAPROXY_GROUP": "local",
+                "micrometer_prometheus": "/actuator/prometheus"
+            },
+            "container": {
+                "docker": {
+                    "image": "docker.local/app-test:1.0.0",
+                    "portMappings": null
+                },
+                "portMappings": []
+            },
+            "portDefinitions": [
+				{
+					"port" : 9091,
+					"name" : "prometheus",
+					"labels": {
+						"metrics": "/metrics"
+					}
+				}
+			],
+            "networks": [
+                {
+                    "name": "",
+                    "mode": "container/bridge"
+                }
+            ],
+            "requirePorts": false
+        }
+    ]
+}`
+	var appList *AppList
+	if err := json.Unmarshal([]byte(jsonResponse), &appList); err != nil {
+		t.Fatalf("unmarshal jsonResponse failed: %s", err)
+	}
+	result := getAppsLabels(appList)
+	expect := []*promutils.Labels{
+		promutils.NewLabelsFromMap(map[string]string{
+			"__address__":         "pre2:20651",
+			"__meta_marathon_app": "/app-test",
+			"__meta_marathon_app_label_HAPROXY_0_HTTP_BACKEND_PROXYPASS_GLUE": "  reqirep  \"^([^ :]*)\\ {proxypath}/?(.*)\" \"\\1\\ /\\2\"\n",
+			"__meta_marathon_app_label_HAPROXY_0_HTTP_BACKEND_PROXYPASS_PATH": "/app-test",
+			"__meta_marathon_app_label_HAPROXY_0_PATH":                        "-i /app-test",
+			"__meta_marathon_app_label_HAPROXY_0_VHOST":                       "pre1,pre2,pre3,pre4,pre1:9000,pre",
+			"__meta_marathon_app_label_HAPROXY_GROUP":                         "local",
+			"__meta_marathon_app_label_micrometer_prometheus":                 "/actuator/prometheus",
+			"__meta_marathon_image":                                           "docker.local/app-test:1.0.0",
+			"__meta_marathon_port_index":                                      "0",
+			"__meta_marathon_task":                                            "app-test.b44e0f85-a586-11ef-b308-02429c08177d",
+			"__meta_marathon_port_mapping_label_portMappingsLabel1":           "portMappingsValue1",
+		}), promutils.NewLabelsFromMap(map[string]string{
+			"__address__":         "pre3:20681",
+			"__meta_marathon_app": "/app-test",
+			"__meta_marathon_app_label_HAPROXY_0_HTTP_BACKEND_PROXYPASS_GLUE": "  reqirep  \"^([^ :]*)\\ {proxypath}/?(.*)\" \"\\1\\ /\\2\"\n",
+			"__meta_marathon_app_label_HAPROXY_0_HTTP_BACKEND_PROXYPASS_PATH": "/app-test",
+			"__meta_marathon_app_label_HAPROXY_0_PATH":                        "-i /app-test",
+			"__meta_marathon_app_label_HAPROXY_0_VHOST":                       "pre1,pre2,pre3,pre4,pre1:9000,pre",
+			"__meta_marathon_app_label_HAPROXY_GROUP":                         "local",
+			"__meta_marathon_app_label_micrometer_prometheus":                 "/actuator/prometheus",
+			"__meta_marathon_image":                                           "docker.local/app-test:1.0.0",
+			"__meta_marathon_port_index":                                      "0",
+			"__meta_marathon_task":                                            "app-test.7cbfcce5-a586-11ef-b308-02429c08177d",
+			"__meta_marathon_port_mapping_label_portMappingsLabel1":           "portMappingsValue1",
+		}), promutils.NewLabelsFromMap(map[string]string{
+			"__address__":         "pre1:20337",
+			"__meta_marathon_app": "/app-test",
+			"__meta_marathon_app_label_HAPROXY_0_HTTP_BACKEND_PROXYPASS_GLUE": "  reqirep  \"^([^ :]*)\\ {proxypath}/?(.*)\" \"\\1\\ /\\2\"\n",
+			"__meta_marathon_app_label_HAPROXY_0_HTTP_BACKEND_PROXYPASS_PATH": "/app-test",
+			"__meta_marathon_app_label_HAPROXY_0_PATH":                        "-i /app-test",
+			"__meta_marathon_app_label_HAPROXY_0_VHOST":                       "pre1,pre2,pre3,pre4,pre1:9000,pre",
+			"__meta_marathon_app_label_HAPROXY_GROUP":                         "local",
+			"__meta_marathon_app_label_micrometer_prometheus":                 "/actuator/prometheus",
+			"__meta_marathon_image":                                           "docker.local/app-test:1.0.0",
+			"__meta_marathon_port_index":                                      "0",
+			"__meta_marathon_task":                                            "app-test.b0a7c3d4-a586-11ef-b308-02429c08177d",
+			"__meta_marathon_port_mapping_label_portMappingsLabel1":           "portMappingsValue1",
+		}), promutils.NewLabelsFromMap(map[string]string{
+			"__address__":         "pre2:20668",
+			"__meta_marathon_app": "/app-test",
+			"__meta_marathon_app_label_HAPROXY_0_HTTP_BACKEND_PROXYPASS_GLUE": "  reqirep  \"^([^ :]*)\\ {proxypath}/?(.*)\" \"\\1\\ /\\2\"\n",
+			"__meta_marathon_app_label_HAPROXY_0_HTTP_BACKEND_PROXYPASS_PATH": "/app-test",
+			"__meta_marathon_app_label_HAPROXY_0_PATH":                        "-i /app-test",
+			"__meta_marathon_app_label_HAPROXY_0_VHOST":                       "pre1,pre2,pre3,pre4,pre1:9000,pre",
+			"__meta_marathon_app_label_HAPROXY_GROUP":                         "local",
+			"__meta_marathon_app_label_micrometer_prometheus":                 "/actuator/prometheus",
+			"__meta_marathon_image":                                           "docker.local/app-test:1.0.0",
+			"__meta_marathon_port_index":                                      "0",
+			"__meta_marathon_task":                                            "app-test.7c26c12c-a586-11ef-b308-02429c08177d",
+			"__meta_marathon_port_mapping_label_portMappingsLabel1":           "portMappingsValue1",
+		}), promutils.NewLabelsFromMap(map[string]string{
+			"__address__":         "pre2:20651",
+			"__meta_marathon_app": "/app-test-for-port-definition",
+			"__meta_marathon_app_label_HAPROXY_0_HTTP_BACKEND_PROXYPASS_GLUE": "  reqirep  \"^([^ :]*)\\ {proxypath}/?(.*)\" \"\\1\\ /\\2\"\n",
+			"__meta_marathon_app_label_HAPROXY_0_HTTP_BACKEND_PROXYPASS_PATH": "/app-test",
+			"__meta_marathon_app_label_HAPROXY_0_PATH":                        "-i /app-test",
+			"__meta_marathon_app_label_HAPROXY_0_VHOST":                       "pre1,pre2,pre3,pre4,pre1:9000,pre",
+			"__meta_marathon_app_label_HAPROXY_GROUP":                         "local",
+			"__meta_marathon_app_label_micrometer_prometheus":                 "/actuator/prometheus",
+			"__meta_marathon_image":                                           "docker.local/app-test:1.0.0",
+			"__meta_marathon_port_index":                                      "0",
+			"__meta_marathon_task":                                            "app-test.b44e0f85-a586-11ef-b308-02429c08177d",
+			"__meta_marathon_port_definition_label_metrics":                   "/metrics",
+		}),
+	}
+	discoveryutils.TestEqualLabelss(t, result, expect)
+}
diff --git a/lib/promscrape/discovery/marathon/mock_server_test.go b/lib/promscrape/discovery/marathon/mock_server_test.go
new file mode 100644
index 000000000..a4f218343
--- /dev/null
+++ b/lib/promscrape/discovery/marathon/mock_server_test.go
@@ -0,0 +1,23 @@
+package marathon
+
+import (
+	"net/http"
+	"net/http/httptest"
+)
+
+type mockMarathonServer struct {
+	*httptest.Server
+	responseFunc func() []byte
+}
+
+func newMockMarathonServer(responseFunc func() []byte) *mockMarathonServer {
+	var s mockMarathonServer
+	s.responseFunc = responseFunc
+	s.Server = httptest.NewServer(http.HandlerFunc(s.handler))
+	return &s
+}
+
+func (s *mockMarathonServer) handler(w http.ResponseWriter, _ *http.Request) {
+	data := s.responseFunc()
+	w.Write(data)
+}