Repository: VictoriaMetrics/VictoriaMetrics
Pull request #442 (2020-04-26 13:03:25)

app/vmalert: initial remote-write support for alerts state persistence.
If `remotewrite.url` flag is set, vmalert will send alerts state  via remote-write protocol to remote storage. The sending is asynchronous to avoid blocking calls in rules evaluation loop.


Patch:
diff --git a/Makefile b/Makefile
index ec022428d..0a4076ad0 100644
--- a/Makefile
+++ b/Makefile
@@ -13,6 +13,7 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(shell date
 all: \
 	victoria-metrics-prod \
 	vmagent-prod \
+	vmalert-prot \
 	vmbackup-prod \
 	vmrestore-prod
 
@@ -25,17 +26,20 @@ clean:
 publish: \
 	publish-victoria-metrics \
 	publish-vmagent \
+	publish-vmalert \
 	publish-vmbackup \
 	publish-vmrestore
 
 package: \
 	package-victoria-metrics \
 	package-vmagent \
+	package-vmalert \
 	package-vmbackup \
 	package-vmrestore
 
 vmutils: \
 	vmagent \
+	vmalert \
 	vmbackup \
 	vmrestore
 
@@ -49,9 +53,10 @@ release-victoria-metrics: victoria-metrics-prod
 
 release-vmutils: \
 	vmagent-prod \
+	vmalert-prod \
 	vmbackup-prod \
 	vmrestore-prod
-	cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmagent-prod vmbackup-prod vmrestore-prod && \
+	cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmagent-prod vmalert-prod vmbackup-prod vmrestore-prod && \
 		sha256sum vmutils-$(PKG_TAG).tar.gz > vmutils-$(PKG_TAG)_checksums.txt
 
 pprof-cpu:
@@ -78,6 +83,7 @@ errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
diff --git a/README.md b/README.md
index eb5ef83e8..24ee73182 100644
--- a/README.md
+++ b/README.md
@@ -260,6 +260,7 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la
 * [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config)
 * [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)
 * [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config)
+* [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config)
 * [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config)
 
 In the future other `*_sd_config` types will be supported.
diff --git a/app/vmagent/README.md b/app/vmagent/README.md
index 67dcc8c01..71b952103 100644
--- a/app/vmagent/README.md
+++ b/app/vmagent/README.md
@@ -133,13 +133,19 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details.
 * `kubernetes_sd_configs` - for scraping targets in Kubernetes (k8s).
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
+* `ec2_sd_configs` - for scraping targets in Amazone EC2.
+  See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
+  `vmagent` doesn't support `role_arn` config param yet.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
-  `vmagent` supports empty `zone` arg inside `gce_sd_config` - in this case it autodetects all the zones for the given project.
+  `vmagent` provides the following additional functionality `gce_sd_config`:
+  * if `project` arg is missing, then `vmagent` uses the project for the instance where it runs;
+  * if `zone` arg is missing, then `vmagent` uses the zone for the instance where it runs;
+  * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
+  * `zone` may contain arbitrary number of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
 
 The following service discovery mechanisms will be added to `vmagent` soon:
 
-* [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config)
 * [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
 * [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
 
diff --git a/app/vmalert/README.md b/app/vmalert/README.md
index b0d21a6d0..a9fe645e0 100644
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -42,7 +42,7 @@ Then configure `vmalert` accordingly:
         -notifier.url=http://localhost:9093
 ```
 
-Example for `.rules` file bay be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/testdata/rules0-good.rules)
+Example for `.rules` file may be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/testdata/rules0-good.rules)
 
 `vmalert` runs evaluation for every group in a separate goroutine.
 Rules in group evaluated one-by-one sequentially. 
@@ -68,8 +68,12 @@ Usage of vmalert:
         How often to evaluate the rules. Default 1m (default 1m0s)
   -external.url string
         External URL is used as alert's source for sent alerts to the notifier
+  -httpListenAddr string
+        Address to listen for http connections (default ":8880")
   -notifier.url string
         Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
+  -remotewrite.url string
+        Optional URL to remote-write compatible storage where to write timeseriesbased on active alerts. E.g. http://127.0.0.1:8428
   -rule value
         Path to the file with alert rules. 
         Supports patterns. Flag can be specified multiple times. 
diff --git a/app/vmalert/deployment/Dockerfile b/app/vmalert/deployment/Dockerfile
new file mode 100644
index 000000000..db2a365ab
--- /dev/null
+++ b/app/vmalert/deployment/Dockerfile
@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8880
+
+ENTRYPOINT ["/vmalert-prod"]
+ARG src_binary
+COPY $src_binary ./vmalert-prod
diff --git a/app/vmalert/main.go b/app/vmalert/main.go
index 3bd87b5c0..d600bdc55 100644
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -29,11 +30,13 @@ Examples:
  -rule /path/to/file. Path to a single file with alerting rules
  -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
 absolute path to all .yaml files in root.`)
-	validateTemplates  = flag.Bool("rule.validateTemplates", true, "Indicates to validate annotation and label templates")
-	httpListenAddr     = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
-	datasourceURL      = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter. e.g. http://127.0.0.1:8428")
-	basicAuthUsername  = flag.String("datasource.basicAuth.username", "", "Optional basic auth username to use for -datasource.url")
-	basicAuthPassword  = flag.String("datasource.basicAuth.password", "", "Optional basic auth password to use for -datasource.url")
+	validateTemplates = flag.Bool("rule.validateTemplates", true, "Indicates to validate annotation and label templates")
+	httpListenAddr    = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
+	datasourceURL     = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter. e.g. http://127.0.0.1:8428")
+	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username to use for -datasource.url")
+	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password to use for -datasource.url")
+	remoteWriteURL    = flag.String("remotewrite.url", "", "Optional URL to remote-write compatible storage where to write timeseries"+
+		"based on active alerts. E.g. http://127.0.0.1:8428")
 	evaluationInterval = flag.Duration("evaluationInterval", 1*time.Minute, "How often to evaluate the rules. Default 1m")
 	notifierURL        = flag.String("notifier.url", "", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
 	externalURL        = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
@@ -56,7 +59,7 @@ func main() {
 	logger.Infof("reading alert rules configuration file from %s", strings.Join(*rulePath, ";"))
 	groups, err := Parse(*rulePath, *validateTemplates)
 	if err != nil {
-		logger.Fatalf("Cannot parse configuration file: %s", err)
+		logger.Fatalf("cannot parse configuration file: %s", err)
 	}
 
 	w := &watchdog{
@@ -65,6 +68,18 @@ func main() {
 			return fmt.Sprintf("%s/api/v1/%s/%s/status", eu, group, name)
 		}, &http.Client{}),
 	}
+
+	if *remoteWriteURL != "" {
+		c, err := remotewrite.NewClient(ctx, remotewrite.Config{
+			Addr:          *remoteWriteURL,
+			FlushInterval: *evaluationInterval,
+		})
+		if err != nil {
+			logger.Fatalf("failed to init remotewrite client: %s", err)
+		}
+		w.rw = c
+	}
+
 	wg := sync.WaitGroup{}
 	for i := range groups {
 		wg.Add(1)
@@ -82,12 +97,19 @@ func main() {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	cancel()
+	if w.rw != nil {
+		err := w.rw.Close()
+		if err != nil {
+			logger.Fatalf("cannot stop the remotewrite: %s", err)
+		}
+	}
 	wg.Wait()
 }
 
 type watchdog struct {
 	storage       *datasource.VMStorage
 	alertProvider notifier.Notifier
+	rw            *remotewrite.Client
 }
 
 var (
@@ -97,6 +119,13 @@ var (
 	execTotal    = metrics.NewCounter(`vmalert_execution_total`)
 	execErrors   = metrics.NewCounter(`vmalert_execution_errors_total`)
 	execDuration = metrics.NewSummary(`vmalert_execution_duration_seconds`)
+
+	alertsFired      = metrics.NewCounter(`vmalert_alerts_fired_total`)
+	alertsSent       = metrics.NewCounter(`vmalert_alerts_sent_total`)
+	alertsSendErrors = metrics.NewCounter(`vmalert_alerts_send_errors_total`)
+
+	remoteWriteSent   = metrics.NewCounter(`vmalert_remotewrite_sent_total`)
+	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
 )
 
 func (w *watchdog) run(ctx context.Context, group Group, evaluationInterval time.Duration) {
@@ -122,7 +151,26 @@ func (w *watchdog) run(ctx context.Context, group Group, evaluationInterval time
 					continue
 				}
 
-				if err := rule.Send(ctx, w.alertProvider); err != nil {
+				var alertsToSend []notifier.Alert
+				for _, a := range rule.alerts {
+					if a.State != notifier.StatePending {
+						alertsToSend = append(alertsToSend, *a)
+					}
+					if a.State == notifier.StateInactive || w.rw == nil {
+						continue
+					}
+					tss := rule.AlertToTimeSeries(a, execStart)
+					for _, ts := range tss {
+						remoteWriteSent.Inc()
+						if err := w.rw.Push(ts); err != nil {
+							remoteWriteErrors.Inc()
+							logger.Errorf("failed to push timeseries to remotewrite: %s", err)
+						}
+					}
+				}
+				alertsSent.Add(len(alertsToSend))
+				if err := w.alertProvider.Send(alertsToSend); err != nil {
+					alertsSendErrors.Inc()
 					logger.Errorf("failed to send alert for rule %q.%q: %s", group.Name, rule.Name, err)
 				}
 			}
diff --git a/app/vmalert/notifier/alert.go b/app/vmalert/notifier/alert.go
index 766791307..3a2b5de62 100644
--- a/app/vmalert/notifier/alert.go
+++ b/app/vmalert/notifier/alert.go
@@ -103,18 +103,3 @@ func templateAnnotation(dst io.Writer, text string, data alertTplData) error {
 	}
 	return nil
 }
-
-type errGroup struct {
-	errs []string
-}
-
-func (eg *errGroup) err() error {
-	if eg == nil || len(eg.errs) == 0 {
-		return nil
-	}
-	return eg
-}
-
-func (eg *errGroup) Error() string {
-	return fmt.Sprintf("errors:%s", strings.Join(eg.errs, "\n"))
-}
diff --git a/app/vmalert/notifier/utils.go b/app/vmalert/notifier/utils.go
new file mode 100644
index 000000000..8cf7c901c
--- /dev/null
+++ b/app/vmalert/notifier/utils.go
@@ -0,0 +1,21 @@
+package notifier
+
+import (
+	"fmt"
+	"strings"
+)
+
+type errGroup struct {
+	errs []string
+}
+
+func (eg *errGroup) err() error {
+	if eg == nil || len(eg.errs) == 0 {
+		return nil
+	}
+	return eg
+}
+
+func (eg *errGroup) Error() string {
+	return fmt.Sprintf("errors:%s", strings.Join(eg.errs, "\n"))
+}
diff --git a/app/vmalert/remotewrite/remotewrite.go b/app/vmalert/remotewrite/remotewrite.go
new file mode 100644
index 000000000..716e3aa07
--- /dev/null
+++ b/app/vmalert/remotewrite/remotewrite.go
@@ -0,0 +1,187 @@
+package remotewrite
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
+)
+
+// Client is an asynchronous HTTP client for writing
+// timeseries via remote write protocol.
+type Client struct {
+	addr           string
+	c              *http.Client
+	input          chan prompbmarshal.TimeSeries
+	baUser, baPass string
+	flushInterval  time.Duration
+	maxBatchSize   int
+	maxQueueSize   int
+
+	wg     sync.WaitGroup
+	doneCh chan struct{}
+}
+
+type Config struct {
+	// Addr of remote storage
+	Addr string
+
+	BasicAuthUser string
+	BasicAuthPass string
+
+	// MaxBatchSize defines max number of timeseries
+	// to be flushed at once
+	MaxBatchSize int
+	// MaxQueueSize defines max length of input queue
+	// populated by Push method
+	MaxQueueSize int
+	// FlushInterval defines time interval for flushing batches
+	FlushInterval time.Duration
+	// WriteTimeout defines timeout for HTTP write request
+	// to remote storage
+	WriteTimeout time.Duration
+}
+
+const (
+	defaultMaxBatchSize  = 1e3
+	defaultMaxQueueSize  = 100
+	defaultFlushInterval = 5 * time.Second
+	defaultWriteTimeout  = 30 * time.Second
+)
+
+const writePath = "/api/v1/write"
+
+// NewClient returns asynchronous client for
+// writing timeseries via remotewrite protocol.
+func NewClient(ctx context.Context, cfg Config) (*Client, error) {
+	if cfg.Addr == "" {
+		return nil, fmt.Errorf("config.Addr can't be empty")
+	}
+	if cfg.MaxBatchSize == 0 {
+		cfg.MaxBatchSize = defaultMaxBatchSize
+	}
+	if cfg.MaxQueueSize == 0 {
+		cfg.MaxQueueSize = defaultMaxQueueSize
+	}
+	if cfg.FlushInterval == 0 {
+		cfg.FlushInterval = defaultFlushInterval
+	}
+	if cfg.WriteTimeout == 0 {
+		cfg.WriteTimeout = defaultWriteTimeout
+	}
+	c := &Client{
+		c: &http.Client{
+			Timeout: cfg.WriteTimeout,
+		},
+		addr:          strings.TrimSuffix(cfg.Addr, "/") + writePath,
+		baUser:        cfg.BasicAuthUser,
+		baPass:        cfg.BasicAuthPass,
+		flushInterval: cfg.FlushInterval,
+		maxBatchSize:  cfg.MaxBatchSize,
+		doneCh:        make(chan struct{}),
+		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
+	}
+	c.run(ctx)
+	return c, nil
+}
+
+// Push adds timeseries into queue for writing into remote storage.
+// Push returns and error if client is stopped or if queue is full.
+func (c *Client) Push(s prompbmarshal.TimeSeries) error {
+	select {
+	case <-c.doneCh:
+		return fmt.Errorf("client is closed")
+	case c.input <- s:
+		return nil
+	default:
+		return fmt.Errorf("failed to push timeseries - queue is full (%d entries)",
+			c.maxQueueSize)
+	}
+}
+
+// Close stops the client and waits for all goroutines
+// to exit.
+func (c *Client) Close() error {
+	if c.doneCh == nil {
+		return fmt.Errorf("client is already closed")
+	}
+	close(c.input)
+	close(c.doneCh)
+	c.wg.Wait()
+	return nil
+}
+
+func (c *Client) run(ctx context.Context) {
+	ticker := time.NewTicker(c.flushInterval)
+	wr := prompbmarshal.WriteRequest{}
+	shutdown := func() {
+		for ts := range c.input {
+			wr.Timeseries = append(wr.Timeseries, ts)
+		}
+		lastCtx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		c.flush(lastCtx, wr)
+		cancel()
+	}
+	c.wg.Add(1)
+	go func() {
+		defer c.wg.Done()
+		defer ticker.Stop()
+		for {
+			select {
+			case <-c.doneCh:
+				shutdown()
+				return
+			case <-ctx.Done():
+				shutdown()
+				return
+			case <-ticker.C:
+				c.flush(ctx, wr)
+				wr = prompbmarshal.WriteRequest{}
+			case ts := <-c.input:
+				wr.Timeseries = append(wr.Timeseries, ts)
+				if len(wr.Timeseries) >= c.maxBatchSize {
+					c.flush(ctx, wr)
+					wr = prompbmarshal.WriteRequest{}
+				}
+			}
+		}
+	}()
+}
+
+func (c *Client) flush(ctx context.Context, wr prompbmarshal.WriteRequest) {
+	if len(wr.Timeseries) < 1 {
+		return
+	}
+	data, err := wr.Marshal()
+	if err != nil {
+		logger.Errorf("failed to marshal WriteRequest: %s", err)
+		return
+	}
+	req, err := http.NewRequest("POST", c.addr, bytes.NewReader(snappy.Encode(nil, data)))
+	if err != nil {
+		logger.Errorf("failed to create new HTTP request: %s", err)
+		return
+	}
+	if c.baPass != "" {
+		req.SetBasicAuth(c.baUser, c.baPass)
+	}
+	resp, err := c.c.Do(req.WithContext(ctx))
+	if err != nil {
+		logger.Errorf("error getting response from %s:%s", req.URL, err)
+		return
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusNoContent {
+		body, _ := ioutil.ReadAll(resp.Body)
+		logger.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
+		return
+	}
+}
diff --git a/app/vmalert/rule.go b/app/vmalert/rule.go
index b68538937..1a6a78c91 100644
--- a/app/vmalert/rule.go
+++ b/app/vmalert/rule.go
@@ -13,7 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
-	"github.com/VictoriaMetrics/metrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Group grouping array of alert
@@ -117,36 +117,6 @@ func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
 	return nil
 }
 
-// Send sends the active alerts via given
-// notifier.Notifier.
-// See for reference https://prometheus.io/docs/alerting/clients/
-// TODO: add tests for endAt value
-func (r *Rule) Send(_ context.Context, ap notifier.Notifier) error {
-	// copy alerts to new list to avoid locks
-	var alertsCopy []notifier.Alert
-	r.mu.Lock()
-	for _, a := range r.alerts {
-		if a.State == notifier.StatePending {
-			continue
-		}
-		// it is safe to dereference instead of deep-copy
-		// because only simple types may be changed during rule.Exec
-		alertsCopy = append(alertsCopy, *a)
-	}
-	r.mu.Unlock()
-
-	if len(alertsCopy) < 1 {
-		return nil
-	}
-	alertsSent.Add(len(alertsCopy))
-	return ap.Send(alertsCopy)
-}
-
-var (
-	alertsFired = metrics.NewCounter(`vmalert_alerts_fired_total`)
-	alertsSent  = metrics.NewCounter(`vmalert_alerts_sent_total`)
-)
-
 // TODO: consider hashing algorithm in VM
 func hash(m datasource.Metric) uint64 {
 	hash := fnv.New64a()
@@ -235,3 +205,65 @@ func (r *Rule) newAlertAPI(a notifier.Alert) *APIAlert {
 		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
 	}
 }
+
+const (
+	// AlertMetricName is the metric name for synthetic alert timeseries.
+	alertMetricName = "ALERTS"
+	// AlertForStateMetricName is the metric name for 'for' state of alert.
+	alertForStateMetricName = "ALERTS_FOR_STATE"
+
+	// AlertNameLabel is the label name indicating the name of an alert.
+	alertNameLabel = "alertname"
+	// AlertStateLabel is the label name indicating the state of an alert.
+	alertStateLabel = "alertstate"
+)
+
+func (r *Rule) AlertToTimeSeries(a *notifier.Alert, timestamp time.Time) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(r.Name, a, timestamp))
+	if r.For > 0 {
+		tss = append(tss, alertForToTimeSeries(r.Name, a, timestamp))
+	}
+	return tss
+}
+
+func alertToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertMetricName
+	labels[alertNameLabel] = name
+	labels[alertStateLabel] = a.State.String()
+	return newTimeSeries(1, labels, timestamp)
+}
+
+func alertForToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertForStateMetricName
+	labels[alertNameLabel] = name
+	return newTimeSeries(float64(a.Start.Unix()), labels, timestamp)
+}
+
+func newTimeSeries(value float64, labels map[string]string, timestamp time.Time) prompbmarshal.TimeSeries {
+	ts := prompbmarshal.TimeSeries{}
+	ts.Samples = append(ts.Samples, prompbmarshal.Sample{
+		Value:     value,
+		Timestamp: timestamp.UnixNano() / 1e6,
+	})
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		ts.Labels = append(ts.Labels, prompbmarshal.Label{
+			Name:  key,
+			Value: labels[key],
+		})
+	}
+	return ts
+}
diff --git a/app/vmselect/main.go b/app/vmselect/main.go
index 8a3d174f3..f2157b228 100644
--- a/app/vmselect/main.go
+++ b/app/vmselect/main.go
@@ -8,11 +8,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
@@ -43,9 +41,6 @@ func getDefaultMaxConcurrentRequests() int {
 
 // Init initializes vmselect
 func Init() {
-	tmpDirPath := *vmstorage.DataPath + "/tmp"
-	fs.RemoveDirContents(tmpDirPath)
-	netstorage.InitTmpBlocksDir(tmpDirPath)
 	promql.InitRollupResultCache(*vmstorage.DataPath + "/cache/rollupResult")
 
 	concurrencyCh = make(chan struct{}, *maxConcurrentRequests)
diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go
index d32767381..570be1cbb 100644
--- a/app/vmselect/netstorage/netstorage.go
+++ b/app/vmselect/netstorage/netstorage.go
@@ -53,9 +53,8 @@ type Results struct {
 	fetchData bool
 	deadline  Deadline
 
-	tbf *tmpBlocksFile
-
 	packedTimeseries []packedTimeseries
+	sr               *storage.Search
 }
 
 // Len returns the number of results in rss.
@@ -65,8 +64,12 @@ func (rss *Results) Len() int {
 
 // Cancel cancels rss work.
 func (rss *Results) Cancel() {
-	putTmpBlocksFile(rss.tbf)
-	rss.tbf = nil
+	rss.mustClose()
+}
+
+func (rss *Results) mustClose() {
+	putStorageSearch(rss.sr)
+	rss.sr = nil
 }
 
 // RunParallel runs in parallel f for all the results from rss.
@@ -76,10 +79,7 @@ func (rss *Results) Cancel() {
 //
 // rss becomes unusable after the call to RunParallel.
 func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
-	defer func() {
-		putTmpBlocksFile(rss.tbf)
-		rss.tbf = nil
-	}()
+	defer rss.mustClose()
 
 	workersCount := 1 + len(rss.packedTimeseries)/32
 	if workersCount > gomaxprocs {
@@ -106,7 +106,7 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 					err = fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.String())
 					break
 				}
-				if err = pts.Unpack(rss.tbf, rs, rss.tr, rss.fetchData, maxWorkersCount); err != nil {
+				if err = pts.Unpack(rs, rss.tr, rss.fetchData, maxWorkersCount); err != nil {
 					break
 				}
 				if len(rs.Timestamps) == 0 && rss.fetchData {
@@ -156,18 +156,18 @@ var gomaxprocs = runtime.GOMAXPROCS(-1)
 
 type packedTimeseries struct {
 	metricName string
-	addrs      []tmpBlockAddr
+	brs        []storage.BlockRef
 }
 
 // Unpack unpacks pts to dst.
-func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.TimeRange, fetchData bool, maxWorkersCount int) error {
+func (pts *packedTimeseries) Unpack(dst *Result, tr storage.TimeRange, fetchData bool, maxWorkersCount int) error {
 	dst.reset()
 
 	if err := dst.MetricName.Unmarshal(bytesutil.ToUnsafeBytes(pts.metricName)); err != nil {
 		return fmt.Errorf("cannot unmarshal metricName %q: %s", pts.metricName, err)
 	}
 
-	workersCount := 1 + len(pts.addrs)/32
+	workersCount := 1 + len(pts.brs)/32
 	if workersCount > maxWorkersCount {
 		workersCount = maxWorkersCount
 	}
@@ -175,19 +175,19 @@ func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.
 		logger.Panicf("BUG: workersCount cannot be zero")
 	}
 
-	sbs := make([]*sortBlock, 0, len(pts.addrs))
+	sbs := make([]*sortBlock, 0, len(pts.brs))
 	var sbsLock sync.Mutex
 
-	workCh := make(chan tmpBlockAddr, workersCount)
+	workCh := make(chan storage.BlockRef, workersCount)
 	doneCh := make(chan error)
 
 	// Start workers
 	for i := 0; i < workersCount; i++ {
 		go func() {
 			var err error
-			for addr := range workCh {
+			for br := range workCh {
 				sb := getSortBlock()
-				if err = sb.unpackFrom(tbf, addr, tr, fetchData); err != nil {
+				if err = sb.unpackFrom(br, tr, fetchData); err != nil {
 					break
 				}
 
@@ -204,10 +204,10 @@ func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.
 	}
 
 	// Feed workers with work
-	for _, addr := range pts.addrs {
-		workCh <- addr
+	for _, br := range pts.brs {
+		workCh <- br
 	}
-	pts.addrs = pts.addrs[:0]
+	pts.brs = pts.brs[:0]
 	close(workCh)
 
 	// Wait until workers finish
@@ -314,8 +314,8 @@ func (sb *sortBlock) reset() {
 	sb.NextIdx = 0
 }
 
-func (sb *sortBlock) unpackFrom(tbf *tmpBlocksFile, addr tmpBlockAddr, tr storage.TimeRange, fetchData bool) error {
-	tbf.MustReadBlockAt(&sb.b, addr)
+func (sb *sortBlock) unpackFrom(br storage.BlockRef, tr storage.TimeRange, fetchData bool) error {
+	br.MustReadBlock(&sb.b, fetchData)
 	if fetchData {
 		if err := sb.b.UnmarshalData(); err != nil {
 			return fmt.Errorf("cannot unmarshal block: %s", err)
@@ -483,6 +483,8 @@ func putStorageSearch(sr *storage.Search) {
 var ssPool sync.Pool
 
 // ProcessSearchQuery performs sq on storage nodes until the given deadline.
+//
+// Results.RunParallel or Results.Cancel must be called on the returned Results.
 func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadline) (*Results, error) {
 	// Setup search.
 	tfss, err := setupTfss(sq.TagFilterss)
@@ -498,60 +500,40 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadli
 	defer vmstorage.WG.Done()
 
 	sr := getStorageSearch()
-	defer putStorageSearch(sr)
-	sr.Init(vmstorage.Storage, tfss, tr, fetchData, *maxMetricsPerSearch)
+	sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch)
 
-	tbf := getTmpBlocksFile()
-	m := make(map[string][]tmpBlockAddr)
+	m := make(map[string][]storage.BlockRef)
+	var orderedMetricNames []string
 	blocksRead := 0
-	bb := tmpBufPool.Get()
-	defer tmpBufPool.Put(bb)
 	for sr.NextMetricBlock() {
 		blocksRead++
-		bb.B = storage.MarshalBlock(bb.B[:0], sr.MetricBlock.Block)
-		addr, err := tbf.WriteBlockData(bb.B)
-		if err != nil {
-			putTmpBlocksFile(tbf)
-			return nil, fmt.Errorf("cannot write data block #%d to temporary blocks file: %s", blocksRead, err)
-		}
 		if time.Until(deadline.Deadline) < 0 {
-			putTmpBlocksFile(tbf)
 			return nil, fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
 		}
-		metricName := sr.MetricBlock.MetricName
-		m[string(metricName)] = append(m[string(metricName)], addr)
+		metricName := sr.MetricBlockRef.MetricName
+		brs := m[string(metricName)]
+		if len(brs) == 0 {
+			orderedMetricNames = append(orderedMetricNames, string(metricName))
+		}
+		m[string(metricName)] = append(brs, *sr.MetricBlockRef.BlockRef)
 	}
 	if err := sr.Error(); err != nil {
-		putTmpBlocksFile(tbf)
 		return nil, fmt.Errorf("search error after reading %d data blocks: %s", blocksRead, err)
 	}
-	if err := tbf.Finalize(); err != nil {
-		putTmpBlocksFile(tbf)
-		return nil, fmt.Errorf("cannot finalize temporary blocks file with %d blocks: %s", blocksRead, err)
-	}
 
 	var rss Results
-	rss.packedTimeseries = make([]packedTimeseries, len(m))
 	rss.tr = tr
 	rss.fetchData = fetchData
 	rss.deadline = deadline
-	rss.tbf = tbf
-	i := 0
-	for metricName, addrs := range m {
-		pts := &rss.packedTimeseries[i]
-		i++
-		pts.metricName = metricName
-		pts.addrs = addrs
+	pts := make([]packedTimeseries, len(orderedMetricNames))
+	for i, metricName := range orderedMetricNames {
+		pts[i] = packedTimeseries{
+			metricName: metricName,
+			brs:        m[metricName],
+		}
 	}
-
-	// Sort rss.packedTimeseries by the first addr offset in order
-	// to reduce the number of disk seeks during unpacking in RunParallel.
-	// In this case tmpBlocksFile must be read almost sequentially.
-	sort.Slice(rss.packedTimeseries, func(i, j int) bool {
-		pts := rss.packedTimeseries
-		return pts[i].addrs[0].offset < pts[j].addrs[0].offset
-	})
-
+	rss.packedTimeseries = pts
+	rss.sr = sr
 	return &rss, nil
 }
 
diff --git a/app/vmselect/netstorage/tmp_blocks_file.go b/app/vmselect/netstorage/tmp_blocks_file.go
deleted file mode 100644
index 7a278898f..000000000
--- a/app/vmselect/netstorage/tmp_blocks_file.go
+++ /dev/null
@@ -1,185 +0,0 @@
-package netstorage
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// InitTmpBlocksDir initializes directory to store temporary search results.
-//
-// It stores data in system-defined temporary directory if tmpDirPath is empty.
-func InitTmpBlocksDir(tmpDirPath string) {
-	if len(tmpDirPath) == 0 {
-		tmpDirPath = os.TempDir()
-	}
-	tmpBlocksDir = tmpDirPath + "/searchResults"
-	fs.MustRemoveAll(tmpBlocksDir)
-	if err := fs.MkdirAllIfNotExist(tmpBlocksDir); err != nil {
-		logger.Panicf("FATAL: cannot create %q: %s", tmpBlocksDir, err)
-	}
-}
-
-var tmpBlocksDir string
-
-func maxInmemoryTmpBlocksFile() int {
-	mem := memory.Allowed()
-	maxLen := mem / 1024
-	if maxLen < 64*1024 {
-		return 64 * 1024
-	}
-	if maxLen > 4*1024*1024 {
-		return 4 * 1024 * 1024
-	}
-	return maxLen
-}
-
-var _ = metrics.NewGauge(`vm_tmp_blocks_max_inmemory_file_size_bytes`, func() float64 {
-	return float64(maxInmemoryTmpBlocksFile())
-})
-
-type tmpBlocksFile struct {
-	buf []byte
-
-	f *os.File
-	r *fs.ReaderAt
-
-	offset uint64
-}
-
-func getTmpBlocksFile() *tmpBlocksFile {
-	v := tmpBlocksFilePool.Get()
-	if v == nil {
-		return &tmpBlocksFile{
-			buf: make([]byte, 0, maxInmemoryTmpBlocksFile()),
-		}
-	}
-	return v.(*tmpBlocksFile)
-}
-
-func putTmpBlocksFile(tbf *tmpBlocksFile) {
-	tbf.MustClose()
-	tbf.buf = tbf.buf[:0]
-	tbf.f = nil
-	tbf.r = nil
-	tbf.offset = 0
-	tmpBlocksFilePool.Put(tbf)
-}
-
-var tmpBlocksFilePool sync.Pool
-
-type tmpBlockAddr struct {
-	offset uint64
-	size   int
-}
-
-func (addr tmpBlockAddr) String() string {
-	return fmt.Sprintf("offset %d, size %d", addr.offset, addr.size)
-}
-
-var tmpBlocksFilesCreated = metrics.NewCounter(`vm_tmp_blocks_files_created_total`)
-
-// WriteBlockData writes b to tbf.
-//
-// It returns errors since the operation may fail on space shortage
-// and this must be handled.
-func (tbf *tmpBlocksFile) WriteBlockData(b []byte) (tmpBlockAddr, error) {
-	var addr tmpBlockAddr
-	addr.offset = tbf.offset
-	addr.size = len(b)
-	tbf.offset += uint64(addr.size)
-	if len(tbf.buf)+len(b) <= cap(tbf.buf) {
-		// Fast path - the data fits tbf.buf
-		tbf.buf = append(tbf.buf, b...)
-		return addr, nil
-	}
-
-	// Slow path: flush the data from tbf.buf to file.
-	if tbf.f == nil {
-		f, err := ioutil.TempFile(tmpBlocksDir, "")
-		if err != nil {
-			return addr, err
-		}
-		tbf.f = f
-		tmpBlocksFilesCreated.Inc()
-	}
-	_, err := tbf.f.Write(tbf.buf)
-	tbf.buf = append(tbf.buf[:0], b...)
-	if err != nil {
-		return addr, fmt.Errorf("cannot write block to %q: %s", tbf.f.Name(), err)
-	}
-	return addr, nil
-}
-
-func (tbf *tmpBlocksFile) Finalize() error {
-	if tbf.f == nil {
-		return nil
-	}
-	fname := tbf.f.Name()
-	if _, err := tbf.f.Write(tbf.buf); err != nil {
-		return fmt.Errorf("cannot write the remaining %d bytes to %q: %s", len(tbf.buf), fname, err)
-	}
-	tbf.buf = tbf.buf[:0]
-	r, err := fs.OpenReaderAt(fname)
-	if err != nil {
-		logger.Panicf("FATAL: cannot open %q: %s", fname, err)
-	}
-	// Hint the OS that the file is read almost sequentiallly.
-	// This should reduce the number of disk seeks, which is important
-	// for HDDs.
-	r.MustFadviseSequentialRead(true)
-	tbf.r = r
-	return nil
-}
-
-func (tbf *tmpBlocksFile) MustReadBlockAt(dst *storage.Block, addr tmpBlockAddr) {
-	var buf []byte
-	if tbf.f == nil {
-		buf = tbf.buf[addr.offset : addr.offset+uint64(addr.size)]
-	} else {
-		bb := tmpBufPool.Get()
-		defer tmpBufPool.Put(bb)
-		bb.B = bytesutil.Resize(bb.B, addr.size)
-		tbf.r.MustReadAt(bb.B, int64(addr.offset))
-		buf = bb.B
-	}
-	tail, err := storage.UnmarshalBlock(dst, buf)
-	if err != nil {
-		logger.Panicf("FATAL: cannot unmarshal data at %s: %s", addr, err)
-	}
-	if len(tail) > 0 {
-		logger.Panicf("FATAL: unexpected non-empty tail left after unmarshaling data at %s; len(tail)=%d", addr, len(tail))
-	}
-}
-
-var tmpBufPool bytesutil.ByteBufferPool
-
-func (tbf *tmpBlocksFile) MustClose() {
-	if tbf.f == nil {
-		return
-	}
-	if tbf.r != nil {
-		// tbf.r could be nil if Finalize wasn't called.
-		tbf.r.MustClose()
-	}
-	fname := tbf.f.Name()
-
-	// Remove the file at first, then close it.
-	// This way the OS shouldn't try to flush file contents to storage
-	// on close.
-	if err := os.Remove(fname); err != nil {
-		logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
-	}
-	if err := tbf.f.Close(); err != nil {
-		logger.Panicf("FATAL: cannot close %q: %s", fname, err)
-	}
-	tbf.f = nil
-}
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index eb5ef83e8..24ee73182 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -260,6 +260,7 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la
 * [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config)
 * [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)
 * [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config)
+* [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config)
 * [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config)
 
 In the future other `*_sd_config` types will be supported.
diff --git a/docs/vmagent.md b/docs/vmagent.md
index 67dcc8c01..71b952103 100644
--- a/docs/vmagent.md
+++ b/docs/vmagent.md
@@ -133,13 +133,19 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details.
 * `kubernetes_sd_configs` - for scraping targets in Kubernetes (k8s).
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
+* `ec2_sd_configs` - for scraping targets in Amazone EC2.
+  See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
+  `vmagent` doesn't support `role_arn` config param yet.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
-  `vmagent` supports empty `zone` arg inside `gce_sd_config` - in this case it autodetects all the zones for the given project.
+  `vmagent` provides the following additional functionality `gce_sd_config`:
+  * if `project` arg is missing, then `vmagent` uses the project for the instance where it runs;
+  * if `zone` arg is missing, then `vmagent` uses the zone for the instance where it runs;
+  * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
+  * `zone` may contain arbitrary number of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
 
 The following service discovery mechanisms will be added to `vmagent` soon:
 
-* [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config)
 * [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
 * [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
 
diff --git a/lib/fs/fadvise_darwin.go b/lib/fs/fadvise_darwin.go
deleted file mode 100644
index 73cfe81a7..000000000
--- a/lib/fs/fadvise_darwin.go
+++ /dev/null
@@ -1,10 +0,0 @@
-package fs
-
-import (
-	"os"
-)
-
-func fadviseSequentialRead(f *os.File, prefetch bool) error {
-	// TODO: implement this properly
-	return nil
-}
diff --git a/lib/fs/fadvise_unix.go b/lib/fs/fadvise_unix.go
deleted file mode 100644
index 21018796e..000000000
--- a/lib/fs/fadvise_unix.go
+++ /dev/null
@@ -1,22 +0,0 @@
-// +build linux freebsd
-
-package fs
-
-import (
-	"fmt"
-	"os"
-
-	"golang.org/x/sys/unix"
-)
-
-func fadviseSequentialRead(f *os.File, prefetch bool) error {
-	fd := int(f.Fd())
-	mode := unix.FADV_SEQUENTIAL
-	if prefetch {
-		mode |= unix.FADV_WILLNEED
-	}
-	if err := unix.Fadvise(int(fd), 0, 0, mode); err != nil {
-		return fmt.Errorf("error returned from unix.Fadvise(%d): %s", mode, err)
-	}
-	return nil
-}
diff --git a/lib/fs/reader_at.go b/lib/fs/reader_at.go
index 26bf4df3f..23ceb91cb 100644
--- a/lib/fs/reader_at.go
+++ b/lib/fs/reader_at.go
@@ -67,15 +67,6 @@ func (r *ReaderAt) MustClose() {
 	readersCount.Dec()
 }
 
-// MustFadviseSequentialRead hints the OS that f is read mostly sequentially.
-//
-// if prefetch is set, then the OS is hinted to prefetch f data.
-func (r *ReaderAt) MustFadviseSequentialRead(prefetch bool) {
-	if err := fadviseSequentialRead(r.f, prefetch); err != nil {
-		logger.Panicf("FATAL: error in fadviseSequentialRead(%q, %v): %s", r.f.Name(), prefetch, err)
-	}
-}
-
 // OpenReaderAt opens ReaderAt for reading from filename.
 //
 // MustClose must be called on the returned ReaderAt when it is no longer needed.
@@ -94,7 +85,6 @@ func OpenReaderAt(path string) (*ReaderAt, error) {
 		}
 		r.mmapData = data
 	}
-	r.MustFadviseSequentialRead(false)
 	readersCount.Inc()
 	return &r, nil
 }
diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go
index 1a7939c58..0b18d388f 100644
--- a/lib/promscrape/config.go
+++ b/lib/promscrape/config.go
@@ -14,6 +14,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes"
 	"gopkg.in/yaml.v2"
@@ -61,6 +62,7 @@ type ScrapeConfig struct {
 	StaticConfigs        []StaticConfig              `yaml:"static_configs"`
 	FileSDConfigs        []FileSDConfig              `yaml:"file_sd_configs"`
 	KubernetesSDConfigs  []kubernetes.SDConfig       `yaml:"kubernetes_sd_configs"`
+	EC2SDConfigs         []ec2.SDConfig              `yaml:"ec2_sd_configs"`
 	GCESDConfigs         []gce.SDConfig              `yaml:"gce_sd_configs"`
 	RelabelConfigs       []promrelabel.RelabelConfig `yaml:"relabel_configs"`
 	MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs"`
@@ -149,6 +151,14 @@ func (cfg *Config) kubernetesSDConfigsCount() int {
 	return n
 }
 
+func (cfg *Config) ec2SDConfigsCount() int {
+	n := 0
+	for i := range cfg.ScrapeConfigs {
+		n += len(cfg.ScrapeConfigs[i].EC2SDConfigs)
+	}
+	return n
+}
+
 func (cfg *Config) gceSDConfigsCount() int {
 	n := 0
 	for i := range cfg.ScrapeConfigs {
@@ -178,6 +188,19 @@ func (cfg *Config) getKubernetesSDScrapeWork() []ScrapeWork {
 	return dst
 }
 
+// getEC2SDScrapeWork returns `ec2_sd_configs` ScrapeWork from cfg.
+func (cfg *Config) getEC2SDScrapeWork() []ScrapeWork {
+	var dst []ScrapeWork
+	for i := range cfg.ScrapeConfigs {
+		sc := &cfg.ScrapeConfigs[i]
+		for j := range sc.EC2SDConfigs {
+			sdc := &sc.EC2SDConfigs[j]
+			dst = appendEC2ScrapeWork(dst, sdc, sc.swc)
+		}
+	}
+	return dst
+}
+
 // getGCESDScrapeWork returns `gce_sd_configs` ScrapeWork from cfg.
 func (cfg *Config) getGCESDScrapeWork() []ScrapeWork {
 	var dst []ScrapeWork
@@ -323,6 +346,15 @@ func appendKubernetesScrapeWork(dst []ScrapeWork, sdc *kubernetes.SDConfig, base
 	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "kubernetes_sd_config")
 }
 
+func appendEC2ScrapeWork(dst []ScrapeWork, sdc *ec2.SDConfig, swc *scrapeWorkConfig) []ScrapeWork {
+	targetLabels, err := ec2.GetLabels(sdc)
+	if err != nil {
+		logger.Errorf("error when discovering ec2 nodes for `job_name` %q: %s; skipping it", swc.jobName, err)
+		return dst
+	}
+	return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "ec2_sd_config")
+}
+
 func appendGCEScrapeWork(dst []ScrapeWork, sdc *gce.SDConfig, swc *scrapeWorkConfig) []ScrapeWork {
 	targetLabels, err := gce.GetLabels(sdc)
 	if err != nil {
diff --git a/lib/promscrape/discovery/ec2/api.go b/lib/promscrape/discovery/ec2/api.go
new file mode 100644
index 000000000..ffc69ddeb
--- /dev/null
+++ b/lib/promscrape/discovery/ec2/api.go
@@ -0,0 +1,218 @@
+package ec2
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+type apiConfig struct {
+	endpoint  string
+	region    string
+	accessKey string
+	secretKey string
+	filters   string
+	port      int
+}
+
+func getAPIConfig(sdc *SDConfig) (*apiConfig, error) {
+	apiConfigMapLock.Lock()
+	defer apiConfigMapLock.Unlock()
+
+	if !hasAPIConfigMapCleaner {
+		hasAPIConfigMapCleaner = true
+		go apiConfigMapCleaner()
+	}
+
+	e := apiConfigMap[sdc]
+	if e != nil {
+		e.lastAccessTime = time.Now()
+		return e.cfg, nil
+	}
+	cfg, err := newAPIConfig(sdc)
+	if err != nil {
+		return nil, err
+	}
+	apiConfigMap[sdc] = &apiConfigMapEntry{
+		cfg:            cfg,
+		lastAccessTime: time.Now(),
+	}
+	return cfg, nil
+}
+
+func apiConfigMapCleaner() {
+	tc := time.NewTicker(15 * time.Minute)
+	for currentTime := range tc.C {
+		apiConfigMapLock.Lock()
+		for k, e := range apiConfigMap {
+			if currentTime.Sub(e.lastAccessTime) > 10*time.Minute {
+				delete(apiConfigMap, k)
+			}
+		}
+		apiConfigMapLock.Unlock()
+	}
+}
+
+type apiConfigMapEntry struct {
+	cfg            *apiConfig
+	lastAccessTime time.Time
+}
+
+var (
+	apiConfigMap           = make(map[*SDConfig]*apiConfigMapEntry)
+	apiConfigMapLock       sync.Mutex
+	hasAPIConfigMapCleaner bool
+)
+
+func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
+	region := sdc.Region
+	if len(region) == 0 {
+		r, err := getDefaultRegion()
+		if err != nil {
+			return nil, fmt.Errorf("cannot determine default ec2 region; probably, `region` param in `ec2_sd_configs` is missing; the error: %s", err)
+		}
+		region = r
+	}
+	accessKey := sdc.AccessKey
+	if len(accessKey) == 0 {
+		accessKey = os.Getenv("AWS_ACCESS_KEY_ID")
+		if len(accessKey) == 0 {
+			return nil, fmt.Errorf("missing `access_key` in AWS_ACCESS_KEY_ID env var; probably, `access_key` must be set in `ec2_sd_config`?")
+		}
+	}
+	secretKey := sdc.SecretKey
+	if len(secretKey) == 0 {
+		secretKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
+		if len(secretKey) == 0 {
+			return nil, fmt.Errorf("miising `secret_key` in AWS_SECRET_ACCESS_KEY env var; probably, `secret_key` must be set in `ec2_sd_config`?")
+		}
+	}
+	filters := getFiltersQueryString(sdc.Filters)
+	port := 80
+	if sdc.Port != nil {
+		port = *sdc.Port
+	}
+	return &apiConfig{
+		endpoint:  sdc.Endpoint,
+		region:    region,
+		accessKey: accessKey,
+		secretKey: secretKey,
+		filters:   filters,
+		port:      port,
+	}, nil
+}
+
+func getFiltersQueryString(filters []Filter) string {
+	// See how to build filters query string at examples at https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html
+	var args []string
+	for i, f := range filters {
+		args = append(args, fmt.Sprintf("Filter.%d.Name=%s", i+1, url.QueryEscape(f.Name)))
+		for j, v := range f.Values {
+			args = append(args, fmt.Sprintf("Filter.%d.Value.%d=%s", i+1, j+1, url.QueryEscape(v)))
+		}
+	}
+	return strings.Join(args, "&")
+}
+
+func getDefaultRegion() (string, error) {
+	data, err := getMetadataByPath("dynamic/instance-identity/document")
+	if err != nil {
+		return "", err
+	}
+	var id IdentityDocument
+	if err := json.Unmarshal(data, &id); err != nil {
+		return "", fmt.Errorf("cannot parse identity document: %s", err)
+	}
+	return id.Region, nil
+}
+
+// IdentityDocument is identity document returned from AWS metadata server.
+//
+// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-identity-documents.html
+type IdentityDocument struct {
+	Region string
+}
+
+func getMetadataByPath(apiPath string) ([]byte, error) {
+	// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html
+
+	// Obtain session token
+	sessionTokenURL := "http://169.254.169.254/latest/api/token"
+	req, err := http.NewRequest("PUT", sessionTokenURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request for IMDSv2 session token at url %q: %s", sessionTokenURL, err)
+	}
+	req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "60")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain IMDSv2 session token from %q; probably, `region` is missing in `ec2_sd_config`; error: %s", sessionTokenURL, err)
+	}
+	token, err := readResponseBody(resp, sessionTokenURL)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read IMDSv2 session token from %q; probably, `region` is missing in `ec2_sd_config`; error: %s", sessionTokenURL, err)
+	}
+
+	// Use session token in the request.
+	apiURL := "http://169.254.169.254/latest/" + apiPath
+	req, err = http.NewRequest("GET", apiURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request to %q: %s", apiURL, err)
+	}
+	req.Header.Set("X-aws-ec2-metadata-token", string(token))
+	resp, err = http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain response for %q: %s", apiURL, err)
+	}
+	return readResponseBody(resp, apiURL)
+}
+
+func getAPIResponse(cfg *apiConfig, action, nextPageToken string) ([]byte, error) {
+	// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Query-Requests.html
+	endpoint := fmt.Sprintf("https://ec2.%s.amazonaws.com/", cfg.region)
+	if len(cfg.endpoint) > 0 {
+		endpoint = cfg.endpoint
+		// endpoint may contain only hostname. Convert it to proper url then.
+		if !strings.Contains(endpoint, "://") {
+			endpoint = "https://" + endpoint
+		}
+		if !strings.HasSuffix(endpoint, "/") {
+			endpoint += "/"
+		}
+	}
+	apiURL := fmt.Sprintf("%s?Action=%s", endpoint, url.QueryEscape(action))
+	if len(cfg.filters) > 0 {
+		apiURL += "&" + cfg.filters
+	}
+	if len(nextPageToken) > 0 {
+		apiURL += fmt.Sprintf("&NextToken=%s", url.QueryEscape(nextPageToken))
+	}
+	apiURL += "&Version=2013-10-15"
+	req, err := newSignedRequest(apiURL, "ec2", cfg.region, cfg.accessKey, cfg.secretKey)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create signed request: %s", err)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot perform http request to %q: %s", apiURL, err)
+	}
+	return readResponseBody(resp, apiURL)
+}
+
+func readResponseBody(resp *http.Response, apiURL string) ([]byte, error) {
+	data, err := ioutil.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if err != nil {
+		return nil, fmt.Errorf("cannot read response from %q: %s", apiURL, err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code for %q; got %d; want %d; response body: %q",
+			apiURL, resp.StatusCode, http.StatusOK, data)
+	}
+	return data, nil
+}
diff --git a/lib/promscrape/discovery/ec2/ec2.go b/lib/promscrape/discovery/ec2/ec2.go
new file mode 100644
index 000000000..f49902b5b
--- /dev/null
+++ b/lib/promscrape/discovery/ec2/ec2.go
@@ -0,0 +1,44 @@
+package ec2
+
+import (
+	"fmt"
+)
+
+// SDConfig represents service discovery config for ec2.
+//
+// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config
+type SDConfig struct {
+	Region    string `yaml:"region"`
+	Endpoint  string `yaml:"endpoint"`
+	AccessKey string `yaml:"access_key"`
+	SecretKey string `yaml:"secret_key"`
+	Profile   string `yaml:"profile"`
+	// TODO: add support for RoleARN
+	// RoleARN   string `yaml:"role_arn"`
+	// RefreshInterval time.Duration `yaml:"refresh_interval"`
+	// refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option.
+	Port    *int     `yaml:"port"`
+	Filters []Filter `yaml:"filters"`
+}
+
+// Filter is ec2 filter.
+//
+// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html
+// and https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Filter.html
+type Filter struct {
+	Name   string   `yaml:"name"`
+	Values []string `yaml:"values"`
+}
+
+// GetLabels returns ec2 labels according to sdc.
+func GetLabels(sdc *SDConfig) ([]map[string]string, error) {
+	cfg, err := getAPIConfig(sdc)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get API config: %s", err)
+	}
+	ms, err := getInstancesLabels(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("error when fetching instances data from EC2: %s", err)
+	}
+	return ms, nil
+}
diff --git a/lib/promscrape/discovery/ec2/instance.go b/lib/promscrape/discovery/ec2/instance.go
new file mode 100644
index 000000000..6483860a7
--- /dev/null
+++ b/lib/promscrape/discovery/ec2/instance.go
@@ -0,0 +1,177 @@
+package ec2
+
+import (
+	"encoding/xml"
+	"fmt"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+)
+
+// getInstancesLabels returns labels for ec2 instances obtained from the given cfg
+func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) {
+	rs, err := getReservations(cfg)
+	if err != nil {
+		return nil, err
+	}
+	var ms []map[string]string
+	for _, r := range rs {
+		for _, inst := range r.InstanceSet.Items {
+			ms = inst.appendTargetLabels(ms, r.OwnerID, cfg.port)
+		}
+	}
+	return ms, nil
+}
+
+func getReservations(cfg *apiConfig) ([]Reservation, error) {
+	// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html
+	action := "DescribeInstances"
+	var rs []Reservation
+	pageToken := ""
+	for {
+		data, err := getAPIResponse(cfg, action, pageToken)
+		if err != nil {
+			return nil, fmt.Errorf("cannot obtain instances: %s", err)
+		}
+		ir, err := parseInstancesResponse(data)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse instance list: %s", err)
+		}
+		rs = append(rs, ir.ReservationSet.Items...)
+		if len(ir.NextPageToken) == 0 {
+			return rs, nil
+		}
+		pageToken = ir.NextPageToken
+	}
+}
+
+// InstancesResponse represents response to https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html
+type InstancesResponse struct {
+	ReservationSet ReservationSet `xml:"reservationSet"`
+	NextPageToken  string         `xml:"nextToken"`
+}
+
+// ReservationSet represetns ReservationSet from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html
+type ReservationSet struct {
+	Items []Reservation `xml:"item"`
+}
+
+// Reservation represents Reservation from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Reservation.html
+type Reservation struct {
+	OwnerID     string      `xml:"ownerId"`
+	InstanceSet InstanceSet `xml:"instancesSet"`
+}
+
+// InstanceSet represents InstanceSet from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Reservation.html
+type InstanceSet struct {
+	Items []Instance `xml:"item"`
+}
+
+// Instance represents Instance from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Instance.html
+type Instance struct {
+	PrivateIPAddress    string              `xml:"privateIpAddress"`
+	Architecture        string              `xml:"architecture"`
+	Placement           Placement           `xml:"placement"`
+	ID                  string              `xml:"instanceId"`
+	Lifecycle           string              `xml:"instanceLifecycle"`
+	State               InstanceState       `xml:"instanceState"`
+	Type                string              `xml:"instanceType"`
+	Platform            string              `xml:"platform"`
+	SubnetID            string              `xml:"subnetId"`
+	PrivateDNSName      string              `xml:"privateDnsName"`
+	PublicDNSName       string              `xml:"dnsName"`
+	PublicIPAddress     string              `xml:"ipAddress"`
+	VPCID               string              `xml:"vpcId"`
+	NetworkInterfaceSet NetworkInterfaceSet `xml:"networkInterfaceSet"`
+	TagSet              TagSet              `xml:"tagSet"`
+}
+
+// Placement represents Placement from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Placement.html
+type Placement struct {
+	AvailabilityZone string `xml:"availabilityZone"`
+}
+
+// InstanceState represents InstanceState from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_InstanceState.html
+type InstanceState struct {
+	Name string `xml:"name"`
+}
+
+// NetworkInterfaceSet represents NetworkInterfaceSet from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Instance.html
+type NetworkInterfaceSet struct {
+	Items []NetworkInterface `xml:"item"`
+}
+
+// NetworkInterface represents NetworkInterface from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_InstanceNetworkInterface.html
+type NetworkInterface struct {
+	SubnetID string `xml:"subnetId"`
+}
+
+// TagSet represents TagSet from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Instance.html
+type TagSet struct {
+	Items []Tag `xml:"item"`
+}
+
+// Tag represents Tag from https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Tag.html
+type Tag struct {
+	Key   string `xml:"key"`
+	Value string `xml:"value"`
+}
+
+func parseInstancesResponse(data []byte) (*InstancesResponse, error) {
+	var v InstancesResponse
+	if err := xml.Unmarshal(data, &v); err != nil {
+		return nil, fmt.Errorf("cannot unmarshal InstancesResponse from %q: %s", data, err)
+	}
+	return &v, nil
+}
+
+func (inst *Instance) appendTargetLabels(ms []map[string]string, ownerID string, port int) []map[string]string {
+	if len(inst.PrivateIPAddress) == 0 {
+		// Cannot scrape instance without private IP address
+		return ms
+	}
+	addr := discoveryutils.JoinHostPort(inst.PrivateIPAddress, port)
+	m := map[string]string{
+		"__address__":                   addr,
+		"__meta_ec2_architecture":       inst.Architecture,
+		"__meta_ec2_availability_zone":  inst.Placement.AvailabilityZone,
+		"__meta_ec2_instance_id":        inst.ID,
+		"__meta_ec2_instance_lifecycle": inst.Lifecycle,
+		"__meta_ec2_instance_state":     inst.State.Name,
+		"__meta_ec2_instance_type":      inst.Type,
+		"__meta_ec2_owner_id":           ownerID,
+		"__meta_ec2_platform":           inst.Platform,
+		"__meta_ec2_primary_subnet_id":  inst.SubnetID,
+		"__meta_ec2_private_dns_name":   inst.PrivateDNSName,
+		"__meta_ec2_private_ip":         inst.PrivateIPAddress,
+		"__meta_ec2_public_dns_name":    inst.PublicDNSName,
+		"__meta_ec2_public_ip":          inst.PublicIPAddress,
+		"__meta_ec2_vpc_id":             inst.VPCID,
+	}
+	if len(inst.VPCID) > 0 {
+		// Deduplicate VPC Subnet IDs maintaining the order of the network interfaces returned by EC2.
+		subnets := make([]string, 0, len(inst.NetworkInterfaceSet.Items))
+		seenSubnets := make(map[string]bool, len(inst.NetworkInterfaceSet.Items))
+		for _, ni := range inst.NetworkInterfaceSet.Items {
+			if len(ni.SubnetID) == 0 {
+				continue
+			}
+			if !seenSubnets[ni.SubnetID] {
+				seenSubnets[ni.SubnetID] = true
+				subnets = append(subnets, ni.SubnetID)
+			}
+		}
+		// We surround the separated list with the separator as well. This way regular expressions
+		// in relabeling rules don't have to consider tag positions.
+		m["__meta_ec2_subnet_id"] = "," + strings.Join(subnets, ",") + ","
+	}
+	for _, t := range inst.TagSet.Items {
+		if len(t.Key) == 0 || len(t.Value) == 0 {
+			continue
+		}
+		name := discoveryutils.SanitizeLabelName(t.Key)
+		m["__meta_ec2_tag_"+name] = t.Value
+	}
+	ms = append(ms, m)
+	return ms
+}
diff --git a/lib/promscrape/discovery/ec2/sign.go b/lib/promscrape/discovery/ec2/sign.go
new file mode 100644
index 000000000..8f0244313
--- /dev/null
+++ b/lib/promscrape/discovery/ec2/sign.go
@@ -0,0 +1,99 @@
+package ec2
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// newSignedRequest signed request for apiURL according to aws signature algorithm.
+//
+// See the algorithm at https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+func newSignedRequest(apiURL, service, region, accessKey, secretKey string) (*http.Request, error) {
+	t := time.Now().UTC()
+	return newSignedRequestWithTime(apiURL, service, region, accessKey, secretKey, t)
+}
+
+func newSignedRequestWithTime(apiURL, service, region, accessKey, secretKey string, t time.Time) (*http.Request, error) {
+	uri, err := url.Parse(apiURL)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse %q: %s", apiURL, err)
+	}
+
+	// Create canonicalRequest
+	amzdate := t.Format("20060102T150405Z")
+	datestamp := t.Format("20060102")
+	canonicalURL := uri.Path
+	canonicalQS := uri.Query().Encode()
+	canonicalHeaders := fmt.Sprintf("host:%s\nx-amz-date:%s\n", uri.Host, amzdate)
+	signedHeaders := "host;x-amz-date"
+	payloadHash := hashHex("")
+	tmp := []string{
+		"GET",
+		canonicalURL,
+		canonicalQS,
+		canonicalHeaders,
+		signedHeaders,
+		payloadHash,
+	}
+	canonicalRequest := strings.Join(tmp, "\n")
+
+	// Create stringToSign
+	algorithm := "AWS4-HMAC-SHA256"
+	credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
+	tmp = []string{
+		algorithm,
+		amzdate,
+		credentialScope,
+		hashHex(canonicalRequest),
+	}
+	stringToSign := strings.Join(tmp, "\n")
+
+	// Calculate the signature
+	signingKey := getSignatureKey(secretKey, datestamp, region, service)
+	signature := hmacHex(signingKey, stringToSign)
+
+	// Calculate autheader
+	authHeader := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", algorithm, accessKey, credentialScope, signedHeaders, signature)
+
+	req, err := http.NewRequest("GET", apiURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request from %q: %s", apiURL, err)
+	}
+	req.Header.Set("x-amz-date", amzdate)
+	req.Header.Set("Authorization", authHeader)
+	return req, nil
+}
+
+func getSignatureKey(key, datestamp, region, service string) string {
+	kDate := hmacBin("AWS4"+key, datestamp)
+	kRegion := hmacBin(kDate, region)
+	kService := hmacBin(kRegion, service)
+	return hmacBin(kService, "aws4_request")
+}
+
+func hashHex(s string) string {
+	h := sha256.Sum256([]byte(s))
+	return hex.EncodeToString(h[:])
+}
+
+func hmacHex(key, data string) string {
+	digest := hmacBin(key, data)
+	return hex.EncodeToString([]byte(digest))
+}
+
+func hmacBin(key, data string) string {
+	h := hmac.New(sha256.New, []byte(key))
+	_, err := h.Write([]byte(data))
+	if err != nil {
+		logger.Panicf("BUG: unexpected error when writing to hmac: %s", err)
+	}
+	return string(h.Sum(nil))
+}
diff --git a/lib/promscrape/discovery/gce/api.go b/lib/promscrape/discovery/gce/api.go
index 7fd03cf8f..caa4efc98 100644
--- a/lib/promscrape/discovery/gce/api.go
+++ b/lib/promscrape/discovery/gce/api.go
@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"golang.org/x/oauth2/google"
 )
 
@@ -77,16 +78,32 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot create oauth2 client for gce: %s", err)
 	}
-	var zones []string
-	if len(sdc.Zone) == 0 {
-		// Autodetect zones for sdc.Project.
-		zs, err := getZonesForProject(client, sdc.Project, sdc.Filter)
+	project := sdc.Project
+	if len(project) == 0 {
+		proj, err := getCurrentProject()
 		if err != nil {
-			return nil, fmt.Errorf("cannot obtain zones for project %q: %s", sdc.Project, err)
+			return nil, fmt.Errorf("cannot determine the current project; make sure `vmagent` runs inside GCE; error: %s", err)
+		}
+		project = proj
+		logger.Infof("autodetected the current GCE project: %q", project)
+	}
+	zones := sdc.Zone.zones
+	if len(zones) == 0 {
+		// Autodetect the current zone.
+		zone, err := getCurrentZone()
+		if err != nil {
+			return nil, fmt.Errorf("cannot determine the current zone; make sure `vmagent` runs inside GCE; error: %s", err)
+		}
+		zones = append(zones, zone)
+		logger.Infof("autodetected the current GCE zone: %q", zone)
+	} else if len(zones) == 1 && zones[0] == "*" {
+		// Autodetect zones for project.
+		zs, err := getZonesForProject(client, project, sdc.Filter)
+		if err != nil {
+			return nil, fmt.Errorf("cannot obtain zones for project %q: %s", project, err)
 		}
 		zones = zs
-	} else {
-		zones = []string{sdc.Zone}
+		logger.Infof("autodetected all the zones for the GCE project %q: %q", project, zones)
 	}
 	tagSeparator := ","
 	if sdc.TagSeparator != nil {
@@ -99,7 +116,7 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 	return &apiConfig{
 		client:       client,
 		zones:        zones,
-		project:      sdc.Project,
+		project:      project,
 		filter:       sdc.Filter,
 		tagSeparator: tagSeparator,
 		port:         port,
@@ -107,12 +124,16 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 }
 
 func getAPIResponse(client *http.Client, apiURL, filter, pageToken string) ([]byte, error) {
-	apiURL = appendNonEmptyQueryArg(apiURL, filter)
-	apiURL = appendNonEmptyQueryArg(apiURL, pageToken)
+	apiURL = appendNonEmptyQueryArg(apiURL, "filter", filter)
+	apiURL = appendNonEmptyQueryArg(apiURL, "pageToken", pageToken)
 	resp, err := client.Get(apiURL)
 	if err != nil {
 		return nil, fmt.Errorf("cannot query %q: %s", apiURL, err)
 	}
+	return readResponseBody(resp, apiURL)
+}
+
+func readResponseBody(resp *http.Response, apiURL string) ([]byte, error) {
 	data, err := ioutil.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 	if err != nil {
@@ -125,13 +146,50 @@ func getAPIResponse(client *http.Client, apiURL, filter, pageToken string) ([]by
 	return data, nil
 }
 
-func appendNonEmptyQueryArg(apiURL, arg string) string {
-	if len(arg) == 0 {
+func appendNonEmptyQueryArg(apiURL, argName, argValue string) string {
+	if len(argValue) == 0 {
 		return apiURL
 	}
 	prefix := "?"
 	if strings.Contains(apiURL, "?") {
 		prefix = "&"
 	}
-	return apiURL + fmt.Sprintf("%spageToken=%s", prefix, url.QueryEscape(arg))
+	return apiURL + fmt.Sprintf("%s%s=%s", prefix, url.QueryEscape(argName), url.QueryEscape(argValue))
+}
+
+func getCurrentZone() (string, error) {
+	// See https://cloud.google.com/compute/docs/storing-retrieving-metadata#default
+	data, err := getGCEMetadata("instance/zone")
+	if err != nil {
+		return "", err
+	}
+	parts := strings.Split(string(data), "/")
+	if len(parts) != 4 {
+		return "", fmt.Errorf("unexpected data returned from GCE; it must contain something like `projects/projectnum/zones/zone`; data: %q", data)
+	}
+	return parts[3], nil
+}
+
+func getCurrentProject() (string, error) {
+	// See https://cloud.google.com/compute/docs/storing-retrieving-metadata#default
+	data, err := getGCEMetadata("project/project-id")
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+func getGCEMetadata(path string) ([]byte, error) {
+	// See https://cloud.google.com/compute/docs/storing-retrieving-metadata#default
+	metadataURL := "http://metadata.google.internal/computeMetadata/v1/" + path
+	req, err := http.NewRequest("GET", metadataURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create http request for %q: %s", metadataURL, err)
+	}
+	req.Header.Set("Metadata-Flavor", "Google")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain response to %q: %s", metadataURL, err)
+	}
+	return readResponseBody(resp, metadataURL)
 }
diff --git a/lib/promscrape/discovery/gce/gce.go b/lib/promscrape/discovery/gce/gce.go
index ea68627f6..9fe537c25 100644
--- a/lib/promscrape/discovery/gce/gce.go
+++ b/lib/promscrape/discovery/gce/gce.go
@@ -8,15 +8,45 @@ import (
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config
 type SDConfig struct {
-	Project string `yaml:"project"`
-	Zone    string `yaml:"zone"`
-	Filter  string `yaml:"filter"`
+	Project string   `yaml:"project"`
+	Zone    ZoneYAML `yaml:"zone"`
+	Filter  string   `yaml:"filter"`
 	// RefreshInterval time.Duration `yaml:"refresh_interval"`
 	// refresh_interval is obtained from `-promscrape.gceSDCheckInterval` command-line option.
 	Port         *int    `yaml:"port"`
 	TagSeparator *string `yaml:"tag_separator"`
 }
 
+// ZoneYAML holds info about zones.
+type ZoneYAML struct {
+	zones []string
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler
+func (z *ZoneYAML) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var v interface{}
+	if err := unmarshal(&v); err != nil {
+		return err
+	}
+	var zones []string
+	switch t := v.(type) {
+	case string:
+		zones = []string{t}
+	case []interface{}:
+		for _, vv := range t {
+			zone, ok := vv.(string)
+			if !ok {
+				return fmt.Errorf("unexpected zone type detected: %T; contents: %#v", vv, vv)
+			}
+			zones = append(zones, zone)
+		}
+	default:
+		return fmt.Errorf("unexpected type unmarshaled for ZoneYAML: %T; contents: %#v", v, v)
+	}
+	z.zones = zones
+	return nil
+}
+
 // GetLabels returns gce labels according to sdc.
 func GetLabels(sdc *SDConfig) ([]map[string]string, error) {
 	cfg, err := getAPIConfig(sdc)
diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go
index 32d96fbb0..ab45097d1 100644
--- a/lib/promscrape/scraper.go
+++ b/lib/promscrape/scraper.go
@@ -22,6 +22,9 @@ var (
 	kubernetesSDCheckInterval = flag.Duration("promscrape.kubernetesSDCheckInterval", 30*time.Second, "Interval for checking for changes in Kubernetes API server. "+
 		"This works only if `kubernetes_sd_configs` is configured in '-promscrape.config' file. "+
 		"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details")
+	ec2SDCheckInterval = flag.Duration("promscrape.ec2SDCheckInterval", time.Minute, "Interval for checking for changes in ec2. "+
+		"This works only if `ec2_sd_configs` is configured in '-promscrape.config' file. "+
+		"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config for details")
 	gceSDCheckInterval = flag.Duration("promscrape.gceSDCheckInterval", time.Minute, "Interval for checking for changes in gce. "+
 		"This works only if `gce_sd_configs` is configured in '-promscrape.config' file. "+
 		"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details")
@@ -93,6 +96,11 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 			runKubernetesSDScrapers(cfg, pushData, stopCh)
 		}()
 		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			runEC2SDScrapers(cfg, pushData, stopCh)
+		}()
+		wg.Add(1)
 		go func() {
 			defer wg.Done()
 			runGCESDScrapers(cfg, pushData, stopCh)
@@ -202,6 +210,51 @@ var (
 	kubernetesSDReloads = metrics.NewCounter(`vm_promscrape_reloads_total{type="kubernetes_sd"}`)
 )
 
+func runEC2SDScrapers(cfg *Config, pushData func(wr *prompbmarshal.WriteRequest), stopCh <-chan struct{}) {
+	if cfg.ec2SDConfigsCount() == 0 {
+		return
+	}
+	sws := cfg.getEC2SDScrapeWork()
+	ticker := time.NewTicker(*ec2SDCheckInterval)
+	defer ticker.Stop()
+	mustStop := false
+	for !mustStop {
+		localStopCh := make(chan struct{})
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func(sws []ScrapeWork) {
+			defer wg.Done()
+			logger.Infof("starting %d scrapers for `ec2_sd_config` targets", len(sws))
+			ec2SDTargets.Set(uint64(len(sws)))
+			runScrapeWorkers(sws, pushData, localStopCh)
+			ec2SDTargets.Set(0)
+			logger.Infof("stopped all the %d scrapers for `ec2_sd_config` targets", len(sws))
+		}(sws)
+	waitForChans:
+		select {
+		case <-ticker.C:
+			swsNew := cfg.getEC2SDScrapeWork()
+			if equalStaticConfigForScrapeWorks(swsNew, sws) {
+				// Nothing changed, continue waiting for updated scrape work
+				goto waitForChans
+			}
+			logger.Infof("restarting scrapers for changed `ec2_sd_config` targets")
+			sws = swsNew
+		case <-stopCh:
+			mustStop = true
+		}
+
+		close(localStopCh)
+		wg.Wait()
+		ec2SDReloads.Inc()
+	}
+}
+
+var (
+	ec2SDTargets = metrics.NewCounter(`vm_promscrape_targets{type="ec2_sd"}`)
+	ec2SDReloads = metrics.NewCounter(`vm_promscrape_reloads_total{type="ec2_sd"}`)
+)
+
 func runGCESDScrapers(cfg *Config, pushData func(wr *prompbmarshal.WriteRequest), stopCh <-chan struct{}) {
 	if cfg.gceSDConfigsCount() == 0 {
 		return
diff --git a/lib/storage/part_search.go b/lib/storage/part_search.go
index 03aa7ec7b..7e1d02150 100644
--- a/lib/storage/part_search.go
+++ b/lib/storage/part_search.go
@@ -15,8 +15,8 @@ import (
 // partSearch represents blocks stream for the given search args
 // passed to Init.
 type partSearch struct {
-	// Block contains the found block after NextBlock call.
-	Block Block
+	// BlockRef contains the reference to the found block after NextBlock call.
+	BlockRef BlockRef
 
 	// p is the part to search.
 	p *part
@@ -30,9 +30,6 @@ type partSearch struct {
 	// tr is a time range to search.
 	tr TimeRange
 
-	// Skip populating timestampsData and valuesData in Block if fetchData=false.
-	fetchData bool
-
 	metaindex []metaindexRow
 
 	ibCache *indexBlockCache
@@ -49,11 +46,10 @@ type partSearch struct {
 }
 
 func (ps *partSearch) reset() {
-	ps.Block.Reset()
+	ps.BlockRef.reset()
 	ps.p = nil
 	ps.tsids = nil
 	ps.tsidIdx = 0
-	ps.fetchData = true
 	ps.metaindex = nil
 	ps.ibCache = nil
 	ps.bhs = nil
@@ -74,7 +70,7 @@ var isInTest = func() bool {
 //
 // tsids must be sorted.
 // tsids cannot be modified after the Init call, since it is owned by ps.
-func (ps *partSearch) Init(p *part, tsids []TSID, tr TimeRange, fetchData bool) {
+func (ps *partSearch) Init(p *part, tsids []TSID, tr TimeRange) {
 	ps.reset()
 	ps.p = p
 
@@ -86,7 +82,6 @@ func (ps *partSearch) Init(p *part, tsids []TSID, tr TimeRange, fetchData bool)
 		ps.tsids = tsids
 	}
 	ps.tr = tr
-	ps.fetchData = fetchData
 	ps.metaindex = p.metaindex
 	ps.ibCache = p.ibCache
 
@@ -95,7 +90,7 @@ func (ps *partSearch) Init(p *part, tsids []TSID, tr TimeRange, fetchData bool)
 	ps.nextTSID()
 }
 
-// NextBlock advances to the next Block.
+// NextBlock advances to the next BlockRef.
 //
 // Returns true on success.
 //
@@ -130,7 +125,7 @@ func (ps *partSearch) nextTSID() bool {
 		ps.err = io.EOF
 		return false
 	}
-	ps.Block.bh.TSID = ps.tsids[ps.tsidIdx]
+	ps.BlockRef.bh.TSID = ps.tsids[ps.tsidIdx]
 	ps.tsidIdx++
 	return true
 }
@@ -139,20 +134,20 @@ func (ps *partSearch) nextBHS() bool {
 	for len(ps.metaindex) > 0 {
 		// Optimization: skip tsid values smaller than the minimum value
 		// from ps.metaindex.
-		for ps.Block.bh.TSID.Less(&ps.metaindex[0].TSID) {
+		for ps.BlockRef.bh.TSID.Less(&ps.metaindex[0].TSID) {
 			if !ps.nextTSID() {
 				return false
 			}
 		}
-		// Invariant: ps.Block.bh.TSID >= ps.metaindex[0].TSID
+		// Invariant: ps.BlockRef.bh.TSID >= ps.metaindex[0].TSID
 
-		ps.metaindex = skipSmallMetaindexRows(ps.metaindex, &ps.Block.bh.TSID)
-		// Invariant: len(ps.metaindex) > 0 && ps.Block.bh.TSID >= ps.metaindex[0].TSID
+		ps.metaindex = skipSmallMetaindexRows(ps.metaindex, &ps.BlockRef.bh.TSID)
+		// Invariant: len(ps.metaindex) > 0 && ps.BlockRef.bh.TSID >= ps.metaindex[0].TSID
 
 		mr := &ps.metaindex[0]
 		ps.metaindex = ps.metaindex[1:]
-		if ps.Block.bh.TSID.Less(&mr.TSID) {
-			logger.Panicf("BUG: invariant violation: ps.Block.bh.TSID cannot be smaller than mr.TSID; got %+v vs %+v", &ps.Block.bh.TSID, &mr.TSID)
+		if ps.BlockRef.bh.TSID.Less(&mr.TSID) {
+			logger.Panicf("BUG: invariant violation: ps.BlockRef.bh.TSID cannot be smaller than mr.TSID; got %+v vs %+v", &ps.BlockRef.bh.TSID, &mr.TSID)
 		}
 
 		if mr.MaxTimestamp < ps.tr.MinTimestamp {
@@ -165,7 +160,7 @@ func (ps *partSearch) nextBHS() bool {
 		}
 
 		// Found the index block which may contain the required data
-		// for the ps.Block.bh.TSID and the given timestamp range.
+		// for the ps.BlockRef.bh.TSID and the given timestamp range.
 		if ps.indexBlockReuse != nil {
 			putIndexBlock(ps.indexBlockReuse)
 			ps.indexBlockReuse = nil
@@ -249,15 +244,15 @@ func (ps *partSearch) searchBHS() bool {
 		bh := &ps.bhs[i]
 
 	nextTSID:
-		if bh.TSID.Less(&ps.Block.bh.TSID) {
+		if bh.TSID.Less(&ps.BlockRef.bh.TSID) {
 			// Skip blocks with small tsid values.
 			continue
 		}
 
-		// Invariant: ps.Block.bh.TSID <= bh.TSID
+		// Invariant: ps.BlockRef.bh.TSID <= bh.TSID
 
-		if bh.TSID.MetricID != ps.Block.bh.TSID.MetricID {
-			// ps.Block.bh.TSID < bh.TSID: no more blocks with the given tsid.
+		if bh.TSID.MetricID != ps.BlockRef.bh.TSID.MetricID {
+			// ps.BlockRef.bh.TSID < bh.TSID: no more blocks with the given tsid.
 			// Proceed to the next (bigger) tsid.
 			if !ps.nextTSID() {
 				return false
@@ -284,7 +279,7 @@ func (ps *partSearch) searchBHS() bool {
 
 		// Found the tsid block with the matching timestamp range.
 		// Read it.
-		ps.readBlock(bh)
+		ps.BlockRef.init(ps.p, bh)
 
 		ps.bhs = ps.bhs[i+1:]
 		return true
@@ -293,17 +288,3 @@ func (ps *partSearch) searchBHS() bool {
 	ps.bhs = nil
 	return false
 }
-
-func (ps *partSearch) readBlock(bh *blockHeader) {
-	ps.Block.Reset()
-	ps.Block.bh = *bh
-	if !ps.fetchData {
-		return
-	}
-
-	ps.Block.timestampsData = bytesutil.Resize(ps.Block.timestampsData[:0], int(bh.TimestampsBlockSize))
-	ps.p.timestampsFile.MustReadAt(ps.Block.timestampsData, int64(bh.TimestampsBlockOffset))
-
-	ps.Block.valuesData = bytesutil.Resize(ps.Block.valuesData[:0], int(bh.ValuesBlockSize))
-	ps.p.valuesFile.MustReadAt(ps.Block.valuesData, int64(bh.ValuesBlockOffset))
-}
diff --git a/lib/storage/partition_search.go b/lib/storage/partition_search.go
index 3e2ed4020..b7ceaa015 100644
--- a/lib/storage/partition_search.go
+++ b/lib/storage/partition_search.go
@@ -10,8 +10,8 @@ import (
 
 // partitionSearch represents a search in the partition.
 type partitionSearch struct {
-	// Block is the block found after NextBlock call.
-	Block *Block
+	// BlockRef is the block found after NextBlock call.
+	BlockRef *BlockRef
 
 	// pt is a partition to search.
 	pt *partition
@@ -30,7 +30,7 @@ type partitionSearch struct {
 }
 
 func (pts *partitionSearch) reset() {
-	pts.Block = nil
+	pts.BlockRef = nil
 	pts.pt = nil
 
 	for i := range pts.pws {
@@ -59,7 +59,7 @@ func (pts *partitionSearch) reset() {
 // tsids cannot be modified after the Init call, since it is owned by pts.
 //
 /// MustClose must be called when partition search is done.
-func (pts *partitionSearch) Init(pt *partition, tsids []TSID, tr TimeRange, fetchData bool) {
+func (pts *partitionSearch) Init(pt *partition, tsids []TSID, tr TimeRange) {
 	if pts.needClosing {
 		logger.Panicf("BUG: missing partitionSearch.MustClose call before the next call to Init")
 	}
@@ -88,7 +88,7 @@ func (pts *partitionSearch) Init(pt *partition, tsids []TSID, tr TimeRange, fetc
 	}
 	pts.psPool = pts.psPool[:len(pts.pws)]
 	for i, pw := range pts.pws {
-		pts.psPool[i].Init(pw.p, tsids, tr, fetchData)
+		pts.psPool[i].Init(pw.p, tsids, tr)
 	}
 
 	// Initialize the psHeap.
@@ -114,7 +114,7 @@ func (pts *partitionSearch) Init(pt *partition, tsids []TSID, tr TimeRange, fetc
 		return
 	}
 	heap.Init(&pts.psHeap)
-	pts.Block = &pts.psHeap[0].Block
+	pts.BlockRef = &pts.psHeap[0].BlockRef
 	pts.nextBlockNoop = true
 }
 
@@ -145,7 +145,7 @@ func (pts *partitionSearch) nextBlock() error {
 	psMin := pts.psHeap[0]
 	if psMin.NextBlock() {
 		heap.Fix(&pts.psHeap, 0)
-		pts.Block = &pts.psHeap[0].Block
+		pts.BlockRef = &pts.psHeap[0].BlockRef
 		return nil
 	}
 
@@ -159,7 +159,7 @@ func (pts *partitionSearch) nextBlock() error {
 		return io.EOF
 	}
 
-	pts.Block = &pts.psHeap[0].Block
+	pts.BlockRef = &pts.psHeap[0].BlockRef
 	return nil
 }
 
@@ -188,7 +188,7 @@ func (psh *partSearchHeap) Len() int {
 
 func (psh *partSearchHeap) Less(i, j int) bool {
 	x := *psh
-	return x[i].Block.bh.Less(&x[j].Block.bh)
+	return x[i].BlockRef.bh.Less(&x[j].BlockRef.bh)
 }
 
 func (psh *partSearchHeap) Swap(i, j int) {
diff --git a/lib/storage/search.go b/lib/storage/search.go
index 831818686..500608b8f 100644
--- a/lib/storage/search.go
+++ b/lib/storage/search.go
@@ -9,77 +9,55 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-// MetricBlock is a time series block for a single metric.
-type MetricBlock struct {
-	MetricName []byte
-
-	Block *Block
-}
-
-// Marshal marshals MetricBlock to dst
-func (mb *MetricBlock) Marshal(dst []byte) []byte {
-	dst = encoding.MarshalBytes(dst, mb.MetricName)
-	return MarshalBlock(dst, mb.Block)
-}
-
-// MarshalBlock marshals b to dst.
+// BlockRef references a Block.
 //
-// b.MarshalData must be called on b before calling MarshalBlock.
-func MarshalBlock(dst []byte, b *Block) []byte {
-	dst = b.bh.Marshal(dst)
-	dst = encoding.MarshalBytes(dst, b.timestampsData)
-	dst = encoding.MarshalBytes(dst, b.valuesData)
-	return dst
+// BlockRef is valid only until the corresponding Search is valid,
+// i.e. it becomes invalid after Search.MustClose is called.
+type BlockRef struct {
+	p  *part
+	bh blockHeader
 }
 
-// Unmarshal unmarshals MetricBlock from src
-func (mb *MetricBlock) Unmarshal(src []byte) ([]byte, error) {
-	if mb.Block == nil {
-		logger.Panicf("BUG: MetricBlock.Block must be non-nil when calling Unmarshal!")
-	} else {
-		mb.Block.Reset()
-	}
-	tail, mn, err := encoding.UnmarshalBytes(src)
-	if err != nil {
-		return tail, fmt.Errorf("cannot unmarshal MetricName: %s", err)
-	}
-	mb.MetricName = append(mb.MetricName[:0], mn...)
-	src = tail
+func (br *BlockRef) reset() {
+	br.p = nil
+	br.bh = blockHeader{}
+}
 
-	return UnmarshalBlock(mb.Block, src)
+func (br *BlockRef) init(p *part, bh *blockHeader) {
+	br.p = p
+	br.bh = *bh
 }
 
-// UnmarshalBlock unmarshal Block from src to dst.
+// MustReadBlock reads block from br to dst.
 //
-// dst.UnmarshalData isn't called on the block.
-func UnmarshalBlock(dst *Block, src []byte) ([]byte, error) {
-	tail, err := dst.bh.Unmarshal(src)
-	if err != nil {
-		return tail, fmt.Errorf("cannot unmarshal blockHeader: %s", err)
+// if fetchData is false, then only block header is read, otherwise all the data is read.
+func (br *BlockRef) MustReadBlock(dst *Block, fetchData bool) {
+	dst.Reset()
+	dst.bh = br.bh
+	if !fetchData {
+		return
 	}
-	src = tail
 
-	tail, tds, err := encoding.UnmarshalBytes(src)
-	if err != nil {
-		return tail, fmt.Errorf("cannot unmarshal timestampsData: %s", err)
-	}
-	dst.timestampsData = append(dst.timestampsData[:0], tds...)
-	src = tail
+	dst.timestampsData = bytesutil.Resize(dst.timestampsData[:0], int(br.bh.TimestampsBlockSize))
+	br.p.timestampsFile.MustReadAt(dst.timestampsData, int64(br.bh.TimestampsBlockOffset))
 
-	tail, vd, err := encoding.UnmarshalBytes(src)
-	if err != nil {
-		return tail, fmt.Errorf("cannot unmarshal valuesData: %s", err)
-	}
-	dst.valuesData = append(dst.valuesData[:0], vd...)
-	src = tail
+	dst.valuesData = bytesutil.Resize(dst.valuesData[:0], int(br.bh.ValuesBlockSize))
+	br.p.valuesFile.MustReadAt(dst.valuesData, int64(br.bh.ValuesBlockOffset))
+}
 
-	return src, nil
+// MetricBlockRef contains reference to time series block for a single metric.
+type MetricBlockRef struct {
+	// The metric name
+	MetricName []byte
+
+	// The block reference. Call BlockRef.MustReadBlock in order to obtain the block.
+	BlockRef *BlockRef
 }
 
 // Search is a search for time series.
 type Search struct {
-	// MetricBlock is updated with each Search.NextMetricBlock call.
-	MetricBlock MetricBlock
+	// MetricBlockRef is updated with each Search.NextMetricBlock call.
+	MetricBlockRef MetricBlockRef
 
 	storage *Storage
 
@@ -91,8 +69,8 @@ type Search struct {
 }
 
 func (s *Search) reset() {
-	s.MetricBlock.MetricName = s.MetricBlock.MetricName[:0]
-	s.MetricBlock.Block = nil
+	s.MetricBlockRef.MetricName = s.MetricBlockRef.MetricName[:0]
+	s.MetricBlockRef.BlockRef = nil
 
 	s.storage = nil
 	s.ts.reset()
@@ -103,7 +81,7 @@ func (s *Search) reset() {
 // Init initializes s from the given storage, tfss and tr.
 //
 // MustClose must be called when the search is done.
-func (s *Search) Init(storage *Storage, tfss []*TagFilters, tr TimeRange, fetchData bool, maxMetrics int) {
+func (s *Search) Init(storage *Storage, tfss []*TagFilters, tr TimeRange, maxMetrics int) {
 	if s.needClosing {
 		logger.Panicf("BUG: missing MustClose call before the next call to Init")
 	}
@@ -118,7 +96,7 @@ func (s *Search) Init(storage *Storage, tfss []*TagFilters, tr TimeRange, fetchD
 	// It is ok to call Init on error from storage.searchTSIDs.
 	// Init must be called before returning because it will fail
 	// on Seach.MustClose otherwise.
-	s.ts.Init(storage.tb, tsids, tr, fetchData)
+	s.ts.Init(storage.tb, tsids, tr)
 
 	if err != nil {
 		s.err = err
@@ -145,15 +123,15 @@ func (s *Search) Error() error {
 	return s.err
 }
 
-// NextMetricBlock proceeds to the next MetricBlock.
+// NextMetricBlock proceeds to the next MetricBlockRef.
 func (s *Search) NextMetricBlock() bool {
 	if s.err != nil {
 		return false
 	}
 	for s.ts.NextBlock() {
-		tsid := &s.ts.Block.bh.TSID
+		tsid := &s.ts.BlockRef.bh.TSID
 		var err error
-		s.MetricBlock.MetricName, err = s.storage.searchMetricName(s.MetricBlock.MetricName[:0], tsid.MetricID)
+		s.MetricBlockRef.MetricName, err = s.storage.searchMetricName(s.MetricBlockRef.MetricName[:0], tsid.MetricID)
 		if err != nil {
 			if err == io.EOF {
 				// Skip missing metricName for tsid.MetricID.
@@ -163,7 +141,7 @@ func (s *Search) NextMetricBlock() bool {
 			s.err = err
 			return false
 		}
-		s.MetricBlock.Block = s.ts.Block
+		s.MetricBlockRef.BlockRef = s.ts.BlockRef
 		return true
 	}
 	if err := s.ts.Error(); err != nil {
diff --git a/lib/storage/table_search.go b/lib/storage/table_search.go
index 1f4f4ca56..b82ff9011 100644
--- a/lib/storage/table_search.go
+++ b/lib/storage/table_search.go
@@ -11,7 +11,7 @@ import (
 
 // tableSearch performs searches in the table.
 type tableSearch struct {
-	Block *Block
+	BlockRef *BlockRef
 
 	tb *table
 
@@ -29,7 +29,7 @@ type tableSearch struct {
 }
 
 func (ts *tableSearch) reset() {
-	ts.Block = nil
+	ts.BlockRef = nil
 	ts.tb = nil
 
 	for i := range ts.ptws {
@@ -58,7 +58,7 @@ func (ts *tableSearch) reset() {
 // tsids cannot be modified after the Init call, since it is owned by ts.
 //
 // MustClose must be called then the tableSearch is done.
-func (ts *tableSearch) Init(tb *table, tsids []TSID, tr TimeRange, fetchData bool) {
+func (ts *tableSearch) Init(tb *table, tsids []TSID, tr TimeRange) {
 	if ts.needClosing {
 		logger.Panicf("BUG: missing MustClose call before the next call to Init")
 	}
@@ -89,7 +89,7 @@ func (ts *tableSearch) Init(tb *table, tsids []TSID, tr TimeRange, fetchData boo
 	}
 	ts.ptsPool = ts.ptsPool[:len(ts.ptws)]
 	for i, ptw := range ts.ptws {
-		ts.ptsPool[i].Init(ptw.pt, tsids, tr, fetchData)
+		ts.ptsPool[i].Init(ptw.pt, tsids, tr)
 	}
 
 	// Initialize the ptsHeap.
@@ -115,13 +115,13 @@ func (ts *tableSearch) Init(tb *table, tsids []TSID, tr TimeRange, fetchData boo
 		return
 	}
 	heap.Init(&ts.ptsHeap)
-	ts.Block = ts.ptsHeap[0].Block
+	ts.BlockRef = ts.ptsHeap[0].BlockRef
 	ts.nextBlockNoop = true
 }
 
 // NextBlock advances to the next block.
 //
-// The blocks are sorted by (TDIS, MinTimestamp). Two subsequent blocks
+// The blocks are sorted by (TSID, MinTimestamp). Two subsequent blocks
 // for the same TSID may contain overlapped time ranges.
 func (ts *tableSearch) NextBlock() bool {
 	if ts.err != nil {
@@ -146,7 +146,7 @@ func (ts *tableSearch) nextBlock() error {
 	ptsMin := ts.ptsHeap[0]
 	if ptsMin.NextBlock() {
 		heap.Fix(&ts.ptsHeap, 0)
-		ts.Block = ts.ptsHeap[0].Block
+		ts.BlockRef = ts.ptsHeap[0].BlockRef
 		return nil
 	}
 
@@ -160,7 +160,7 @@ func (ts *tableSearch) nextBlock() error {
 		return io.EOF
 	}
 
-	ts.Block = ts.ptsHeap[0].Block
+	ts.BlockRef = ts.ptsHeap[0].BlockRef
 	return nil
 }
 
@@ -192,7 +192,7 @@ func (ptsh *partitionSearchHeap) Len() int {
 
 func (ptsh *partitionSearchHeap) Less(i, j int) bool {
 	x := *ptsh
-	return x[i].Block.bh.Less(&x[j].Block.bh)
+	return x[i].BlockRef.bh.Less(&x[j].BlockRef.bh)
 }
 
 func (ptsh *partitionSearchHeap) Swap(i, j int) {


Test patch:
diff --git a/app/vmalert/rule_test.go b/app/vmalert/rule_test.go
index 9538c94f2..f3ccded06 100644
--- a/app/vmalert/rule_test.go
+++ b/app/vmalert/rule_test.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestRule_Validate(t *testing.T) {
@@ -24,6 +25,122 @@ func TestRule_Validate(t *testing.T) {
 	}
 }
 
+func TestRule_AlertToTimeSeries(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		rule  *Rule
+		alert *notifier.Alert
+		expTS []prompbmarshal.TimeSeries
+	}{
+		{
+			newTestRule("instant", 0),
+			&notifier.Alert{State: notifier.StateFiring},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("instant extra labels", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				"job":      "foo",
+				"instance": "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant extra labels",
+					"job":           "foo",
+					"instance":      "bar",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("instant labels override", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				alertStateLabel: "foo",
+				"__name__":      "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant labels override",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("for", time.Second),
+			&notifier.Alert{State: notifier.StateFiring, Start: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "for",
+				}, timestamp),
+				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
+					"__name__":     alertForStateMetricName,
+					alertNameLabel: "for",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("for pending", 10*time.Second),
+			&notifier.Alert{State: notifier.StatePending, Start: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StatePending.String(),
+					alertNameLabel:  "for pending",
+				}, timestamp),
+				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
+					"__name__":     alertForStateMetricName,
+					alertNameLabel: "for pending",
+				}, timestamp),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			tss := tc.rule.AlertToTimeSeries(tc.alert, timestamp)
+			if len(tc.expTS) != len(tss) {
+				t.Fatalf("expected number of timeseries %d; got %d", len(tc.expTS), len(tss))
+			}
+			for i := range tc.expTS {
+				expTS, gotTS := tc.expTS[i], tss[i]
+				if len(expTS.Samples) != len(gotTS.Samples) {
+					t.Fatalf("expected number of samples %d; got %d", len(expTS.Samples), len(gotTS.Samples))
+				}
+				for i, exp := range expTS.Samples {
+					got := gotTS.Samples[i]
+					if got.Value != exp.Value {
+						t.Errorf("expected value %.2f; got %.2f", exp.Value, got.Value)
+					}
+					if got.Timestamp != exp.Timestamp {
+						t.Errorf("expected timestamp %d; got %d", exp.Timestamp, got.Timestamp)
+					}
+				}
+				if len(expTS.Labels) != len(gotTS.Labels) {
+					t.Fatalf("expected number of labels %d; got %d", len(expTS.Labels), len(gotTS.Labels))
+				}
+				for i, exp := range expTS.Labels {
+					got := gotTS.Labels[i]
+					if got.Name != exp.Name {
+						t.Errorf("expected label name %q; got %q", exp.Name, got.Name)
+					}
+					if got.Value != exp.Value {
+						t.Errorf("expected label value %q; got %q", exp.Value, got.Value)
+					}
+				}
+			}
+		})
+	}
+}
+
 func newTestRule(name string, waitFor time.Duration) *Rule {
 	return &Rule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor}
 }
diff --git a/app/vmselect/netstorage/tmp_blocks_file_test.go b/app/vmselect/netstorage/tmp_blocks_file_test.go
deleted file mode 100644
index eba2292a9..000000000
--- a/app/vmselect/netstorage/tmp_blocks_file_test.go
+++ /dev/null
@@ -1,153 +0,0 @@
-package netstorage
-
-import (
-	"fmt"
-	"math/rand"
-	"os"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-)
-
-func TestMain(m *testing.M) {
-	rand.Seed(time.Now().UnixNano())
-	tmpDir := "TestTmpBlocks"
-	InitTmpBlocksDir(tmpDir)
-	statusCode := m.Run()
-	if err := os.RemoveAll(tmpDir); err != nil {
-		logger.Panicf("cannot remove %q: %s", tmpDir, err)
-	}
-	os.Exit(statusCode)
-}
-
-func TestTmpBlocksFileSerial(t *testing.T) {
-	if err := testTmpBlocksFile(); err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-}
-
-func TestTmpBlocksFileConcurrent(t *testing.T) {
-	concurrency := 3
-	ch := make(chan error, concurrency)
-	for i := 0; i < concurrency; i++ {
-		go func() {
-			ch <- testTmpBlocksFile()
-		}()
-	}
-	for i := 0; i < concurrency; i++ {
-		select {
-		case err := <-ch:
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
-		case <-time.After(30 * time.Second):
-			t.Fatalf("timeout")
-		}
-	}
-}
-
-func testTmpBlocksFile() error {
-	createBlock := func() *storage.Block {
-		rowsCount := rand.Intn(8000) + 1
-		var timestamps, values []int64
-		ts := int64(rand.Intn(1023434))
-		for i := 0; i < rowsCount; i++ {
-			ts += int64(rand.Intn(1000) + 1)
-			timestamps = append(timestamps, ts)
-			values = append(values, int64(i*i+rand.Intn(20)))
-		}
-		tsid := &storage.TSID{
-			MetricID: 234211,
-		}
-		scale := int16(rand.Intn(123))
-		precisionBits := uint8(rand.Intn(63) + 1)
-		var b storage.Block
-		b.Init(tsid, timestamps, values, scale, precisionBits)
-		_, _, _ = b.MarshalData(0, 0)
-		return &b
-	}
-	for _, size := range []int{1024, 16 * 1024, maxInmemoryTmpBlocksFile() / 2, 2 * maxInmemoryTmpBlocksFile()} {
-		err := func() error {
-			tbf := getTmpBlocksFile()
-			defer putTmpBlocksFile(tbf)
-
-			// Write blocks until their summary size exceeds `size`.
-			var addrs []tmpBlockAddr
-			var blocks []*storage.Block
-			bb := tmpBufPool.Get()
-			defer tmpBufPool.Put(bb)
-			for tbf.offset < uint64(size) {
-				b := createBlock()
-				bb.B = storage.MarshalBlock(bb.B[:0], b)
-				addr, err := tbf.WriteBlockData(bb.B)
-				if err != nil {
-					return fmt.Errorf("cannot write block at offset %d: %s", tbf.offset, err)
-				}
-				if addr.offset+uint64(addr.size) != tbf.offset {
-					return fmt.Errorf("unexpected addr=%+v for offset %v", &addr, tbf.offset)
-				}
-				addrs = append(addrs, addr)
-				blocks = append(blocks, b)
-			}
-			if err := tbf.Finalize(); err != nil {
-				return fmt.Errorf("cannot finalize tbf: %s", err)
-			}
-
-			// Read blocks in parallel and verify them
-			concurrency := 2
-			workCh := make(chan int)
-			doneCh := make(chan error)
-			for i := 0; i < concurrency; i++ {
-				go func() {
-					doneCh <- func() error {
-						var b1 storage.Block
-						for idx := range workCh {
-							addr := addrs[idx]
-							b := blocks[idx]
-							if err := b.UnmarshalData(); err != nil {
-								return fmt.Errorf("cannot unmarshal data from the original block: %s", err)
-							}
-							b1.Reset()
-							tbf.MustReadBlockAt(&b1, addr)
-							if err := b1.UnmarshalData(); err != nil {
-								return fmt.Errorf("cannot unmarshal data from tbf: %s", err)
-							}
-							if b1.RowsCount() != b.RowsCount() {
-								return fmt.Errorf("unexpected number of rows in tbf block; got %d; want %d", b1.RowsCount(), b.RowsCount())
-							}
-							if !reflect.DeepEqual(b1.Timestamps(), b.Timestamps()) {
-								return fmt.Errorf("unexpected timestamps; got\n%v\nwant\n%v", b1.Timestamps(), b.Timestamps())
-							}
-							if !reflect.DeepEqual(b1.Values(), b.Values()) {
-								return fmt.Errorf("unexpected values; got\n%v\nwant\n%v", b1.Values(), b.Values())
-							}
-						}
-						return nil
-					}()
-				}()
-			}
-			for i := range addrs {
-				workCh <- i
-			}
-			close(workCh)
-			for i := 0; i < concurrency; i++ {
-				select {
-				case err := <-doneCh:
-					if err != nil {
-						return err
-					}
-				case <-time.After(time.Second):
-					return fmt.Errorf("timeout")
-				}
-			}
-			return nil
-		}()
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
diff --git a/lib/decimal/decimal_test.go b/lib/decimal/decimal_test.go
index 1791a2408..87762cc5e 100644
--- a/lib/decimal/decimal_test.go
+++ b/lib/decimal/decimal_test.go
@@ -164,7 +164,7 @@ func TestMaxUpExponent(t *testing.T) {
 
 		e := maxUpExponent(v)
 		if e != eExpected {
-			t.Fatalf("unexpected e for v=%d; got %d; epxecting %d", v, e, eExpected)
+			t.Fatalf("unexpected e for v=%d; got %d; expecting %d", v, e, eExpected)
 		}
 		e = maxUpExponent(-v)
 		if e != eExpected {
diff --git a/lib/encoding/encoding_cgo_test.go b/lib/encoding/encoding_cgo_test.go
index 2d31cb7a4..b85c5f9d5 100644
--- a/lib/encoding/encoding_cgo_test.go
+++ b/lib/encoding/encoding_cgo_test.go
@@ -70,9 +70,9 @@ func TestMarshalInt64ArraySize(t *testing.T) {
 		v += 30e3 + int64(rand.NormFloat64()*1e3)
 	}
 
-	testMarshalInt64ArraySize(t, va, 1, 500, 1300)
-	testMarshalInt64ArraySize(t, va, 2, 500, 1400)
-	testMarshalInt64ArraySize(t, va, 3, 800, 1800)
+	testMarshalInt64ArraySize(t, va, 1, 180, 1400)
+	testMarshalInt64ArraySize(t, va, 2, 250, 1550)
+	testMarshalInt64ArraySize(t, va, 3, 600, 1800)
 	testMarshalInt64ArraySize(t, va, 4, 1300, 2100)
 	testMarshalInt64ArraySize(t, va, 5, 2000, 3200)
 	testMarshalInt64ArraySize(t, va, 6, 3000, 4800)
diff --git a/lib/encoding/encoding_test.go b/lib/encoding/encoding_test.go
index 588a25dac..fd3173aa3 100644
--- a/lib/encoding/encoding_test.go
+++ b/lib/encoding/encoding_test.go
@@ -214,6 +214,6 @@ func testMarshalInt64ArraySize(t *testing.T, va []int64, precisionBits uint8, mi
 		t.Fatalf("too big size for marshaled %d items with precisionBits %d: got %d; expecting %d", len(va), precisionBits, len(b), maxSizeExpected)
 	}
 	if len(b) < minSizeExpected {
-		t.Fatalf("too small size for marshaled %d items with precisionBits %d: got %d; epxecting %d", len(va), precisionBits, len(b), minSizeExpected)
+		t.Fatalf("too small size for marshaled %d items with precisionBits %d: got %d; expecting %d", len(va), precisionBits, len(b), minSizeExpected)
 	}
 }
diff --git a/lib/promscrape/discovery/ec2/instance_test.go b/lib/promscrape/discovery/ec2/instance_test.go
new file mode 100644
index 000000000..3c1f736de
--- /dev/null
+++ b/lib/promscrape/discovery/ec2/instance_test.go
@@ -0,0 +1,219 @@
+package ec2
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+)
+
+func TestParseInstancesResponse(t *testing.T) {
+	data := `<?xml version="1.0" encoding="UTF-8"?>
+<DescribeInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-15/">
+    <requestId>98667f8e-7fb6-441b-a612-41c6268c6399</requestId>
+    <reservationSet>
+        <item>
+            <reservationId>r-05534f81f74ea7036</reservationId>
+            <ownerId>793614593844</ownerId>
+            <groupSet/>
+            <instancesSet>
+                <item>
+                    <instanceId>i-0e730b692d9c15460</instanceId>
+                    <imageId>ami-0eb89db7593b5d434</imageId>
+                    <instanceState>
+                        <code>16</code>
+                        <name>running</name>
+                    </instanceState>
+                    <privateDnsName>ip-172-31-11-152.eu-west-2.compute.internal</privateDnsName>
+                    <dnsName>ec2-3-8-232-141.eu-west-2.compute.amazonaws.com</dnsName>
+                    <reason/>
+                    <keyName>my-laptop</keyName>
+                    <amiLaunchIndex>0</amiLaunchIndex>
+                    <productCodes/>
+                    <instanceType>t2.micro</instanceType>
+                    <launchTime>2020-04-27T09:19:26.000Z</launchTime>
+                    <placement>
+                        <availabilityZone>eu-west-2c</availabilityZone>
+                        <groupName/>
+                        <tenancy>default</tenancy>
+                    </placement>
+                    <monitoring>
+                        <state>disabled</state>
+                    </monitoring>
+                    <subnetId>subnet-57044c3e</subnetId>
+                    <vpcId>vpc-f1eaad99</vpcId>
+                    <privateIpAddress>172.31.11.152</privateIpAddress>
+                    <ipAddress>3.8.232.141</ipAddress>
+                    <sourceDestCheck>true</sourceDestCheck>
+                    <groupSet>
+                        <item>
+                            <groupId>sg-05d74e4e8551bd020</groupId>
+                            <groupName>launch-wizard-1</groupName>
+                        </item>
+                    </groupSet>
+                    <architecture>x86_64</architecture>
+                    <rootDeviceType>ebs</rootDeviceType>
+                    <rootDeviceName>/dev/sda1</rootDeviceName>
+                    <blockDeviceMapping>
+                        <item>
+                            <deviceName>/dev/sda1</deviceName>
+                            <ebs>
+                                <volumeId>vol-0153ef24058482522</volumeId>
+                                <status>attached</status>
+                                <attachTime>2020-04-27T09:19:27.000Z</attachTime>
+                                <deleteOnTermination>true</deleteOnTermination>
+                            </ebs>
+                        </item>
+                    </blockDeviceMapping>
+                    <virtualizationType>hvm</virtualizationType>
+                    <clientToken/>
+                    <tagSet>
+                        <item>
+                            <key>foo</key>
+                            <value>bar</value>
+                        </item>
+                    </tagSet>
+                    <hypervisor>xen</hypervisor>
+                    <networkInterfaceSet>
+                        <item>
+                            <networkInterfaceId>eni-01d7b338ea037a60b</networkInterfaceId>
+                            <subnetId>subnet-57044c3e</subnetId>
+                            <vpcId>vpc-f1eaad99</vpcId>
+                            <description/>
+                            <ownerId>793614593844</ownerId>
+                            <status>in-use</status>
+                            <macAddress>02:3b:63:46:13:9a</macAddress>
+                            <privateIpAddress>172.31.11.152</privateIpAddress>
+                            <privateDnsName>ip-172-31-11-152.eu-west-2.compute.internal</privateDnsName>
+                            <sourceDestCheck>true</sourceDestCheck>
+                            <groupSet>
+                                <item>
+                                    <groupId>sg-05d74e4e8551bd020</groupId>
+                                    <groupName>launch-wizard-1</groupName>
+                                </item>
+                            </groupSet>
+                            <attachment>
+                                <attachmentId>eni-attach-030cc2cdffe745682</attachmentId>
+                                <deviceIndex>0</deviceIndex>
+                                <status>attached</status>
+                                <attachTime>2020-04-27T09:19:26.000Z</attachTime>
+                                <deleteOnTermination>true</deleteOnTermination>
+                            </attachment>
+                            <association>
+                                <publicIp>3.8.232.141</publicIp>
+                                <publicDnsName>ec2-3-8-232-141.eu-west-2.compute.amazonaws.com</publicDnsName>
+                                <ipOwnerId>amazon</ipOwnerId>
+                            </association>
+                            <privateIpAddressesSet>
+                                <item>
+                                    <privateIpAddress>172.31.11.152</privateIpAddress>
+                                    <privateDnsName>ip-172-31-11-152.eu-west-2.compute.internal</privateDnsName>
+                                    <primary>true</primary>
+                                    <association>
+                                    <publicIp>3.8.232.141</publicIp>
+                                    <publicDnsName>ec2-3-8-232-141.eu-west-2.compute.amazonaws.com</publicDnsName>
+                                    <ipOwnerId>amazon</ipOwnerId>
+                                    </association>
+                                </item>
+                            </privateIpAddressesSet>
+                        </item>
+                    </networkInterfaceSet>
+                    <ebsOptimized>false</ebsOptimized>
+		    <instanceLifecycle>spot</instanceLifecycle>
+		    <platform>windows</platform>
+                </item>
+            </instancesSet>
+        </item>
+    </reservationSet>
+</DescribeInstancesResponse>
+`
+	ir, err := parseInstancesResponse([]byte(data))
+	if err != nil {
+		t.Fatalf("unexpected error when parsing data: %s", err)
+	}
+	irExpected := &InstancesResponse{
+		ReservationSet: ReservationSet{
+			Items: []Reservation{
+				{
+					OwnerID: "793614593844",
+					InstanceSet: InstanceSet{
+						Items: []Instance{
+							{
+								PrivateIPAddress: "172.31.11.152",
+								Architecture:     "x86_64",
+								Placement: Placement{
+									AvailabilityZone: "eu-west-2c",
+								},
+								ID:        "i-0e730b692d9c15460",
+								Lifecycle: "spot",
+								State: InstanceState{
+									Name: "running",
+								},
+								Type:            "t2.micro",
+								Platform:        "windows",
+								SubnetID:        "subnet-57044c3e",
+								PrivateDNSName:  "ip-172-31-11-152.eu-west-2.compute.internal",
+								PublicDNSName:   "ec2-3-8-232-141.eu-west-2.compute.amazonaws.com",
+								PublicIPAddress: "3.8.232.141",
+								VPCID:           "vpc-f1eaad99",
+								NetworkInterfaceSet: NetworkInterfaceSet{
+									Items: []NetworkInterface{
+										{
+											SubnetID: "subnet-57044c3e",
+										},
+									},
+								},
+								TagSet: TagSet{
+									Items: []Tag{
+										{
+											Key:   "foo",
+											Value: "bar",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	if !reflect.DeepEqual(ir, irExpected) {
+		t.Fatalf("unexpected InstancesResponse parsed;\ngot\n%+v\nwant\n%+v", ir, irExpected)
+	}
+
+	rs := ir.ReservationSet.Items[0]
+	ownerID := rs.OwnerID
+	port := 423
+	inst := rs.InstanceSet.Items[0]
+	labelss := inst.appendTargetLabels(nil, ownerID, port)
+	var sortedLabelss [][]prompbmarshal.Label
+	for _, labels := range labelss {
+		sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
+	}
+	expectedLabels := [][]prompbmarshal.Label{
+		discoveryutils.GetSortedLabels(map[string]string{
+			"__address__":                   "172.31.11.152:423",
+			"__meta_ec2_architecture":       "x86_64",
+			"__meta_ec2_availability_zone":  "eu-west-2c",
+			"__meta_ec2_instance_id":        "i-0e730b692d9c15460",
+			"__meta_ec2_instance_lifecycle": "spot",
+			"__meta_ec2_instance_state":     "running",
+			"__meta_ec2_instance_type":      "t2.micro",
+			"__meta_ec2_owner_id":           "793614593844",
+			"__meta_ec2_platform":           "windows",
+			"__meta_ec2_primary_subnet_id":  "subnet-57044c3e",
+			"__meta_ec2_private_dns_name":   "ip-172-31-11-152.eu-west-2.compute.internal",
+			"__meta_ec2_private_ip":         "172.31.11.152",
+			"__meta_ec2_public_dns_name":    "ec2-3-8-232-141.eu-west-2.compute.amazonaws.com",
+			"__meta_ec2_public_ip":          "3.8.232.141",
+			"__meta_ec2_subnet_id":          ",subnet-57044c3e,",
+			"__meta_ec2_tag_foo":            "bar",
+			"__meta_ec2_vpc_id":             "vpc-f1eaad99",
+		}),
+	}
+	if !reflect.DeepEqual(sortedLabelss, expectedLabels) {
+		t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, expectedLabels)
+	}
+}
diff --git a/lib/promscrape/discovery/ec2/sign_test.go b/lib/promscrape/discovery/ec2/sign_test.go
new file mode 100644
index 000000000..6a1152012
--- /dev/null
+++ b/lib/promscrape/discovery/ec2/sign_test.go
@@ -0,0 +1,27 @@
+package ec2
+
+import (
+	"testing"
+	"time"
+)
+
+func TestNewSignedRequest(t *testing.T) {
+	f := func(apiURL string, authHeaderExpected string) {
+		t.Helper()
+		service := "ec2"
+		region := "us-east-1"
+		accessKey := "fake-access-key"
+		secretKey := "foobar"
+		ct := time.Unix(0, 0).UTC()
+		req, err := newSignedRequestWithTime(apiURL, service, region, accessKey, secretKey, ct)
+		if err != nil {
+			t.Fatalf("error in newSignedRequest: %s", err)
+		}
+		authHeader := req.Header.Get("Authorization")
+		if authHeader != authHeaderExpected {
+			t.Fatalf("unexpected auth header;\ngot\n%s\nwant\n%s", authHeader, authHeaderExpected)
+		}
+	}
+	f("https://ec2.amazonaws.com/?Action=DescribeRegions&Version=2013-10-15",
+		"AWS4-HMAC-SHA256 Credential=fake-access-key/19700101/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-date, Signature=79dc8f54719a4c11edcd5811824a071361b3514172a3f5c903b7e279dfa6a710")
+}
diff --git a/lib/storage/part_search_test.go b/lib/storage/part_search_test.go
index 11166debf..d06d2ac01 100644
--- a/lib/storage/part_search_test.go
+++ b/lib/storage/part_search_test.go
@@ -1247,11 +1247,11 @@ func testPartSearch(t *testing.T, p *part, tsids []TSID, tr TimeRange, expectedR
 
 func testPartSearchSerial(p *part, tsids []TSID, tr TimeRange, expectedRawBlocks []rawBlock) error {
 	var ps partSearch
-	ps.Init(p, tsids, tr, true)
+	ps.Init(p, tsids, tr)
 	var bs []Block
 	for ps.NextBlock() {
 		var b Block
-		b.CopyFrom(&ps.Block)
+		ps.BlockRef.MustReadBlock(&b, true)
 		bs = append(bs, b)
 	}
 	if err := ps.Error(); err != nil {
diff --git a/lib/storage/partition_search_test.go b/lib/storage/partition_search_test.go
index 7f8b515b8..fd997600e 100644
--- a/lib/storage/partition_search_test.go
+++ b/lib/storage/partition_search_test.go
@@ -240,10 +240,10 @@ func testPartitionSearchSerial(pt *partition, tsids []TSID, tr TimeRange, rbsExp
 
 	bs := []Block{}
 	var pts partitionSearch
-	pts.Init(pt, tsids, tr, true)
+	pts.Init(pt, tsids, tr)
 	for pts.NextBlock() {
 		var b Block
-		b.CopyFrom(pts.Block)
+		pts.BlockRef.MustReadBlock(&b, true)
 		bs = append(bs, b)
 	}
 	if err := pts.Error(); err != nil {
@@ -265,18 +265,9 @@ func testPartitionSearchSerial(pt *partition, tsids []TSID, tr TimeRange, rbsExp
 	}
 
 	// verify that empty tsids returns empty result
-	pts.Init(pt, []TSID{}, tr, true)
+	pts.Init(pt, []TSID{}, tr)
 	if pts.NextBlock() {
-		return fmt.Errorf("unexpected block got for an empty tsids list: %+v", pts.Block)
-	}
-	if err := pts.Error(); err != nil {
-		return fmt.Errorf("unexpected error on empty tsids list: %s", err)
-	}
-	pts.MustClose()
-
-	pts.Init(pt, []TSID{}, tr, false)
-	if pts.NextBlock() {
-		return fmt.Errorf("unexpected block got for an empty tsids list: %+v", pts.Block)
+		return fmt.Errorf("unexpected block got for an empty tsids list: %+v", pts.BlockRef)
 	}
 	if err := pts.Error(); err != nil {
 		return fmt.Errorf("unexpected error on empty tsids list: %s", err)
diff --git a/lib/storage/search_test.go b/lib/storage/search_test.go
index 6500907cc..d5f6feb2e 100644
--- a/lib/storage/search_test.go
+++ b/lib/storage/search_test.go
@@ -208,15 +208,20 @@ func testSearchInternal(st *Storage, tr TimeRange, mrs []MetricRow, accountsCoun
 			expectedMrs = append(expectedMrs, *mr)
 		}
 
+		type metricBlock struct {
+			MetricName []byte
+			Block      *Block
+		}
+
 		// Search
-		s.Init(st, []*TagFilters{tfs}, tr, true, 1e5)
-		var mbs []MetricBlock
+		s.Init(st, []*TagFilters{tfs}, tr, 1e5)
+		var mbs []metricBlock
 		for s.NextMetricBlock() {
 			var b Block
-			b.CopyFrom(s.MetricBlock.Block)
+			s.MetricBlockRef.BlockRef.MustReadBlock(&b, true)
 
-			var mb MetricBlock
-			mb.MetricName = append(mb.MetricName, s.MetricBlock.MetricName...)
+			var mb metricBlock
+			mb.MetricName = append(mb.MetricName, s.MetricBlockRef.MetricName...)
 			mb.Block = &b
 			mbs = append(mbs, mb)
 		}
diff --git a/lib/storage/storage_test.go b/lib/storage/storage_test.go
index 75ecacb23..8cd44faa1 100644
--- a/lib/storage/storage_test.go
+++ b/lib/storage/storage_test.go
@@ -583,24 +583,13 @@ func testStorageDeleteMetrics(s *Storage, workerNum int) error {
 		MaxTimestamp: 2e10,
 	}
 	metricBlocksCount := func(tfs *TagFilters) int {
-		// Verify the number of blocks with fetchData=true
+		// Verify the number of blocks
 		n := 0
-		sr.Init(s, []*TagFilters{tfs}, tr, true, 1e5)
+		sr.Init(s, []*TagFilters{tfs}, tr, 1e5)
 		for sr.NextMetricBlock() {
 			n++
 		}
 		sr.MustClose()
-
-		// Make sure the number of blocks with fetchData=false is the same.
-		m := 0
-		sr.Init(s, []*TagFilters{tfs}, tr, false, 1e5)
-		for sr.NextMetricBlock() {
-			m++
-		}
-		sr.MustClose()
-		if n != m {
-			return -1
-		}
 		return n
 	}
 	for i := 0; i < metricsCount; i++ {
diff --git a/lib/storage/table_search_test.go b/lib/storage/table_search_test.go
index 6c75a1a08..c821b5e93 100644
--- a/lib/storage/table_search_test.go
+++ b/lib/storage/table_search_test.go
@@ -251,10 +251,10 @@ func testTableSearchSerial(tb *table, tsids []TSID, tr TimeRange, rbsExpected []
 
 	bs := []Block{}
 	var ts tableSearch
-	ts.Init(tb, tsids, tr, true)
+	ts.Init(tb, tsids, tr)
 	for ts.NextBlock() {
 		var b Block
-		b.CopyFrom(ts.Block)
+		ts.BlockRef.MustReadBlock(&b, true)
 		bs = append(bs, b)
 	}
 	if err := ts.Error(); err != nil {
@@ -276,23 +276,14 @@ func testTableSearchSerial(tb *table, tsids []TSID, tr TimeRange, rbsExpected []
 	}
 
 	// verify that empty tsids returns empty result
-	ts.Init(tb, []TSID{}, tr, true)
+	ts.Init(tb, []TSID{}, tr)
 	if ts.NextBlock() {
-		return fmt.Errorf("unexpected block got for an empty tsids list: %+v", ts.Block)
+		return fmt.Errorf("unexpected block got for an empty tsids list: %+v", ts.BlockRef)
 	}
 	if err := ts.Error(); err != nil {
 		return fmt.Errorf("unexpected error on empty tsids list: %s", err)
 	}
 	ts.MustClose()
 
-	ts.Init(tb, []TSID{}, tr, false)
-	if ts.NextBlock() {
-		return fmt.Errorf("unexpected block got for an empty tsids list with fetchData=false: %+v", ts.Block)
-	}
-	if err := ts.Error(); err != nil {
-		return fmt.Errorf("unexpected error on empty tsids list with fetchData=false: %s", err)
-	}
-	ts.MustClose()
-
 	return nil
 }
diff --git a/lib/storage/table_search_timing_test.go b/lib/storage/table_search_timing_test.go
index df9af44f4..fe4d8c8df 100644
--- a/lib/storage/table_search_timing_test.go
+++ b/lib/storage/table_search_timing_test.go
@@ -127,12 +127,14 @@ func benchmarkTableSearch(b *testing.B, rowsCount, tsidsCount, tsidsSearch int,
 	b.RunParallel(func(pb *testing.PB) {
 		var ts tableSearch
 		tsids := make([]TSID, tsidsSearch)
+		var tmpBlock Block
 		for pb.Next() {
 			for i := range tsids {
 				tsids[i].MetricID = 1 + uint64(i)
 			}
-			ts.Init(tb, tsids, tr, fetchData)
+			ts.Init(tb, tsids, tr)
 			for ts.NextBlock() {
+				ts.BlockRef.MustReadBlock(&tmpBlock, fetchData)
 			}
 			ts.MustClose()
 		}