Repository: VictoriaMetrics/VictoriaMetrics
Pull request #588 (2020-06-27 14:35:49)

Vmalert flags
The change adds no new functionality and aims to move flags definitions
to subpackages that are using them. This should improve readability
of the main function.

codecov[bot]: # [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588?src=pr&el=h1) Report
> Merging [#588](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588?src=pr&el=desc) into [master](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/commit/dc4e3f0e0b046db8bbd69bc3e5e9e48795a247ef&el=desc) will **decrease** coverage by `0.04%`.
> The diff coverage is `32.05%`.

[![Impacted file tree graph](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588/graphs/tree.svg?width=650&height=150&src=pr&token=YND8dYVcXn)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #588      +/-   ##
==========================================
- Coverage   68.04%   68.00%   -0.05%     
==========================================
  Files         151      154       +3     
  Lines       21225    21225              
==========================================
- Hits        14442    14433       -9     
- Misses       5620     5627       +7     
- Partials     1163     1165       +2     
```


| [Impacted Files](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [app/vmalert/datasource/init.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588/diff?src=pr&el=tree#diff-YXBwL3ZtYWxlcnQvZGF0YXNvdXJjZS9pbml0Lmdv) | `0.00% <0.00%> (ø)` | |
| [app/vmalert/main.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588/diff?src=pr&el=tree#diff-YXBwL3ZtYWxlcnQvbWFpbi5nbw==) | `25.74% <0.00%> (-8.05%)` | :arrow_down: |
| [app/vmalert/notifier/init.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588/diff?src=pr&el=tree#diff-YXBwL3ZtYWxlcnQvbm90aWZpZXIvaW5pdC5nbw==) | `0.00% <0.00%> (ø)` | |
| [app/vmalert/utils/tls.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588/diff?src=pr&el=tree#diff-YXBwL3ZtYWxlcnQvdXRpbHMvdGxzLmdv) | `80.00% <80.00%> (ø)` | |
| [app/vmalert/manager.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588/diff?src=pr&el=tree#diff-YXBwL3ZtYWxlcnQvbWFuYWdlci5nbw==) | `82.85% <100.00%> (ø)` | |
| [lib/mergeset/table.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588/diff?src=pr&el=tree#diff-bGliL21lcmdlc2V0L3RhYmxlLmdv) | `68.01% <0.00%> (-1.26%)` | :arrow_down: |
| [lib/storage/partition.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588/diff?src=pr&el=tree#diff-bGliL3N0b3JhZ2UvcGFydGl0aW9uLmdv) | `71.54% <0.00%> (ø)` | |
| [app/vmselect/promql/aggr\_incremental.go](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588/diff?src=pr&el=tree#diff-YXBwL3Ztc2VsZWN0L3Byb21xbC9hZ2dyX2luY3JlbWVudGFsLmdv) | `89.34% <0.00%> (ø)` | |
| ... and [1 more](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588/diff?src=pr&el=tree-more) | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588?src=pr&el=footer). Last update [dc4e3f0...a4ffbaf](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/pull/588?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/app/vmalert/README.md b/app/vmalert/README.md
index e6ba23cad..07aba91ee 100644
--- a/app/vmalert/README.md
+++ b/app/vmalert/README.md
@@ -214,6 +214,8 @@ Usage of vmalert:
         Optional basic auth username for -remoteWrite.url
   -remoteWrite.concurrency int
         Defines number of readers that concurrently write into remote storage (default 1)
+  -remoteWrite.flushInterval duration
+        Defines interval of flushes to remote write endpoint (default 5s)
   -remoteWrite.maxBatchSize int
         Defines defines max number of timeseries to be flushed at once (default 1000)
   -remoteWrite.maxQueueSize int
diff --git a/app/vmalert/datasource/init.go b/app/vmalert/datasource/init.go
new file mode 100644
index 000000000..3e371f5c6
--- /dev/null
+++ b/app/vmalert/datasource/init.go
@@ -0,0 +1,38 @@
+package datasource
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+var (
+	addr = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter."+
+		" E.g. http://127.0.0.1:8428")
+	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
+
+	tlsInsecureSkipVerify = flag.Bool("datasource.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -datasource.url")
+	tlsCertFile           = flag.String("datasource.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -datasource.url")
+	tlsKeyFile            = flag.String("datasource.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -datasource.url")
+	tlsCAFile             = flag.String("datasource.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -datasource.url. "+
+		"By default system CA is used")
+	tlsServerName = flag.String("datasource.tlsServerName", "", "Optional TLS server name to use for connections to -datasource.url. "+
+		"By default the server name from -datasource.url is used")
+)
+
+// Init creates a Querier from provided flag values.
+func Init() (Querier, error) {
+	if *addr == "" {
+		flag.PrintDefaults()
+		return nil, fmt.Errorf("datasource.url is empty")
+	}
+	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %s", err)
+	}
+	c := &http.Client{Transport: tr}
+	return NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, c), nil
+}
diff --git a/app/vmalert/main.go b/app/vmalert/main.go
index 74b523258..9c570747a 100644
--- a/app/vmalert/main.go
+++ b/app/vmalert/main.go
@@ -2,12 +2,8 @@ package main
 
 import (
 	"context"
-	"crypto/tls"
-	"crypto/x509"
 	"flag"
 	"fmt"
-	"io/ioutil"
-	"net/http"
 	"net/url"
 	"os"
 	"strconv"
@@ -16,6 +12,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -35,65 +32,17 @@ Examples:
  -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
 absolute path to all .yaml files in root.`)
 
+	httpListenAddr     = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
+	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
+
 	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
-
-	httpListenAddr = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
-
-	datasourceURL = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter."+
-		" E.g. http://127.0.0.1:8428")
-	basicAuthUsername               = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
-	basicAuthPassword               = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
-	datasourceTLSInsecureSkipVerify = flag.Bool("datasource.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -datasource.url")
-	datasourceTLSCertFile           = flag.String("datasource.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -datasource.url")
-	datasourceTLSKeyFile            = flag.String("datasource.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -datasource.url")
-	datasourceTLSCAFile             = flag.String("datasource.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -datasource.url. "+
-		"By default system CA is used")
-	datasourceTLSServerName = flag.String("datasource.tlsServerName", "", "Optional TLS server name to use for connections to -datasource.url. "+
-		"By default the server name from -datasource.url is used")
-
-	remoteWriteURL = flag.String("remoteWrite.url", "", "Optional URL to Victoria Metrics or VMInsert where to persist alerts state"+
-		" and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428")
-	remoteWriteUsername              = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
-	remoteWritePassword              = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
-	remoteWriteMaxQueueSize          = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
-	remoteWriteMaxBatchSize          = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines defines max number of timeseries to be flushed at once")
-	remoteWriteConcurrency           = flag.Int("remoteWrite.concurrency", 1, "Defines number of writers for concurrent writing into remote storage")
-	remoteWriteTLSInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
-	remoteWriteTLSCertFile           = flag.String("remoteWrite.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url")
-	remoteWriteTLSKeyFile            = flag.String("remoteWrite.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url")
-	remoteWriteTLSCAFile             = flag.String("remoteWrite.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
-		"By default system CA is used")
-	remoteWriteTLSServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+
-		"By default the server name from -remoteWrite.url is used")
-
-	remoteReadURL = flag.String("remoteRead.url", "", "Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts"+
-		" state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state."+
-		" E.g. http://127.0.0.1:8428")
-	remoteReadUsername = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
-	remoteReadPassword = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
-	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
-		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
-	remoteReadTLSInsecureSkipVerify = flag.Bool("remoteRead.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteRead.url")
-	remoteReadTLSCertFile           = flag.String("remoteRead.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url")
-	remoteReadTLSKeyFile            = flag.String("remoteRead.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url")
-	remoteReadTLSCAFile             = flag.String("remoteRead.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -remoteRead.url. "+
-		"By default system CA is used")
-	remoteReadTLSServerName = flag.String("remoteRead.tlsServerName", "", "Optional TLS server name to use for connections to -remoteRead.url. "+
-		"By default the server name from -remoteRead.url is used")
-
-	evaluationInterval            = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
-	notifierURL                   = flag.String("notifier.url", "", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
-	notifierTLSInsecureSkipVerify = flag.Bool("notifier.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -notifier.url")
-	notifierTLSCertFile           = flag.String("notifier.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
-	notifierTLSKeyFile            = flag.String("notifier.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
-	notifierTLSCAFile             = flag.String("notifier.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -notifier.url. "+
-		"By default system CA is used")
-	notifierTLSServerName = flag.String("notifier.tlsServerName", "", "Optional TLS server name to use for connections to -notifier.url. "+
-		"By default the server name from -notifier.url is used")
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana, Prometheus or any other service.
 eg. 'explore?orgId=1&left=[\"now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|pathEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\"]}]'.If empty '/api/v1/:groupID/alertID/status' is used`)
+
+	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
+		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
 )
 
 func main() {
@@ -103,64 +52,12 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
-	checkFlags()
-	ctx, cancel := context.WithCancel(context.Background())
-	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
-	if err != nil {
-		logger.Fatalf("can not get external url: %s ", err)
-	}
-	notifier.InitTemplateFunc(eu)
-	aug, err := getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
-	if err != nil {
-		logger.Fatalf("URL generator error: %s", err)
-	}
-
-	dst, err := getTransport(datasourceURL, datasourceTLSCertFile, datasourceTLSKeyFile, datasourceTLSCAFile, datasourceTLSServerName, datasourceTLSInsecureSkipVerify)
-	if err != nil {
-		logger.Fatalf("cannot create datasource transport: %s", err)
-	}
 
-	nt, err := getTransport(notifierURL, notifierTLSCertFile, notifierTLSKeyFile, notifierTLSCAFile, notifierTLSServerName, notifierTLSInsecureSkipVerify)
+	ctx, cancel := context.WithCancel(context.Background())
+	manager, err := newManager(ctx)
 	if err != nil {
-		logger.Fatalf("cannot create notifier transport: %s", err)
-	}
-
-	manager := &manager{
-		groups:   make(map[uint64]*Group),
-		storage:  datasource.NewVMStorage(*datasourceURL, *basicAuthUsername, *basicAuthPassword, &http.Client{Transport: dst}),
-		notifier: notifier.NewAlertManager(*notifierURL, aug, &http.Client{Transport: nt}),
-	}
-	if *remoteWriteURL != "" {
-		t, err := getTransport(remoteWriteURL, remoteWriteTLSCertFile, remoteWriteTLSKeyFile, remoteWriteTLSCAFile, remoteWriteTLSServerName, remoteWriteTLSInsecureSkipVerify)
-		if err != nil {
-			logger.Fatalf("cannot create remoteWrite transport: %s", err)
-		}
-
-		c, err := remotewrite.NewClient(ctx, remotewrite.Config{
-			Addr:          *remoteWriteURL,
-			Concurrency:   *remoteWriteConcurrency,
-			MaxQueueSize:  *remoteWriteMaxQueueSize,
-			MaxBatchSize:  *remoteWriteMaxBatchSize,
-			FlushInterval: *evaluationInterval,
-			BasicAuthUser: *remoteWriteUsername,
-			BasicAuthPass: *remoteWritePassword,
-			Transport:     t,
-		})
-		if err != nil {
-			logger.Fatalf("failed to init remotewrite client: %s", err)
-		}
-		manager.rw = c
+		logger.Fatalf("failed to init: %s", err)
 	}
-
-	if *remoteReadURL != "" {
-		t, err := getTransport(remoteReadURL, remoteReadTLSCertFile, remoteReadTLSKeyFile, remoteReadTLSCAFile, remoteReadTLSServerName, remoteReadTLSInsecureSkipVerify)
-		if err != nil {
-			logger.Fatalf("cannot create remoteRead transport: %s", err)
-		}
-
-		manager.rr = datasource.NewVMStorage(*remoteReadURL, *remoteReadUsername, *remoteReadPassword, &http.Client{Transport: t})
-	}
-
 	if err := manager.start(ctx, *rulePath, *validateTemplates, *validateExpressions); err != nil {
 		logger.Fatalf("failed to start: %s", err)
 	}
@@ -205,6 +102,44 @@ var (
 	configTimestamp    = metrics.NewCounter(`vmalert_config_last_reload_success_timestamp_seconds`)
 )
 
+func newManager(ctx context.Context) (*manager, error) {
+	q, err := datasource.Init()
+	if err != nil {
+		return nil, fmt.Errorf("failed to init datasource: %s", err)
+	}
+	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
+	if err != nil {
+		return nil, fmt.Errorf("failed to init `external.url`: %s", err)
+	}
+	notifier.InitTemplateFunc(eu)
+	aug, err := getAlertURLGenerator(eu, *externalAlertSource, *validateTemplates)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init `external.alert.source`: %s", err)
+	}
+	nt, err := notifier.Init(aug)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init notifier: %s", err)
+	}
+
+	manager := &manager{
+		groups:   make(map[uint64]*Group),
+		querier:  q,
+		notifier: nt,
+	}
+	rw, err := remotewrite.Init(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init remoteWrite: %s", err)
+	}
+	manager.rw = rw
+
+	rr, err := remoteread.Init()
+	if err != nil {
+		return nil, fmt.Errorf("failed to init remoteRead: %s", err)
+	}
+	manager.rr = rr
+	return manager, nil
+}
+
 func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL, error) {
 	if externalURL != "" {
 		return url.Parse(externalURL)
@@ -249,62 +184,6 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 	}, nil
 }
 
-func getTLSConfig(certFile, keyFile, CAFile, serverName *string, insecureSkipVerify *bool) (*tls.Config, error) {
-	var certs []tls.Certificate
-	if *certFile != "" {
-		cert, err := tls.LoadX509KeyPair(*certFile, *keyFile)
-		if err != nil {
-			return nil, fmt.Errorf("cannot load TLS certificate from `cert_file`=%q, `key_file`=%q: %s", *certFile, *keyFile, err)
-		}
-
-		certs = []tls.Certificate{cert}
-	}
-
-	var rootCAs *x509.CertPool
-	if *CAFile != "" {
-		pem, err := ioutil.ReadFile(*CAFile)
-		if err != nil {
-			return nil, fmt.Errorf("cannot read `ca_file` %q: %s", *CAFile, err)
-		}
-
-		rootCAs = x509.NewCertPool()
-		if !rootCAs.AppendCertsFromPEM(pem) {
-			return nil, fmt.Errorf("cannot parse data from `ca_file` %q", *CAFile)
-		}
-	}
-
-	return &tls.Config{
-		Certificates:       certs,
-		InsecureSkipVerify: *insecureSkipVerify,
-		RootCAs:            rootCAs,
-		ServerName:         *serverName,
-	}, nil
-}
-
-func getTransport(URL, certFile, keyFile, CAFile, serverName *string, insecureSkipVerify *bool) (*http.Transport, error) {
-	t := http.DefaultTransport.(*http.Transport).Clone()
-	if !strings.HasPrefix(*URL, "https") {
-		return t, nil
-	}
-	tlsCfg, err := getTLSConfig(certFile, keyFile, CAFile, serverName, insecureSkipVerify)
-	if err != nil {
-		return nil, err
-	}
-	t.TLSClientConfig = tlsCfg
-	return t, nil
-}
-
-func checkFlags() {
-	if *notifierURL == "" {
-		flag.PrintDefaults()
-		logger.Fatalf("notifier.url is empty")
-	}
-	if *datasourceURL == "" {
-		flag.PrintDefaults()
-		logger.Fatalf("datasource.url is empty")
-	}
-}
-
 func usage() {
 	const s = `
 vmalert processes alerts and recording rules.
diff --git a/app/vmalert/manager.go b/app/vmalert/manager.go
index ab8975189..b312cddc8 100644
--- a/app/vmalert/manager.go
+++ b/app/vmalert/manager.go
@@ -15,7 +15,7 @@ import (
 
 // manager controls group states
 type manager struct {
-	storage  datasource.Querier
+	querier  datasource.Querier
 	notifier notifier.Notifier
 
 	rw *remotewrite.Client
@@ -73,7 +73,7 @@ func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) {
 	m.wg.Add(1)
 	id := group.ID()
 	go func() {
-		group.start(ctx, m.storage, m.notifier, m.rw)
+		group.start(ctx, m.querier, m.notifier, m.rw)
 		m.wg.Done()
 	}()
 	m.groups[id] = group
diff --git a/app/vmalert/notifier/init.go b/app/vmalert/notifier/init.go
new file mode 100644
index 000000000..6871cb010
--- /dev/null
+++ b/app/vmalert/notifier/init.go
@@ -0,0 +1,33 @@
+package notifier
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+var (
+	addr                  = flag.String("notifier.url", "", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
+	tlsInsecureSkipVerify = flag.Bool("notifier.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -notifier.url")
+	tlsCertFile           = flag.String("notifier.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -notifier.url")
+	tlsKeyFile            = flag.String("notifier.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -notifier.url")
+	tlsCAFile             = flag.String("notifier.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -notifier.url. "+
+		"By default system CA is used")
+	tlsServerName = flag.String("notifier.tlsServerName", "", "Optional TLS server name to use for connections to -notifier.url. "+
+		"By default the server name from -notifier.url is used")
+)
+
+// Init creates a Notifier object based on provided flags.
+func Init(gen AlertURLGenerator) (Notifier, error) {
+	if *addr == "" {
+		flag.PrintDefaults()
+		return nil, fmt.Errorf("notifier.url is empty")
+	}
+	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %s", err)
+	}
+	return NewAlertManager(*addr, gen, &http.Client{Transport: tr}), nil
+}
diff --git a/app/vmalert/remoteread/init.go b/app/vmalert/remoteread/init.go
new file mode 100644
index 000000000..643332373
--- /dev/null
+++ b/app/vmalert/remoteread/init.go
@@ -0,0 +1,39 @@
+package remoteread
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+var (
+	addr = flag.String("remoteRead.url", "", "Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts"+
+		" state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state."+
+		" E.g. http://127.0.0.1:8428")
+	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
+	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
+	tlsInsecureSkipVerify = flag.Bool("remoteRead.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteRead.url")
+	tlsCertFile           = flag.String("remoteRead.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteRead.url")
+	tlsKeyFile            = flag.String("remoteRead.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteRead.url")
+	tlsCAFile             = flag.String("remoteRead.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -remoteRead.url. "+
+		"By default system CA is used")
+	tlsServerName = flag.String("remoteRead.tlsServerName", "", "Optional TLS server name to use for connections to -remoteRead.url. "+
+		"By default the server name from -remoteRead.url is used")
+)
+
+// Init creates a Querier from provided flag values.
+// Returns nil if addr flag wasn't set.
+func Init() (datasource.Querier, error) {
+	if *addr == "" {
+		return nil, nil
+	}
+	tr, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %s", err)
+	}
+	c := &http.Client{Transport: tr}
+	return datasource.NewVMStorage(*addr, *basicAuthUsername, *basicAuthPassword, c), nil
+}
diff --git a/app/vmalert/remotewrite/init.go b/app/vmalert/remotewrite/init.go
new file mode 100644
index 000000000..4141c163e
--- /dev/null
+++ b/app/vmalert/remotewrite/init.go
@@ -0,0 +1,54 @@
+package remotewrite
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+)
+
+var (
+	addr = flag.String("remoteWrite.url", "", "Optional URL to Victoria Metrics or VMInsert where to persist alerts state"+
+		" and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428")
+	basicAuthUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
+	basicAuthPassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
+
+	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
+	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines defines max number of timeseries to be flushed at once")
+	concurrency   = flag.Int("remoteWrite.concurrency", 1, "Defines number of writers for concurrent writing into remote querier")
+	flushInterval = flag.Duration("remoteWrite.flushInterval", 5*time.Second, "Defines interval of flushes to remote write endpoint")
+
+	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
+	tlsCertFile           = flag.String("remoteWrite.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url")
+	tlsKeyFile            = flag.String("remoteWrite.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url")
+	tlsCAFile             = flag.String("remoteWrite.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
+		"By default system CA is used")
+	tlsServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+
+		"By default the server name from -remoteWrite.url is used")
+)
+
+// Init creates Client object from given flags.
+// Returns nil if addr flag wasn't set.
+func Init(ctx context.Context) (*Client, error) {
+	if *addr == "" {
+		return nil, nil
+	}
+
+	t, err := utils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create transport: %s", err)
+	}
+
+	return NewClient(ctx, Config{
+		Addr:          *addr,
+		Concurrency:   *concurrency,
+		MaxQueueSize:  *maxQueueSize,
+		MaxBatchSize:  *maxBatchSize,
+		FlushInterval: *flushInterval,
+		BasicAuthUser: *basicAuthUsername,
+		BasicAuthPass: *basicAuthPassword,
+		Transport:     t,
+	})
+}
diff --git a/app/vmalert/utils/tls.go b/app/vmalert/utils/tls.go
new file mode 100644
index 000000000..bc7a0c807
--- /dev/null
+++ b/app/vmalert/utils/tls.go
@@ -0,0 +1,58 @@
+package utils
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// Transport creates http.Transport object based on provided URL.
+// Returns Transport with TLS configuration if URL contains `https` prefix
+func Transport(URL, certFile, keyFile, CAFile, serverName string, insecureSkipVerify bool) (*http.Transport, error) {
+	t := http.DefaultTransport.(*http.Transport).Clone()
+	if !strings.HasPrefix(URL, "https") {
+		return t, nil
+	}
+	tlsCfg, err := TLSConfig(certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err != nil {
+		return nil, err
+	}
+	t.TLSClientConfig = tlsCfg
+	return t, nil
+}
+
+// TLSConfig creates tls.Config object from provided arguments
+func TLSConfig(certFile, keyFile, CAFile, serverName string, insecureSkipVerify bool) (*tls.Config, error) {
+	var certs []tls.Certificate
+	if certFile != "" {
+		cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load TLS certificate from `cert_file`=%q, `key_file`=%q: %s", certFile, keyFile, err)
+		}
+
+		certs = []tls.Certificate{cert}
+	}
+
+	var rootCAs *x509.CertPool
+	if CAFile != "" {
+		pem, err := ioutil.ReadFile(CAFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read `ca_file` %q: %s", CAFile, err)
+		}
+
+		rootCAs = x509.NewCertPool()
+		if !rootCAs.AppendCertsFromPEM(pem) {
+			return nil, fmt.Errorf("cannot parse data from `ca_file` %q", CAFile)
+		}
+	}
+
+	return &tls.Config{
+		Certificates:       certs,
+		InsecureSkipVerify: insecureSkipVerify,
+		RootCAs:            rootCAs,
+		ServerName:         serverName,
+	}, nil
+}


Test patch:
diff --git a/app/vmalert/main_test.go b/app/vmalert/main_test.go
index 20fa377c8..853cdcf3f 100644
--- a/app/vmalert/main_test.go
+++ b/app/vmalert/main_test.go
@@ -51,52 +51,3 @@ func TestGetAlertURLGenerator(t *testing.T) {
 		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 }
-
-func TestGetTLSConfig(t *testing.T) {
-	var certFile, keyFile, CAFile, serverName string
-	var insecureSkipVerify bool
-	serverName = "test"
-	insecureSkipVerify = true
-	tlsCfg, err := getTLSConfig(&certFile, &keyFile, &CAFile, &serverName, &insecureSkipVerify)
-	if err != nil {
-		t.Errorf("unexpected error %s", err)
-	}
-	if tlsCfg == nil {
-		t.Errorf("expected tlsConfig to be set, got nil")
-	}
-	if tlsCfg.ServerName != serverName {
-		t.Errorf("unexpected ServerName, want %s, got %s", serverName, tlsCfg.ServerName)
-	}
-	if tlsCfg.InsecureSkipVerify != insecureSkipVerify {
-		t.Errorf("unexpected InsecureSkipVerify, want %v, got %v", insecureSkipVerify, tlsCfg.InsecureSkipVerify)
-	}
-	certFile = "/path/to/nonexisting/cert/file"
-	_, err = getTLSConfig(&certFile, &keyFile, &CAFile, &serverName, &insecureSkipVerify)
-	if err == nil {
-		t.Errorf("expected keypair error, got nil")
-	}
-	certFile = ""
-	CAFile = "/path/to/nonexisting/cert/file"
-	_, err = getTLSConfig(&certFile, &keyFile, &CAFile, &serverName, &insecureSkipVerify)
-	if err == nil {
-		t.Errorf("expected read error, got nil")
-	}
-}
-
-func TestGetTransport(t *testing.T) {
-	var certFile, keyFile, CAFile, serverName string
-	var insecureSkipVerify bool
-	URL := "http://victoriametrics.com"
-	_, err := getTransport(&URL, &certFile, &keyFile, &CAFile, &serverName, &insecureSkipVerify)
-	if err != nil {
-		t.Errorf("unexpected error %s", err)
-	}
-	URL = "https://victoriametrics.com"
-	tr, err := getTransport(&URL, &certFile, &keyFile, &CAFile, &serverName, &insecureSkipVerify)
-	if err != nil {
-		t.Errorf("unexpected error %s", err)
-	}
-	if tr.TLSClientConfig == nil {
-		t.Errorf("expected TLSClientConfig to be set, got nil")
-	}
-}
diff --git a/app/vmalert/manager_test.go b/app/vmalert/manager_test.go
index 6eb0ad8e1..11fcc6d23 100644
--- a/app/vmalert/manager_test.go
+++ b/app/vmalert/manager_test.go
@@ -38,7 +38,7 @@ func TestManagerUpdateError(t *testing.T) {
 func TestManagerUpdateConcurrent(t *testing.T) {
 	m := &manager{
 		groups:   make(map[uint64]*Group),
-		storage:  &fakeQuerier{},
+		querier:  &fakeQuerier{},
 		notifier: &fakeNotifier{},
 	}
 	paths := []string{
@@ -184,7 +184,7 @@ func TestManagerUpdate(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
-			m := &manager{groups: make(map[uint64]*Group), storage: &fakeQuerier{}}
+			m := &manager{groups: make(map[uint64]*Group), querier: &fakeQuerier{}}
 			path := []string{tc.initPath}
 			if err := m.update(ctx, path, true, true, false); err != nil {
 				t.Fatalf("failed to complete initial rules update: %s", err)
diff --git a/app/vmalert/utils/tls_test.go b/app/vmalert/utils/tls_test.go
new file mode 100644
index 000000000..139a85f11
--- /dev/null
+++ b/app/vmalert/utils/tls_test.go
@@ -0,0 +1,52 @@
+package utils
+
+import "testing"
+
+func TestTLSConfig(t *testing.T) {
+	var certFile, keyFile, CAFile, serverName string
+	var insecureSkipVerify bool
+	serverName = "test"
+	insecureSkipVerify = true
+	tlsCfg, err := TLSConfig(certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if tlsCfg == nil {
+		t.Errorf("expected tlsConfig to be set, got nil")
+	}
+	if tlsCfg.ServerName != serverName {
+		t.Errorf("unexpected ServerName, want %s, got %s", serverName, tlsCfg.ServerName)
+	}
+	if tlsCfg.InsecureSkipVerify != insecureSkipVerify {
+		t.Errorf("unexpected InsecureSkipVerify, want %v, got %v", insecureSkipVerify, tlsCfg.InsecureSkipVerify)
+	}
+	certFile = "/path/to/nonexisting/cert/file"
+	_, err = TLSConfig(certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err == nil {
+		t.Errorf("expected keypair error, got nil")
+	}
+	certFile = ""
+	CAFile = "/path/to/nonexisting/cert/file"
+	_, err = TLSConfig(certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err == nil {
+		t.Errorf("expected read error, got nil")
+	}
+}
+
+func TestTransport(t *testing.T) {
+	var certFile, keyFile, CAFile, serverName string
+	var insecureSkipVerify bool
+	URL := "http://victoriametrics.com"
+	_, err := Transport(URL, certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	URL = "https://victoriametrics.com"
+	tr, err := Transport(URL, certFile, keyFile, CAFile, serverName, insecureSkipVerify)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if tr.TLSClientConfig == nil {
+		t.Errorf("expected TLSClientConfig to be set, got nil")
+	}
+}