Repository: cortexproject/cortex
Pull request #2031 (2020-01-23 18:04:00)

tsdb/querier: add tracing to block stores
**What this PR does**: Adds trace points to the user stores when running with the experimental TSDBs.

**Checklist**
- [ ] Tests updated
- [ ] Documentation added
- [ ] `CHANGELOG.md` updated - the order of entries should be `[CHANGE]`, `[FEATURE]`, `[ENHANCEMENT]`, `[BUGFIX]`



Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a70e66066f..09af247e7b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,63 +2,74 @@
 
 ## master / unreleased
 
+* [CHANGE] Removed remaining support for using denormalised tokens in the ring. If you're still running ingesters with denormalised tokens (Cortex 0.4 or earlier, with `-ingester.normalise-tokens=false`), such ingesters will now be completely invisible to distributors and need to be either switched to Cortex 0.6.0 or later, or be configured to use normalised tokens. #2034
+* [CHANGE] Moved `--store.min-chunk-age` to the Querier config as `--querier.query-store-after`, allowing the store to be skipped during query time if the metrics wouldn't be found. The YAML config option `ingestermaxquerylookback` has been renamed to `query_ingesters_within` to match its CLI flag. #1893 
+  * `--store.min-chunk-age` has been removed
+  * `--querier.query-store-after` has been added in it's place.
+* [ENHANCEMENT] Experimental TSDB: Export TSDB Syncer metrics from Compactor component, they are prefixed with `cortex_compactor_`. #2023
+* [ENHANCEMENT] Experimental TSDB: Added dedicated flag `-experimental.tsdb.bucket-store.tenant-sync-concurrency` to configure the maximum number of concurrent tenants for which blocks are synched. #2026
+* [ENHANCEMENT] Experimental TSDB: Expose metrics for objstore operations (prefixed with `cortex_<component>_thanos_objstore_`, component being one of `ingester`, `querier` and `compactor`). #2027
+
+Cortex 0.4.0 is the last version that can *write* denormalised tokens. Cortex 0.5.0 and above always write normalised tokens.
+
+Cortex 0.6.0 is the last version that can *read* denormalised tokens. Starting with Cortex 0.7.0 only normalised tokens are supported, and ingesters writing denormalised tokens to the ring (running Cortex 0.4.0 or earlier with `-ingester.normalise-tokens=false`) are ignored by distributors. Such ingesters should either switch to using normalised tokens, or be upgraded to Cortex 0.5.0 or later.
+
+## 0.6.0 / 2020-01-28
+
 Note that the ruler flags need to be changed in this upgrade. You're moving from a single node ruler to something that might need to be sharded.
-If you are running with a high `-ruler.num-workers` and if you're not able to execute all your rules in `-ruler.evaluation-interval`, then you'll need to shard.
-Further, if you're using the configs service, we've upgraded the migration library and this requires some manual intervention. See full
-instructions below to upgrade your Postgres.
+Further, if you're using the configs service, we've upgraded the migration library and this requires some manual intervention. See full instructions below to upgrade your PostgreSQL.
 
-* [CHANGE] Remove unnecessary configs/flags from the ruler ring config to align with the pattern used in the distributor ring. #1987
-    * Ruler ring related flags are now all prefixed with `ruler.ring.` as opposed to just `ruler.`
-    * Changed the default value for `-ruler.ring.prefix` from `collectors/` to `rulers/` in order to not clash with other keys (ie. ring) stored in the same key-value store.
 * [CHANGE] The frontend component now does not cache results if it finds a `Cache-Control` header and if one of its values is `no-store`. #1974
 * [CHANGE] Flags changed with transition to upstream Prometheus rules manager:
-  * `ruler.client-timeout` is now `ruler.configs.client-timeout` in order to match `ruler.configs.url`
-  * `ruler.group-timeout`has been removed
-  * `ruler.num-workers` has been removed
-  * `ruler.rule-path` has been added to specify where the prometheus rule manager will sync rule files
-  * `ruler.storage.type` has beem added to specify the rule store backend type, currently only the configdb.
-  * `ruler.poll-interval` has been added to specify the interval in which to poll new rule groups.
+  * `-ruler.client-timeout` is now `ruler.configs.client-timeout` in order to match `ruler.configs.url`.
+  * `-ruler.group-timeout`has been removed.
+  * `-ruler.num-workers` has been removed.
+  * `-ruler.rule-path` has been added to specify where the prometheus rule manager will sync rule files.
+  * `-ruler.storage.type` has beem added to specify the rule store backend type, currently only the configdb.
+  * `-ruler.poll-interval` has been added to specify the interval in which to poll new rule groups.
+  * `-ruler.evaluation-interval` default value has changed from `15s` to `1m` to match the default evaluation interval in Prometheus.
+  * Ruler sharding requires a ring which can be configured via the ring flags prefixed by `ruler.ring.`. #1987
 * [CHANGE] Use relative links from /ring page to make it work when used behind reverse proxy. #1896
 * [CHANGE] Deprecated `-distributor.limiter-reload-period` flag. #1766
 * [CHANGE] Ingesters now write only normalised tokens to the ring, although they can still read denormalised tokens used by other ingesters. `-ingester.normalise-tokens` is now deprecated, and ignored. If you want to switch back to using denormalised tokens, you need to downgrade to Cortex 0.4.0. Previous versions don't handle claiming tokens from normalised ingesters correctly. #1809
 * [CHANGE] Overrides mechanism has been renamed to "runtime config", and is now separate from limits. Runtime config is simply a file that is reloaded by Cortex every couple of seconds. Limits and now also multi KV use this mechanism.<br />New arguments were introduced: `-runtime-config.file` (defaults to empty) and `-runtime-config.reload-period` (defaults to 10 seconds), which replace previously used `-limits.per-user-override-config` and `-limits.per-user-override-period` options. Old options are still used if `-runtime-config.file` is not specified. This change is also reflected in YAML configuration, where old `limits.per_tenant_override_config` and `limits.per_tenant_override_period` fields are replaced with `runtime_config.file` and `runtime_config.period` respectively. #1749
 * [CHANGE] Cortex now rejects data with duplicate labels. Previously, such data was accepted, with duplicate labels removed with only one value left. #1964
 * [CHANGE] Changed the default value for `-distributor.ha-tracker.prefix` from `collectors/` to `ha-tracker/` in order to not clash with other keys (ie. ring) stored in the same key-value store. #1940
+* [FEATURE] Write-Ahead-Log added in ingesters for more data reliability against ingester crashes. #1103
+  * `--ingester.wal-enabled`: Setting this to `true` enables writing to WAL during ingestion.
+  * `--ingester.wal-dir`: Directory where the WAL data should be stored and/or recovered from.
+  * `--ingester.checkpoint-enabled`: Set this to `true` to enable checkpointing of in-memory chunks to disk.
+  * `--ingester.checkpoint-duration`: This is the interval at which checkpoints should be created.
+  * `--ingester.recover-from-wal`: Set this to `true` to recover data from an existing WAL.
+  * For more information, please checkout the ["Ingesters with WAL" guide](https://cortexmetrics.io/docs/guides/ingesters-with-wal/).
 * [FEATURE] The distributor can now drop labels from samples (similar to the removal of the replica label for HA ingestion) per user via the `distributor.drop-label` flag. #1726
 * [FEATURE] Added flag `debug.mutex-profile-fraction` to enable mutex profiling #1969
 * [FEATURE] Added `global` ingestion rate limiter strategy. Deprecated `-distributor.limiter-reload-period` flag. #1766
 * [FEATURE] Added support for Microsoft Azure blob storage to be used for storing chunk data. #1913
 * [FEATURE] Added readiness probe endpoint`/ready` to queriers. #1934
-* [FEATURE] EXPERIMENTAL: Added `/series` API endpoint support with TSDB blocks storage. #1830
 * [FEATURE] Added "multi" KV store that can interact with two other KV stores, primary one for all reads and writes, and secondary one, which only receives writes. Primary/secondary store can be modified in runtime via runtime-config mechanism (previously "overrides"). #1749
-* [FEATURE] EXPERIMENTAL: Added TSDB blocks `compactor` component, which iterates over users blocks stored in the bucket and compact them according to the configured block ranges. #1942
+* [FEATURE] Added support to store ring tokens to a file and read it back on startup, instead of generating/fetching the tokens to/from the ring. This feature can be enabled with the flag `-ingester.tokens-file-path`. #1750
+* [FEATURE] Experimental TSDB: Added `/series` API endpoint support with TSDB blocks storage. #1830
+* [FEATURE] Experimental TSDB: Added TSDB blocks `compactor` component, which iterates over users blocks stored in the bucket and compact them according to the configured block ranges. #1942
 * [ENHANCEMENT] metric `cortex_ingester_flush_reasons` gets a new `reason` value: `Spread`, when `-ingester.spread-flushes` option is enabled. #1978
 * [ENHANCEMENT] Added `password` and `enable_tls` options to redis cache configuration. Enables usage of Microsoft Azure Cache for Redis service. #1923
-* [ENHANCEMENT] Experimental TSDB: Open existing TSDB on startup to prevent ingester from becoming ready before it can accept writes. #1917
-  * `--experimental.tsdb.max-tsdb-opening-concurrency-on-startup`
-* [ENHANCEMENT] Experimental TSDB: Added `cortex_ingester_shipper_dir_syncs_total`, `cortex_ingester_shipper_dir_sync_failures_total`, `cortex_ingester_shipper_uploads_total` and `cortex_ingester_shipper_upload_failures_total` metrics from TSDB shipper component. #1983
+* [ENHANCEMENT] Upgraded Kubernetes API version for deployments from `extensions/v1beta1` to `apps/v1`. #1941
+* [ENHANCEMENT] Experimental TSDB: Open existing TSDB on startup to prevent ingester from becoming ready before it can accept writes. The max concurrency is set via `--experimental.tsdb.max-tsdb-opening-concurrency-on-startup`. #1917
 * [ENHANCEMENT] Experimental TSDB: Querier now exports aggregate metrics from Thanos bucket store and in memory index cache (many metrics to list, but all have `cortex_querier_bucket_store_` or `cortex_querier_blocks_index_cache_` prefix). #1996 
 * [ENHANCEMENT] Experimental TSDB: Improved multi-tenant bucket store. #1991
   * Allowed to configure the blocks sync interval via `-experimental.tsdb.bucket-store.sync-interval` (0 disables the sync)
   * Limited the number of tenants concurrently synched by `-experimental.tsdb.bucket-store.block-sync-concurrency`
   * Renamed `cortex_querier_sync_seconds` metric to `cortex_querier_blocks_sync_seconds`
   * Track `cortex_querier_blocks_sync_seconds` metric for the initial sync too
-  * Fixed race condition
-* [ENHANCEMENT] Experimental TSDB: Added dedicated flag `-experimental.tsdb.bucket-store.tenant-sync-concurrency` to configure the maximum number of concurrent tenants for which blocks are synched. #2026
 * [BUGFIX] Fixed unnecessary CAS operations done by the HA tracker when the jitter is enabled. #1861
-* [BUGFIX] Fixed #1904 ingesters getting stuck in a LEAVING state after coming up from an ungraceful exit. #1921
+* [BUGFIX] Fixed ingesters getting stuck in a LEAVING state after coming up from an ungraceful exit. #1921
 * [BUGFIX] Reduce memory usage when ingester Push() errors. #1922
-* [BUGFIX] TSDB: Fixed handling of out of order/bound samples in ingesters with the experimental TSDB blocks storage. #1864
-* [BUGFIX] TSDB: Fixed querying ingesters in `LEAVING` state with the experimental TSDB blocks storage. #1854
-* [BUGFIX] TSDB: Fixed error handling in the series to chunks conversion with the experimental TSDB blocks storage. #1837
-* [BUGFIX] TSDB: Fixed TSDB creation conflict with blocks transfer in a `JOINING` ingester with the experimental TSDB blocks storage. #1818
-* [BUGFIX] TSDB: `experimental.tsdb.ship-interval` of <=0 treated as disabled instead of allowing panic. #1975
-* [BUGFIX] TSDB: Fixed `cortex_ingester_queried_samples` and `cortex_ingester_queried_series` metrics when using block storage. #1981
-* [BUGFIX] TSDB: Fixed `cortex_ingester_memory_series` and `cortex_ingester_memory_users` metrics when using with the experimental TSDB blocks storage. #1982
-* [BUGFIX] TSDB: Fixed `cortex_ingester_memory_series_created_total` and `cortex_ingester_memory_series_removed_total` metrics when using TSDB blocks storage. #1990
 * [BUGFIX] Table Manager: Fixed calculation of expected tables and creation of tables from next active schema considering grace period. #1976
+* [BUGFIX] Experimental TSDB: Fixed ingesters consistency during hand-over when using experimental TSDB blocks storage. #1854 #1818
+* [BUGFIX] Experimental TSDB: Fixed metrics when using experimental TSDB blocks storage. #1981 #1982 #1990 #1983
+* [BUGFIX] Experimental memberlist: Use the advertised address when sending packets to other peers of the Gossip memberlist. #1857
 
-### Upgrading Postgres (if you're using configs service)
+### Upgrading PostgreSQL (if you're using configs service)
 
 Reference: https://github.com/golang-migrate/migrate/tree/master/database/postgres#upgrading-from-v1
 
@@ -70,6 +81,11 @@ Reference: https://github.com/golang-migrate/migrate/tree/master/database/postgr
 migrate  -path <absolute_path_to_cortex>/cmd/cortex/migrations -database postgres://localhost:5432/database force 2 
 ```
 
+### Known issues
+
+- The `cortex_prometheus_rule_group_last_evaluation_timestamp_seconds` metric, tracked by the ruler, is not unregistered for rule groups not being used anymore. This issue will be fixed in the next Cortex release (see [2033](https://github.com/cortexproject/cortex/issues/2033)).
+
+
 ## 0.4.0 / 2019-12-02
 
 * [CHANGE] The frontend component has been refactored to be easier to re-use. When upgrading the frontend, cache entries will be discarded and re-created with the new protobuf schema. #1734
diff --git a/Makefile b/Makefile
index 64b13c235d..f026f58142 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: all test clean images protos exes
+.PHONY: all test clean images protos exes dist
 .DEFAULT_GOAL := all
 
 # Boiler plate for bulding Docker containers.
@@ -205,3 +205,12 @@ check-doc: clean-doc doc
 
 web-serve:
 	cd website && hugo --config config.toml -v server
+
+# Generate binaries for a Cortex release
+dist:
+	rm -fr ./dist
+	mkdir -p ./dist
+	GOOS="linux"  GOARCH="amd64" CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/cortex-linux-amd64   ./cmd/cortex
+	GOOS="darwin" GOARCH="amd64" CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/cortex-darwin-amd64  ./cmd/cortex
+	shasum -a 256 ./dist/cortex-darwin-amd64 | cut -d ' ' -f 1 > ./dist/cortex-darwin-amd64-sha-256
+	shasum -a 256 ./dist/cortex-linux-amd64  | cut -d ' ' -f 1 > ./dist/cortex-linux-amd64-sha-256
diff --git a/RELEASE.md b/RELEASE.md
index 7d679d52e0..3803a70a0c 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -12,8 +12,8 @@ Our goal is to provide a new minor release every 4 weeks. This is a new process
 | v0.2.0         | 2019-08-28                                 | Goutham Veeramachaneni (Github: @gouthamve) |
 | v0.3.0         | 2019-10-09                                 | Bryan Boreham (@bboreham)                   |
 | v0.4.0         | 2019-11-13                                 | Tom Wilkie (@tomwilkie)                     |
-| v0.5.0         | 2020-01-08                                 | _Abandoned_                                   |
-| v0.6.0         | 2020-01-20                                 | **searching for a volunteer**                   |
+| v0.5.0         | 2020-01-08                                 | _Abandoned_                                 |
+| v0.6.0         | 2020-01-22                                 | Marco Pracucci (@pracucci)                  |
 
 ## Release shepherd responsibilities
 
@@ -59,6 +59,8 @@ Entries in the `CHANGELOG.md` are meant to be in this order:
 * `[ENHANCEMENT]`
 * `[BUGFIX]`
 
+To quickly look for the list of PR missing a reference in the `CHANGELOG.md` you can run `./tools/release/check-changelog.sh LAST-RELEASE-TAG...master`.
+
 ### Draft the new release
 
 Tag the new release with a tag named `v<major>.<minor>.<patch>`, e.g. `v0.1.3`. Note the `v` prefix.
@@ -74,3 +76,5 @@ $ git push --tags
 Signing a tag with a GPG key is appreciated, but in case you can't add a GPG key to your Github account using the following [procedure](https://help.github.com/articles/generating-a-gpg-key/), you can replace the `-s` flag by `-a` flag of the `git tag` command to only annotate the tag without signing.
 
 Once a tag is created, the release process through CircleCI will be triggered for this tag. If everything goes smoothly, create a release in the GitHub UI with the changelog for this release.
+
+Finally run `make dist` to build binaries into `./dist` and attach them to the release on GitHub.
diff --git a/VERSION b/VERSION
index 1d0ba9ea18..a918a2aa18 100644
--- a/VERSION
+++ b/VERSION
@@ -1,1 +1,1 @@
-0.4.0
+0.6.0
diff --git a/docs/configuration/arguments.md b/docs/configuration/arguments.md
index c6d236d448..5ac54b6e8c 100644
--- a/docs/configuration/arguments.md
+++ b/docs/configuration/arguments.md
@@ -273,9 +273,9 @@ It also talks to a KVStore and has it's own copies of the same flags used by the
 
    Deprecated. New ingesters always write "normalised" tokens to the ring. Normalised tokens consume less memory to encode and decode; as the ring is unmarshalled regularly, this significantly reduces memory usage of anything that watches the ring.
 
-   Cortex 0.4.0 is the last version that can *write* denormalised tokens. Cortex 0.5.0 and later will always *write* normalised tokens, although it can still *read* denormalised tokens written by older ingesters.
+   Cortex 0.4.0 is the last version that can *write* denormalised tokens. Cortex 0.5.0 and above always write normalised tokens.
    
-   It's perfectly OK to have a mix of ingesters running denormalised (<= 0.4.0) and normalised tokens (either by using `-ingester.normalise-tokens` in Cortex <= 0.4.0, or Cortex 0.5.0+) during upgrades. 
+   Cortex 0.6.0 is the last version that can *read* denormalised tokens. Starting with Cortex 0.7.0 only normalised tokens are supported, and ingesters writing denormalised tokens to the ring (running Cortex 0.4.0 or earlier with `-ingester.normalise-tokens=false`) are ignored by distributors. Such ingesters should either switch to using normalised tokens, or be upgraded to Cortex 0.5.0 or later.
 
 - `-ingester.chunk-encoding`
 
diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index e7bc5ed76f..cb6351032c 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -508,7 +508,12 @@ The `querier_config` configures the Cortex querier.
 # Maximum lookback beyond which queries are not sent to ingester. 0 means all
 # queries are sent to ingester.
 # CLI flag: -querier.query-ingesters-within
-[ingestermaxquerylookback: <duration> | default = 0s]
+[query_ingesters_within: <duration> | default = 0s]
+
+# The time after which a metric should only be queried from storage and not just
+# ingesters. 0 means all queries are sent to store.
+# CLI flag: -querier.query-store-after
+[query_store_after: <duration> | default = 0s]
 
 # The default evaluation interval or step size for subqueries.
 # CLI flag: -querier.default-evaluation-interval
@@ -1517,10 +1522,6 @@ write_dedupe_cache_config:
   # The CLI flags prefix for this block config is: store.index-cache-write
   [fifocache: <fifo_cache_config>]
 
-# Minimum time between chunk update and being saved to the store.
-# CLI flag: -store.min-chunk-age
-[min_chunk_age: <duration> | default = 0s]
-
 # Cache index entries older than this period. 0 to disable.
 # CLI flag: -store.cache-lookups-older-than
 [cache_lookups_older_than: <duration> | default = 0s]
diff --git a/docs/guides/sharded_ruler.md b/docs/guides/sharded_ruler.md
index 70e037b80d..466a1234e0 100644
--- a/docs/guides/sharded_ruler.md
+++ b/docs/guides/sharded_ruler.md
@@ -24,3 +24,5 @@ In addition the ruler requires it's own ring to be configured, for instance:
 ```
 
 The only configuration that is required is to enable sharding and configure a key value store. From there the rulers will shard and handle the division of rules automatically.
+
+Unlike ingesters, rulers do not hand over responsibility: all rules are re-sharded randomly every time a ruler is added to or removed from the ring.
diff --git a/pkg/chunk/chunk_store.go b/pkg/chunk/chunk_store.go
index da7e6eec60..f7a80134f1 100644
--- a/pkg/chunk/chunk_store.go
+++ b/pkg/chunk/chunk_store.go
@@ -45,7 +45,6 @@ type StoreConfig struct {
 	ChunkCacheConfig       cache.Config `yaml:"chunk_cache_config,omitempty"`
 	WriteDedupeCacheConfig cache.Config `yaml:"write_dedupe_cache_config,omitempty"`
 
-	MinChunkAge           time.Duration `yaml:"min_chunk_age,omitempty"`
 	CacheLookupsOlderThan time.Duration `yaml:"cache_lookups_older_than,omitempty"`
 
 	// Limits query start time to be greater than now() - MaxLookBackPeriod, if set.
@@ -61,7 +60,6 @@ func (cfg *StoreConfig) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.chunkCacheStubs, "store.chunk-cache-stubs", false, "If true, don't write the full chunk to cache, just a stub entry.")
 	cfg.WriteDedupeCacheConfig.RegisterFlagsWithPrefix("store.index-cache-write.", "Cache config for index entry writing. ", f)
 
-	f.DurationVar(&cfg.MinChunkAge, "store.min-chunk-age", 0, "Minimum time between chunk update and being saved to the store.")
 	f.DurationVar(&cfg.CacheLookupsOlderThan, "store.cache-lookups-older-than", 0, "Cache index entries older than this period. 0 to disable.")
 	f.DurationVar(&cfg.MaxLookBackPeriod, "store.max-look-back-period", 0, "Limit how long back data can be queried")
 
@@ -269,11 +267,6 @@ func (c *store) validateQueryTimeRange(ctx context.Context, userID string, from
 		return true, nil
 	}
 
-	if from.After(now.Add(-c.cfg.MinChunkAge)) {
-		// no data relevant to this query will have arrived at the store yet
-		return true, nil
-	}
-
 	if c.cfg.MaxLookBackPeriod != 0 {
 		oldestStartTime := model.Now().Add(-c.cfg.MaxLookBackPeriod)
 		if oldestStartTime.After(*from) {
diff --git a/pkg/compactor/compactor.go b/pkg/compactor/compactor.go
index 7c87f9e099..c6de92f45e 100644
--- a/pkg/compactor/compactor.go
+++ b/pkg/compactor/compactor.go
@@ -76,6 +76,9 @@ type Compactor struct {
 	compactionRunsStarted   prometheus.Counter
 	compactionRunsCompleted prometheus.Counter
 	compactionRunsFailed    prometheus.Counter
+
+	// TSDB syncer metrics
+	syncerMetrics *syncerMetrics
 }
 
 // NewCompactor makes a new Compactor.
@@ -88,6 +91,10 @@ func NewCompactor(compactorCfg Config, storageCfg cortex_tsdb.Config, logger log
 		return nil, errors.Wrap(err, "failed to create the bucket client")
 	}
 
+	if registerer != nil {
+		bucketClient = objstore.BucketWithMetrics( /* bucket label value */ "", bucketClient, prometheus.WrapRegistererWithPrefix("cortex_compactor_", registerer))
+	}
+
 	tsdbCompactor, err := tsdb.NewLeveledCompactor(ctx, registerer, logger, compactorCfg.BlockRanges.ToMilliseconds(), downsample.NewPool())
 	if err != nil {
 		cancelCtx()
@@ -132,6 +139,7 @@ func newCompactor(
 	// Register metrics.
 	if registerer != nil {
 		registerer.MustRegister(c.compactionRunsStarted, c.compactionRunsCompleted, c.compactionRunsFailed)
+		c.syncerMetrics = newSyncerMetrics(registerer)
 	}
 
 	// Start the compactor loop.
@@ -220,9 +228,12 @@ func (c *Compactor) compactUsers(ctx context.Context) bool {
 func (c *Compactor) compactUser(ctx context.Context, userID string) error {
 	bucket := cortex_tsdb.NewUserBucketClient(userID, c.bucketClient)
 
+	reg := prometheus.NewRegistry()
+	defer c.syncerMetrics.gatherThanosSyncerMetrics(reg)
+
 	syncer, err := compact.NewSyncer(
 		c.logger,
-		nil, // TODO(pracucci) we should pass the prometheus registerer, but we would need to inject the user label to each metric, otherwise we have clashing metrics
+		reg,
 		bucket,
 		c.compactorCfg.ConsistencyDelay,
 		c.compactorCfg.BlockSyncConcurrency,
diff --git a/pkg/compactor/syncer_metrics.go b/pkg/compactor/syncer_metrics.go
new file mode 100644
index 0000000000..dc6a287b94
--- /dev/null
+++ b/pkg/compactor/syncer_metrics.go
@@ -0,0 +1,132 @@
+package compactor
+
+import (
+	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/go-kit/kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Copied from Thanos, pkg/compact/compact.go.
+// Here we aggregate metrics from all finished syncers.
+type syncerMetrics struct {
+	syncMetas                 prometheus.Counter
+	syncMetaFailures          prometheus.Counter
+	syncMetaDuration          *util.HistogramDataCollector // was prometheus.Histogram before
+	garbageCollectedBlocks    prometheus.Counter
+	garbageCollections        prometheus.Counter
+	garbageCollectionFailures prometheus.Counter
+	garbageCollectionDuration *util.HistogramDataCollector // was prometheus.Histogram before
+	compactions               prometheus.Counter
+	compactionRunsStarted     prometheus.Counter
+	compactionRunsCompleted   prometheus.Counter
+	compactionFailures        prometheus.Counter
+	verticalCompactions       prometheus.Counter
+}
+
+// Copied (and modified with Cortex prefix) from Thanos, pkg/compact/compact.go
+// We also ignore "group" label, since we only use a single group.
+func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics {
+	var m syncerMetrics
+
+	m.syncMetas = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "cortex_compactor_sync_meta_total",
+		Help: "TSDB Syncer: Total number of sync meta operations.",
+	})
+	m.syncMetaFailures = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "cortex_compactor_sync_meta_failures_total",
+		Help: "TSDB Syncer: Total number of failed sync meta operations.",
+	})
+	m.syncMetaDuration = util.NewHistogramDataCollector(prometheus.NewDesc(
+		"cortex_compactor_sync_meta_duration_seconds",
+		"TSDB Syncer: Time it took to sync meta files.",
+		nil, nil))
+
+	m.garbageCollectedBlocks = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "cortex_compactor_garbage_collected_blocks_total",
+		Help: "TSDB Syncer: Total number of deleted blocks by compactor.",
+	})
+	m.garbageCollections = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "cortex_compactor_garbage_collection_total",
+		Help: "TSDB Syncer: Total number of garbage collection operations.",
+	})
+	m.garbageCollectionFailures = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "cortex_compactor_garbage_collection_failures_total",
+		Help: "TSDB Syncer: Total number of failed garbage collection operations.",
+	})
+	m.garbageCollectionDuration = util.NewHistogramDataCollector(prometheus.NewDesc(
+		"cortex_compactor_garbage_collection_duration_seconds",
+		"TSDB Syncer: Time it took to perform garbage collection iteration.",
+		nil, nil))
+
+	m.compactions = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "cortex_compactor_group_compactions_total",
+		Help: "TSDB Syncer: Total number of group compaction attempts that resulted in a new block.",
+	})
+	m.compactionRunsStarted = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "cortex_compactor_group_compaction_runs_started_total",
+		Help: "TSDB Syncer: Total number of group compaction attempts.",
+	})
+	m.compactionRunsCompleted = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "cortex_compactor_group_compaction_runs_completed_total",
+		Help: "TSDB Syncer: Total number of group completed compaction runs. This also includes compactor group runs that resulted with no compaction.",
+	})
+	m.compactionFailures = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "cortex_compactor_group_compactions_failures_total",
+		Help: "TSDB Syncer: Total number of failed group compactions.",
+	})
+	m.verticalCompactions = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "cortex_compactor_group_vertical_compactions_total",
+		Help: "TSDB Syncer: Total number of group compaction attempts that resulted in a new block based on overlapping blocks.",
+	})
+
+	if reg != nil {
+		reg.MustRegister(
+			m.syncMetas,
+			m.syncMetaFailures,
+			m.syncMetaDuration,
+			m.garbageCollectedBlocks,
+			m.garbageCollections,
+			m.garbageCollectionFailures,
+			m.garbageCollectionDuration,
+			m.compactions,
+			m.compactionRunsStarted,
+			m.compactionRunsCompleted,
+			m.compactionFailures,
+			m.verticalCompactions,
+		)
+	}
+	return &m
+}
+
+func (m *syncerMetrics) gatherThanosSyncerMetrics(reg *prometheus.Registry) {
+	if m == nil {
+		return
+	}
+
+	mf, err := reg.Gather()
+	if err != nil {
+		level.Warn(util.Logger).Log("msg", "failed to gather metrics from syncer registry after compaction", "err", err)
+		return
+	}
+
+	mfm, err := util.NewMetricFamilyMap(mf)
+	if err != nil {
+		level.Warn(util.Logger).Log("msg", "failed to gather metrics from syncer registry after compaction", "err", err)
+		return
+	}
+
+	m.syncMetas.Add(mfm.SumCounters("thanos_compact_sync_meta_total"))
+	m.syncMetaFailures.Add(mfm.SumCounters("thanos_compact_sync_meta_failures_total"))
+	m.syncMetaDuration.Add(mfm.SumHistograms("thanos_compact_sync_meta_duration_seconds"))
+	m.garbageCollectedBlocks.Add(mfm.SumCounters("thanos_compact_garbage_collected_blocks_total"))
+	m.garbageCollections.Add(mfm.SumCounters("thanos_compact_garbage_collection_total"))
+	m.garbageCollectionFailures.Add(mfm.SumCounters("thanos_compact_garbage_collection_failures_total"))
+	m.garbageCollectionDuration.Add(mfm.SumHistograms("thanos_compact_garbage_collection_duration_seconds"))
+
+	// These metrics have "group" label, but we sum them all together.
+	m.compactions.Add(mfm.SumCounters("thanos_compact_group_compactions_total"))
+	m.compactionRunsStarted.Add(mfm.SumCounters("thanos_compact_group_compaction_runs_started_total"))
+	m.compactionRunsCompleted.Add(mfm.SumCounters("thanos_compact_group_compaction_runs_completed_total"))
+	m.compactionFailures.Add(mfm.SumCounters("thanos_compact_group_compactions_failures_total"))
+	m.verticalCompactions.Add(mfm.SumCounters("thanos_compact_group_vertical_compactions_total"))
+}
diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go
index 5373cc65f8..48576fd84f 100644
--- a/pkg/cortex/cortex.go
+++ b/pkg/cortex/cortex.go
@@ -144,6 +144,9 @@ func (c *Config) Validate() error {
 	if err := c.Distributor.Validate(); err != nil {
 		return errors.Wrap(err, "invalid distributor config")
 	}
+	if err := c.Querier.Validate(); err != nil {
+		return errors.Wrap(err, "invalid querier config")
+	}
 	return nil
 }
 
diff --git a/pkg/ingester/ingester_v2.go b/pkg/ingester/ingester_v2.go
index dee61ca024..2f7697bfc1 100644
--- a/pkg/ingester/ingester_v2.go
+++ b/pkg/ingester/ingester_v2.go
@@ -69,6 +69,10 @@ func NewV2(cfg Config, clientConfig client.Config, limits *validation.Overrides,
 		return nil, errors.Wrap(err, "failed to create the bucket client")
 	}
 
+	if registerer != nil {
+		bucketClient = objstore.BucketWithMetrics( /* bucket label value */ "", bucketClient, prometheus.WrapRegistererWithPrefix("cortex_ingester_", registerer))
+	}
+
 	i := &Ingester{
 		cfg:          cfg,
 		clientConfig: clientConfig,
diff --git a/pkg/querier/block.go b/pkg/querier/block.go
index 76c5d13210..c281a03b5c 100644
--- a/pkg/querier/block.go
+++ b/pkg/querier/block.go
@@ -10,12 +10,14 @@ import (
 	"github.com/cortexproject/cortex/pkg/ingester/client"
 	"github.com/cortexproject/cortex/pkg/storage/tsdb"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/spanlogger"
 	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
+	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 	"github.com/weaveworks/common/logging"
 	"google.golang.org/grpc/metadata"
@@ -33,6 +35,10 @@ func NewBlockQuerier(cfg tsdb.Config, logLevel logging.Level, registerer prometh
 		return nil, err
 	}
 
+	if registerer != nil {
+		bucketClient = objstore.BucketWithMetrics( /* bucket label value */ "", bucketClient, prometheus.WrapRegistererWithPrefix("cortex_querier_", registerer))
+	}
+
 	us, err := NewUserStore(cfg, bucketClient, logLevel, util.Logger, registerer)
 	if err != nil {
 		return nil, err
@@ -47,6 +53,9 @@ func NewBlockQuerier(cfg tsdb.Config, logLevel logging.Level, registerer prometh
 
 // Get implements the ChunkStore interface. It makes a block query and converts the response into chunks
 func (b *BlockQuerier) Get(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) ([]chunk.Chunk, error) {
+	log, ctx := spanlogger.New(ctx, "BlockQuerier.Get")
+	defer log.Span.Finish()
+
 	client := b.us.client
 
 	// Convert matchers to LabelMatcher
@@ -95,7 +104,7 @@ func (b *BlockQuerier) Get(ctx context.Context, userID string, from, through mod
 		// Convert Thanos store series into Cortex chunks
 		convertedChunks, err := seriesToChunks(userID, resp.GetSeries())
 		if err != nil {
-			level.Error(util.Logger).Log("msg", "failed converting TSDB series to Cortex chunks", "err", err)
+			level.Error(log).Log("msg", "failed converting TSDB series to Cortex chunks", "err", err)
 			return nil, err
 		}
 
diff --git a/pkg/querier/block_store.go b/pkg/querier/block_store.go
index 7b20574d38..65dccce595 100644
--- a/pkg/querier/block_store.go
+++ b/pkg/querier/block_store.go
@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/cortexproject/cortex/pkg/storage/tsdb"
+	"github.com/cortexproject/cortex/pkg/util/spanlogger"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
@@ -232,6 +233,9 @@ func (u *UserStore) syncUserStores(ctx context.Context, f func(context.Context,
 
 // Info makes an info request to the underlying user store
 func (u *UserStore) Info(ctx context.Context, req *storepb.InfoRequest) (*storepb.InfoResponse, error) {
+	log, ctx := spanlogger.New(ctx, "UserStore.Info")
+	defer log.Span.Finish()
+
 	md, ok := metadata.FromIncomingContext(ctx)
 	if !ok {
 		return nil, fmt.Errorf("no metadata")
@@ -252,7 +256,10 @@ func (u *UserStore) Info(ctx context.Context, req *storepb.InfoRequest) (*storep
 
 // Series makes a series request to the underlying user store
 func (u *UserStore) Series(req *storepb.SeriesRequest, srv storepb.Store_SeriesServer) error {
-	md, ok := metadata.FromIncomingContext(srv.Context())
+	log, ctx := spanlogger.New(srv.Context(), "UserStore.Series")
+	defer log.Span.Finish()
+
+	md, ok := metadata.FromIncomingContext(ctx)
 	if !ok {
 		return fmt.Errorf("no metadata")
 	}
@@ -272,6 +279,9 @@ func (u *UserStore) Series(req *storepb.SeriesRequest, srv storepb.Store_SeriesS
 
 // LabelNames makes a labelnames request to the underlying user store
 func (u *UserStore) LabelNames(ctx context.Context, req *storepb.LabelNamesRequest) (*storepb.LabelNamesResponse, error) {
+	log, ctx := spanlogger.New(ctx, "UserStore.LabelNames")
+	defer log.Span.Finish()
+
 	md, ok := metadata.FromIncomingContext(ctx)
 	if !ok {
 		return nil, fmt.Errorf("no metadata")
@@ -292,6 +302,9 @@ func (u *UserStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReque
 
 // LabelValues makes a labelvalues request to the underlying user store
 func (u *UserStore) LabelValues(ctx context.Context, req *storepb.LabelValuesRequest) (*storepb.LabelValuesResponse, error) {
+	log, ctx := spanlogger.New(ctx, "UserStore.LabelValues")
+	defer log.Span.Finish()
+
 	md, ok := metadata.FromIncomingContext(ctx)
 	if !ok {
 		return nil, fmt.Errorf("no metadata")
diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go
index e277bfff0d..3a78987392 100644
--- a/pkg/querier/querier.go
+++ b/pkg/querier/querier.go
@@ -2,6 +2,7 @@ package querier
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"time"
 
@@ -19,13 +20,16 @@ import (
 
 // Config contains the configuration require to create a querier
 type Config struct {
-	MaxConcurrent            int
-	Timeout                  time.Duration
-	Iterators                bool
-	BatchIterators           bool
-	IngesterStreaming        bool
-	MaxSamples               int
-	IngesterMaxQueryLookback time.Duration
+	MaxConcurrent        int
+	Timeout              time.Duration
+	Iterators            bool
+	BatchIterators       bool
+	IngesterStreaming    bool
+	MaxSamples           int
+	QueryIngestersWithin time.Duration `yaml:"query_ingesters_within"`
+
+	// QueryStoreAfter the time after which queries should also be sent to the store and not just ingesters.
+	QueryStoreAfter time.Duration `yaml:"query_store_after"`
 
 	// The default evaluation interval for the promql engine.
 	// Needs to be configured for subqueries to work as it is the default
@@ -36,6 +40,10 @@ type Config struct {
 	metricsRegisterer prometheus.Registerer `yaml:"-"`
 }
 
+var (
+	errBadLookbackConfigs = errors.New("bad settings, query_store_after >= query_ingesters_within which can result in queries not being sent")
+)
+
 // RegisterFlags adds the flags required to config this to the given FlagSet.
 func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&cfg.MaxConcurrent, "querier.max-concurrent", 20, "The maximum number of concurrent queries.")
@@ -47,11 +55,25 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.BatchIterators, "querier.batch-iterators", false, "Use batch iterators to execute query, as opposed to fully materialising the series in memory.  Takes precedent over the -querier.iterators flag.")
 	f.BoolVar(&cfg.IngesterStreaming, "querier.ingester-streaming", false, "Use streaming RPCs to query ingester.")
 	f.IntVar(&cfg.MaxSamples, "querier.max-samples", 50e6, "Maximum number of samples a single query can load into memory.")
-	f.DurationVar(&cfg.IngesterMaxQueryLookback, "querier.query-ingesters-within", 0, "Maximum lookback beyond which queries are not sent to ingester. 0 means all queries are sent to ingester.")
+	f.DurationVar(&cfg.QueryIngestersWithin, "querier.query-ingesters-within", 0, "Maximum lookback beyond which queries are not sent to ingester. 0 means all queries are sent to ingester.")
 	f.DurationVar(&cfg.DefaultEvaluationInterval, "querier.default-evaluation-interval", time.Minute, "The default evaluation interval or step size for subqueries.")
+	f.DurationVar(&cfg.QueryStoreAfter, "querier.query-store-after", 0, "The time after which a metric should only be queried from storage and not just ingesters. 0 means all queries are sent to store.")
 	cfg.metricsRegisterer = prometheus.DefaultRegisterer
 }
 
+// Validate the config
+func (cfg *Config) Validate() error {
+
+	// Ensure the config wont create a situation where no queriers are returned.
+	if cfg.QueryIngestersWithin != 0 && cfg.QueryStoreAfter != 0 {
+		if cfg.QueryStoreAfter >= cfg.QueryIngestersWithin {
+			return errBadLookbackConfigs
+		}
+	}
+
+	return nil
+}
+
 // ChunkStore is the read-interface to the Chunk Store.  Made an interface here
 // to reduce package coupling.
 type ChunkStore interface {
@@ -70,11 +92,11 @@ func New(cfg Config, distributor Distributor, chunkStore ChunkStore) (storage.Qu
 	var queryable storage.Queryable
 	if cfg.IngesterStreaming {
 		dq := newIngesterStreamingQueryable(distributor, iteratorFunc)
-		queryable = newUnifiedChunkQueryable(dq, chunkStore, distributor, iteratorFunc, cfg.IngesterMaxQueryLookback)
+		queryable = newUnifiedChunkQueryable(dq, chunkStore, distributor, iteratorFunc, cfg)
 	} else {
 		cq := newChunkStoreQueryable(chunkStore, iteratorFunc)
 		dq := newDistributorQueryable(distributor)
-		queryable = NewQueryable(dq, cq, distributor, cfg.IngesterMaxQueryLookback)
+		queryable = NewQueryable(dq, cq, distributor, cfg)
 	}
 
 	lazyQueryable := storage.QueryableFunc(func(ctx context.Context, mint int64, maxt int64) (storage.Querier, error) {
@@ -97,23 +119,18 @@ func New(cfg Config, distributor Distributor, chunkStore ChunkStore) (storage.Qu
 }
 
 // NewQueryable creates a new Queryable for cortex.
-func NewQueryable(dq, cq storage.Queryable, distributor Distributor, ingesterMaxQueryLookback time.Duration) storage.Queryable {
+func NewQueryable(dq, cq storage.Queryable, distributor Distributor, cfg Config) storage.Queryable {
 	return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) {
-		cqr, err := cq.Querier(ctx, mint, maxt)
-		if err != nil {
-			return nil, err
-		}
-
 		q := querier{
-			queriers:    []storage.Querier{cqr},
 			distributor: distributor,
 			ctx:         ctx,
 			mint:        mint,
 			maxt:        maxt,
 		}
 
-		// Include ingester only if maxt is within ingesterMaxQueryLookback w.r.t. current time.
-		if ingesterMaxQueryLookback == 0 || maxt >= time.Now().Add(-ingesterMaxQueryLookback).UnixNano()/1e6 {
+		// Include ingester only if maxt is within queryIngestersWithin w.r.t. current time.
+		now := model.Now()
+		if cfg.QueryIngestersWithin == 0 || maxt >= int64(now.Add(-cfg.QueryIngestersWithin)) {
 			dqr, err := dq.Querier(ctx, mint, maxt)
 			if err != nil {
 				return nil, err
@@ -121,6 +138,16 @@ func NewQueryable(dq, cq storage.Queryable, distributor Distributor, ingesterMax
 			q.queriers = append(q.queriers, dqr)
 		}
 
+		// Include store only if mint is within QueryStoreAfter w.r.t current time.
+		if cfg.QueryStoreAfter == 0 || mint <= int64(now.Add(-cfg.QueryStoreAfter)) {
+			cqr, err := cq.Querier(ctx, mint, maxt)
+			if err != nil {
+				return nil, err
+			}
+
+			q.queriers = append(q.queriers, cqr)
+		}
+
 		return q, nil
 	})
 }
diff --git a/pkg/querier/unified_querier.go b/pkg/querier/unified_querier.go
index 16b12b0d33..8211f04cc2 100644
--- a/pkg/querier/unified_querier.go
+++ b/pkg/querier/unified_querier.go
@@ -2,7 +2,6 @@ package querier
 
 import (
 	"context"
-	"time"
 
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
@@ -12,10 +11,9 @@ import (
 	"github.com/cortexproject/cortex/pkg/chunk"
 )
 
-func newUnifiedChunkQueryable(ds, cs ChunkStore, distributor Distributor, chunkIteratorFunc chunkIteratorFunc, ingesterMaxQueryLookback time.Duration) storage.Queryable {
+func newUnifiedChunkQueryable(ds, cs ChunkStore, distributor Distributor, chunkIteratorFunc chunkIteratorFunc, cfg Config) storage.Queryable {
 	return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) {
 		ucq := &unifiedChunkQuerier{
-			stores: []ChunkStore{cs},
 			querier: querier{
 				ctx:         ctx,
 				mint:        mint,
@@ -30,11 +28,17 @@ func newUnifiedChunkQueryable(ds, cs ChunkStore, distributor Distributor, chunkI
 			},
 		}
 
-		// Include ingester only if maxt is within ingesterMaxQueryLookback w.r.t. current time.
-		if ingesterMaxQueryLookback == 0 || maxt >= time.Now().Add(-ingesterMaxQueryLookback).UnixNano()/1e6 {
+		now := model.Now()
+		// Include ingester only if maxt is within QueryIngestersWithin w.r.t. current time.
+		if cfg.QueryIngestersWithin == 0 || maxt >= int64(now.Add(-cfg.QueryIngestersWithin)) {
 			ucq.stores = append(ucq.stores, ds)
 		}
 
+		// Include store only if mint is within QueryStoreAfter w.r.t. current time.
+		if cfg.QueryStoreAfter == 0 || mint <= int64(now.Add(-cfg.QueryStoreAfter)) {
+			ucq.stores = append(ucq.stores, cs)
+		}
+
 		return ucq, nil
 	})
 }
diff --git a/pkg/ring/http.go b/pkg/ring/http.go
index 97b01625a1..9ca4ce2182 100644
--- a/pkg/ring/http.go
+++ b/pkg/ring/http.go
@@ -118,7 +118,7 @@ func (r *Ring) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	sort.Strings(ingesterIDs)
 
 	ingesters := []interface{}{}
-	tokens, owned := countTokens(r.ringDesc)
+	tokens, owned := countTokens(r.ringDesc, r.ringTokens)
 	for _, id := range ingesterIDs {
 		ing := r.ringDesc.Ingesters[id]
 		timestamp := time.Unix(ing.Timestamp, 0)
diff --git a/pkg/ring/model.go b/pkg/ring/model.go
index 7b36a68258..77a025513e 100644
--- a/pkg/ring/model.go
+++ b/pkg/ring/model.go
@@ -49,69 +49,31 @@ func (d *Desc) AddIngester(id, addr string, tokens []uint32, state IngesterState
 		Tokens:    tokens,
 	}
 
-	// Since this ingester is only using normalised tokens, let's delete any denormalised
-	// tokens for this ingester. There may be such tokens eg. if previous instance
-	// of the same ingester was running with denormalized tokens.
-	for ix := 0; ix < len(d.Tokens); {
-		if d.Tokens[ix].Ingester == id {
-			d.Tokens = append(d.Tokens[:ix], d.Tokens[ix+1:]...)
-		} else {
-			ix++
-		}
-	}
-
 	d.Ingesters[id] = ingester
 }
 
 // RemoveIngester removes the given ingester and all its tokens.
 func (d *Desc) RemoveIngester(id string) {
 	delete(d.Ingesters, id)
-	output := []TokenDesc{}
-	for i := 0; i < len(d.Tokens); i++ {
-		if d.Tokens[i].Ingester != id {
-			output = append(output, d.Tokens[i])
-		}
-	}
-	d.Tokens = output
 }
 
 // ClaimTokens transfers all the tokens from one ingester to another,
 // returning the claimed token.
-// This method assumes that Ring is in the correct state, 'from' ingester has no tokens anywhere,
-// and 'to' ingester uses either normalised or non-normalised tokens, but not both. Tokens list must
-// be sorted properly. If all of this is true, everything will be fine.
+// This method assumes that Ring is in the correct state, 'to' ingester has no tokens anywhere.
+// Tokens list must be sorted properly. If all of this is true, everything will be fine.
 func (d *Desc) ClaimTokens(from, to string) Tokens {
 	var result Tokens
 
-	// If the ingester we are claiming from is normalising, get its tokens then erase them from the ring.
 	if fromDesc, found := d.Ingesters[from]; found {
 		result = fromDesc.Tokens
 		fromDesc.Tokens = nil
 		d.Ingesters[from] = fromDesc
 	}
 
-	// If we are storing the tokens in a normalise form, we need to deal with
-	// the migration from denormalised by removing the tokens from the tokens
-	// list.
-	// When all ingesters are in normalised mode, d.Tokens is empty here
-	for i := 0; i < len(d.Tokens); {
-		if d.Tokens[i].Ingester == from {
-			result = append(result, d.Tokens[i].Token)
-			d.Tokens = append(d.Tokens[:i], d.Tokens[i+1:]...)
-			continue
-		}
-		i++
-	}
-
 	ing := d.Ingesters[to]
 	ing.Tokens = result
 	d.Ingesters[to] = ing
 
-	// not necessary, but makes testing simpler
-	if len(d.Tokens) == 0 {
-		d.Tokens = nil
-	}
-
 	return result
 }
 
@@ -128,7 +90,7 @@ func (d *Desc) FindIngestersByState(state IngesterState) []IngesterDesc {
 
 // Ready returns no error when all ingesters are active and healthy.
 func (d *Desc) Ready(now time.Time, heartbeatTimeout time.Duration) error {
-	numTokens := len(d.Tokens)
+	numTokens := 0
 	for id, ingester := range d.Ingesters {
 		if now.Sub(time.Unix(ingester.Timestamp, 0)) > heartbeatTimeout {
 			return fmt.Errorf("ingester %s past heartbeat timeout", id)
@@ -147,7 +109,7 @@ func (d *Desc) Ready(now time.Time, heartbeatTimeout time.Duration) error {
 // TokensFor partitions the tokens into those for the given ID, and those for others.
 func (d *Desc) TokensFor(id string) (tokens, other Tokens) {
 	takenTokens, myTokens := Tokens{}, Tokens{}
-	for _, token := range migrateRing(d) {
+	for _, token := range d.getTokens() {
 		takenTokens = append(takenTokens, token.Token)
 		if token.Ingester == id {
 			myTokens = append(myTokens, token.Token)
@@ -254,9 +216,7 @@ func (d *Desc) Merge(mergeable memberlist.Mergeable, localCAS bool) (memberlist.
 		out.Ingesters[u] = ing
 	}
 
-	// Keep ring normalized.
 	d.Ingesters = thisIngesterMap
-	d.Tokens = nil
 
 	return out, nil
 }
@@ -272,7 +232,6 @@ func (d *Desc) MergeContent() []string {
 }
 
 // buildNormalizedIngestersMap will do the following:
-// - moves all tokens from r.Tokens into individual ingesters
 // - sorts tokens and removes duplicates (only within single ingester)
 // - it doesn't modify input ring
 func buildNormalizedIngestersMap(inputRing *Desc) map[string]IngesterDesc {
@@ -286,20 +245,9 @@ func buildNormalizedIngestersMap(inputRing *Desc) map[string]IngesterDesc {
 		out[n] = ing
 	}
 
-	for _, t := range inputRing.Tokens {
-		// if ingester doesn't exist, we will add empty one (with tokens only)
-		ing := out[t.Ingester]
-
-		// don't add tokens to the LEFT ingesters. We skip such tokens.
-		if ing.State != LEFT {
-			ing.Tokens = append(ing.Tokens, t.Token)
-			out[t.Ingester] = ing
-		}
-	}
-
 	// Sort tokens, and remove duplicates
 	for name, ing := range out {
-		if ing.Tokens == nil {
+		if len(ing.Tokens) == 0 {
 			continue
 		}
 
@@ -307,17 +255,16 @@ func buildNormalizedIngestersMap(inputRing *Desc) map[string]IngesterDesc {
 			sort.Sort(Tokens(ing.Tokens))
 		}
 
-		seen := make(map[uint32]bool)
-
-		n := 0
-		for _, v := range ing.Tokens {
-			if !seen[v] {
-				seen[v] = true
-				ing.Tokens[n] = v
-				n++
+		// tokens are sorted now, we can easily remove duplicates.
+		prev := ing.Tokens[0]
+		for ix := 1; ix < len(ing.Tokens); {
+			if ing.Tokens[ix] == prev {
+				ing.Tokens = append(ing.Tokens[:ix], ing.Tokens[ix+1:]...)
+			} else {
+				prev = ing.Tokens[ix]
+				ix++
 			}
 		}
-		ing.Tokens = ing.Tokens[:n]
 
 		// write updated value back to map
 		out[name] = ing
@@ -426,3 +373,25 @@ func (d *Desc) RemoveTombstones(limit time.Time) {
 		}
 	}
 }
+
+type TokenDesc struct {
+	Token    uint32
+	Ingester string
+}
+
+// Returns sorted list of tokens with ingester names.
+func (d *Desc) getTokens() []TokenDesc {
+	numTokens := 0
+	for _, ing := range d.Ingesters {
+		numTokens += len(ing.Tokens)
+	}
+	tokens := make([]TokenDesc, 0, numTokens)
+	for key, ing := range d.Ingesters {
+		for _, token := range ing.Tokens {
+			tokens = append(tokens, TokenDesc{Token: token, Ingester: key})
+		}
+	}
+
+	sort.Sort(ByToken(tokens))
+	return tokens
+}
diff --git a/pkg/ring/ring.go b/pkg/ring/ring.go
index c6a37ecc6e..578a82cbb9 100644
--- a/pkg/ring/ring.go
+++ b/pkg/ring/ring.go
@@ -94,8 +94,9 @@ type Ring struct {
 	done     chan struct{}
 	quit     context.CancelFunc
 
-	mtx      sync.RWMutex
-	ringDesc *Desc
+	mtx        sync.RWMutex
+	ringDesc   *Desc
+	ringTokens []TokenDesc
 
 	memberOwnershipDesc *prometheus.Desc
 	numMembersDesc      *prometheus.Desc
@@ -169,41 +170,23 @@ func (r *Ring) loop(ctx context.Context) {
 		}
 
 		ringDesc := value.(*Desc)
-		ringDesc.Tokens = migrateRing(ringDesc)
+		ringTokens := ringDesc.getTokens()
+
 		r.mtx.Lock()
 		defer r.mtx.Unlock()
 		r.ringDesc = ringDesc
+		r.ringTokens = ringTokens
 		return true
 	})
 
 	r.KVClient.Stop()
 }
 
-// migrateRing will denormalise the ring's tokens if stored in normal form.
-func migrateRing(desc *Desc) []TokenDesc {
-	numTokens := len(desc.Tokens)
-	for _, ing := range desc.Ingesters {
-		numTokens += len(ing.Tokens)
-	}
-	tokens := make([]TokenDesc, len(desc.Tokens), numTokens)
-	copy(tokens, desc.Tokens)
-	for key, ing := range desc.Ingesters {
-		for _, token := range ing.Tokens {
-			tokens = append(tokens, TokenDesc{
-				Token:    token,
-				Ingester: key,
-			})
-		}
-	}
-	sort.Sort(ByToken(tokens))
-	return tokens
-}
-
 // Get returns n (or more) ingesters which form the replicas for the given key.
 func (r *Ring) Get(key uint32, op Operation, buf []IngesterDesc) (ReplicationSet, error) {
 	r.mtx.RLock()
 	defer r.mtx.RUnlock()
-	if r.ringDesc == nil || len(r.ringDesc.Tokens) == 0 {
+	if r.ringDesc == nil || len(r.ringTokens) == 0 {
 		return ReplicationSet{}, ErrEmptyRing
 	}
 
@@ -214,13 +197,13 @@ func (r *Ring) Get(key uint32, op Operation, buf []IngesterDesc) (ReplicationSet
 		start         = r.search(key)
 		iterations    = 0
 	)
-	for i := start; len(distinctHosts) < n && iterations < len(r.ringDesc.Tokens); i++ {
+	for i := start; len(distinctHosts) < n && iterations < len(r.ringTokens); i++ {
 		iterations++
 		// Wrap i around in the ring.
-		i %= len(r.ringDesc.Tokens)
+		i %= len(r.ringTokens)
 
 		// We want n *distinct* ingesters.
-		token := r.ringDesc.Tokens[i]
+		token := r.ringTokens[i]
 		if _, ok := distinctHosts[token.Ingester]; ok {
 			continue
 		}
@@ -258,7 +241,7 @@ func (r *Ring) GetAll() (ReplicationSet, error) {
 	r.mtx.RLock()
 	defer r.mtx.RUnlock()
 
-	if r.ringDesc == nil || len(r.ringDesc.Tokens) == 0 {
+	if r.ringDesc == nil || len(r.ringTokens) == 0 {
 		return ReplicationSet{}, ErrEmptyRing
 	}
 
@@ -284,10 +267,10 @@ func (r *Ring) GetAll() (ReplicationSet, error) {
 }
 
 func (r *Ring) search(key uint32) int {
-	i := sort.Search(len(r.ringDesc.Tokens), func(x int) bool {
-		return r.ringDesc.Tokens[x].Token > key
+	i := sort.Search(len(r.ringTokens), func(x int) bool {
+		return r.ringTokens[x].Token > key
 	})
-	if i >= len(r.ringDesc.Tokens) {
+	if i >= len(r.ringTokens) {
 		i = 0
 	}
 	return i
@@ -301,9 +284,7 @@ func (r *Ring) Describe(ch chan<- *prometheus.Desc) {
 	ch <- r.numTokensDesc
 }
 
-func countTokens(ringDesc *Desc) (map[string]uint32, map[string]uint32) {
-	tokens := ringDesc.Tokens
-
+func countTokens(ringDesc *Desc, tokens []TokenDesc) (map[string]uint32, map[string]uint32) {
 	owned := map[string]uint32{}
 	numTokens := map[string]uint32{}
 	for i, token := range tokens {
@@ -332,7 +313,7 @@ func (r *Ring) Collect(ch chan<- prometheus.Metric) {
 	r.mtx.RLock()
 	defer r.mtx.RUnlock()
 
-	numTokens, ownedRange := countTokens(r.ringDesc)
+	numTokens, ownedRange := countTokens(r.ringDesc, r.ringTokens)
 	for id, totalOwned := range ownedRange {
 		ch <- prometheus.MustNewConstMetric(
 			r.memberOwnershipDesc,
@@ -392,7 +373,7 @@ func (r *Ring) Collect(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(
 		r.totalTokensDesc,
 		prometheus.GaugeValue,
-		float64(len(r.ringDesc.Tokens)),
+		float64(len(r.ringTokens)),
 		r.name,
 	)
 }
diff --git a/pkg/ring/ring.pb.go b/pkg/ring/ring.pb.go
index 0554fc2b99..72e40d40e0 100644
--- a/pkg/ring/ring.pb.go
+++ b/pkg/ring/ring.pb.go
@@ -60,7 +60,6 @@ func (IngesterState) EnumDescriptor() ([]byte, []int) {
 
 type Desc struct {
 	Ingesters map[string]IngesterDesc `protobuf:"bytes,1,rep,name=ingesters,proto3" json:"ingesters" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	Tokens    []TokenDesc             `protobuf:"bytes,2,rep,name=tokens,proto3" json:"tokens"`
 }
 
 func (m *Desc) Reset()      { *m = Desc{} }
@@ -102,13 +101,6 @@ func (m *Desc) GetIngesters() map[string]IngesterDesc {
 	return nil
 }
 
-func (m *Desc) GetTokens() []TokenDesc {
-	if m != nil {
-		return m.Tokens
-	}
-	return nil
-}
-
 type IngesterDesc struct {
 	Addr      string        `protobuf:"bytes,1,opt,name=addr,proto3" json:"addr,omitempty"`
 	Timestamp int64         `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
@@ -176,96 +168,42 @@ func (m *IngesterDesc) GetTokens() []uint32 {
 	return nil
 }
 
-type TokenDesc struct {
-	Token    uint32 `protobuf:"varint,1,opt,name=token,proto3" json:"token,omitempty"`
-	Ingester string `protobuf:"bytes,2,opt,name=ingester,proto3" json:"ingester,omitempty"`
-}
-
-func (m *TokenDesc) Reset()      { *m = TokenDesc{} }
-func (*TokenDesc) ProtoMessage() {}
-func (*TokenDesc) Descriptor() ([]byte, []int) {
-	return fileDescriptor_26381ed67e202a6e, []int{2}
-}
-func (m *TokenDesc) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *TokenDesc) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_TokenDesc.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *TokenDesc) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_TokenDesc.Merge(m, src)
-}
-func (m *TokenDesc) XXX_Size() int {
-	return m.Size()
-}
-func (m *TokenDesc) XXX_DiscardUnknown() {
-	xxx_messageInfo_TokenDesc.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_TokenDesc proto.InternalMessageInfo
-
-func (m *TokenDesc) GetToken() uint32 {
-	if m != nil {
-		return m.Token
-	}
-	return 0
-}
-
-func (m *TokenDesc) GetIngester() string {
-	if m != nil {
-		return m.Ingester
-	}
-	return ""
-}
-
 func init() {
 	proto.RegisterEnum("ring.IngesterState", IngesterState_name, IngesterState_value)
 	proto.RegisterType((*Desc)(nil), "ring.Desc")
 	proto.RegisterMapType((map[string]IngesterDesc)(nil), "ring.Desc.IngestersEntry")
 	proto.RegisterType((*IngesterDesc)(nil), "ring.IngesterDesc")
-	proto.RegisterType((*TokenDesc)(nil), "ring.TokenDesc")
 }
 
 func init() { proto.RegisterFile("ring.proto", fileDescriptor_26381ed67e202a6e) }
 
 var fileDescriptor_26381ed67e202a6e = []byte{
-	// 426 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xcd, 0x6e, 0xd3, 0x40,
-	0x10, 0xc7, 0x77, 0xe2, 0xb5, 0x89, 0x27, 0xa4, 0x58, 0x03, 0x42, 0x26, 0x42, 0x8b, 0x95, 0x93,
-	0x41, 0x6a, 0x2a, 0x05, 0x0e, 0x08, 0xa9, 0x87, 0x96, 0x1a, 0x94, 0x28, 0x0a, 0x95, 0x89, 0x7a,
-	0x4f, 0xda, 0xc5, 0x44, 0x25, 0x71, 0x65, 0x6f, 0x90, 0x7a, 0xe3, 0x0d, 0xe0, 0x31, 0x78, 0x12,
-	0xd4, 0x63, 0x8e, 0x3d, 0x21, 0xe2, 0x5c, 0x38, 0xf6, 0x11, 0xd0, 0xae, 0xf3, 0x41, 0x6e, 0xf3,
-	0xdb, 0xff, 0xc7, 0xee, 0x58, 0x46, 0xcc, 0xc6, 0xd3, 0xa4, 0x75, 0x95, 0xa5, 0x2a, 0x25, 0xae,
-	0xe7, 0xc6, 0x7e, 0x32, 0x56, 0x9f, 0x67, 0xa3, 0xd6, 0x79, 0x3a, 0x39, 0x48, 0xd2, 0x24, 0x3d,
-	0x30, 0xe2, 0x68, 0xf6, 0xc9, 0x90, 0x01, 0x33, 0x95, 0xa1, 0xe6, 0x2f, 0x40, 0x7e, 0x22, 0xf3,
-	0x73, 0x3a, 0x44, 0x77, 0x3c, 0x4d, 0x64, 0xae, 0x64, 0x96, 0xfb, 0x10, 0x58, 0x61, 0xad, 0xfd,
-	0xa4, 0x65, 0xda, 0xb5, 0xdc, 0xea, 0xac, 0xb5, 0x68, 0xaa, 0xb2, 0xeb, 0x63, 0x7e, 0xf3, 0xfb,
-	0x19, 0x8b, 0xb7, 0x09, 0xda, 0x47, 0x47, 0xa5, 0x97, 0x72, 0x9a, 0xfb, 0x15, 0x93, 0x7d, 0x50,
-	0x66, 0x07, 0xfa, 0x4c, 0x17, 0xac, 0x12, 0x2b, 0x53, 0xe3, 0x14, 0xf7, 0x76, 0x1b, 0xc9, 0x43,
-	0xeb, 0x52, 0x5e, 0xfb, 0x10, 0x40, 0xe8, 0xc6, 0x7a, 0xa4, 0x10, 0xed, 0xaf, 0xc3, 0x2f, 0x33,
-	0xe9, 0x57, 0x02, 0x08, 0x6b, 0x6d, 0x2a, 0x1b, 0xd7, 0x31, 0x5d, 0x1a, 0x97, 0x86, 0x37, 0x95,
-	0xd7, 0xd0, 0xfc, 0x0e, 0x78, 0xff, 0x7f, 0x8d, 0x08, 0xf9, 0xf0, 0xe2, 0x22, 0x5b, 0x35, 0x9a,
-	0x99, 0x9e, 0xa2, 0xab, 0xc6, 0x13, 0x99, 0xab, 0xe1, 0xe4, 0xca, 0xd4, 0x5a, 0xf1, 0xf6, 0x80,
-	0x9e, 0xa3, 0x9d, 0xab, 0xa1, 0x92, 0xbe, 0x15, 0x40, 0xb8, 0xd7, 0x7e, 0xb8, 0x7b, 0xe1, 0x47,
-	0x2d, 0xc5, 0xa5, 0x83, 0x1e, 0x6f, 0xd6, 0x75, 0x02, 0x2b, 0xac, 0xaf, 0xf7, 0xea, 0xf2, 0x2a,
-	0xf7, 0xec, 0x2e, 0xaf, 0xda, 0x9e, 0xd3, 0x3c, 0x44, 0x77, 0xb3, 0x3e, 0x3d, 0x42, 0xdb, 0x58,
-	0xcc, 0x73, 0xea, 0x71, 0x09, 0xd4, 0xc0, 0xea, 0xfa, 0x13, 0x9a, 0xe7, 0xb8, 0xf1, 0x86, 0x5f,
-	0xf4, 0xb0, 0xbe, 0x73, 0x35, 0x21, 0x3a, 0x47, 0x6f, 0x07, 0x9d, 0xb3, 0xc8, 0x63, 0x54, 0xc3,
-	0x7b, 0xbd, 0xe8, 0xe8, 0xac, 0xd3, 0x7f, 0xef, 0x81, 0x86, 0xd3, 0xa8, 0x7f, 0xa2, 0xa1, 0xa2,
-	0xa1, 0xfb, 0xa1, 0xd3, 0xd7, 0x60, 0x51, 0x15, 0x79, 0x2f, 0x7a, 0x37, 0xf0, 0xf8, 0xf1, 0xab,
-	0xf9, 0x42, 0xb0, 0xdb, 0x85, 0x60, 0x77, 0x0b, 0x01, 0xdf, 0x0a, 0x01, 0x3f, 0x0b, 0x01, 0x37,
-	0x85, 0x80, 0x79, 0x21, 0xe0, 0x4f, 0x21, 0xe0, 0x6f, 0x21, 0xd8, 0x5d, 0x21, 0xe0, 0xc7, 0x52,
-	0xb0, 0xf9, 0x52, 0xb0, 0xdb, 0xa5, 0x60, 0x23, 0xc7, 0xfc, 0x24, 0x2f, 0xff, 0x05, 0x00, 0x00,
-	0xff, 0xff, 0x3b, 0x76, 0x95, 0xe8, 0x67, 0x02, 0x00, 0x00,
+	// 387 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0x3f, 0x6f, 0xd3, 0x40,
+	0x18, 0xc6, 0xef, 0xb5, 0xcf, 0xc6, 0x79, 0x43, 0x2b, 0xeb, 0x90, 0x90, 0xa9, 0xd0, 0x61, 0x75,
+	0x32, 0x48, 0xb8, 0x52, 0x60, 0x40, 0x48, 0x0c, 0x2d, 0x35, 0xc8, 0x56, 0x14, 0x2a, 0x53, 0x75,
+	0x77, 0xda, 0xc3, 0x58, 0x25, 0x76, 0x65, 0x5f, 0x90, 0xba, 0xf1, 0x0d, 0xe0, 0x0b, 0xb0, 0xf3,
+	0x51, 0x3a, 0x66, 0xcc, 0x84, 0x88, 0xb3, 0x30, 0xe6, 0x23, 0xa0, 0x3b, 0x27, 0x0a, 0xd9, 0x9e,
+	0xdf, 0x3d, 0x7f, 0xde, 0xe1, 0x10, 0xeb, 0xa2, 0xcc, 0xc3, 0x9b, 0xba, 0x92, 0x15, 0xa3, 0x4a,
+	0x1f, 0x3c, 0xcf, 0x0b, 0xf9, 0x79, 0x3a, 0x0e, 0x2f, 0xab, 0xc9, 0x51, 0x5e, 0xe5, 0xd5, 0x91,
+	0x36, 0xc7, 0xd3, 0x4f, 0x9a, 0x34, 0x68, 0xd5, 0x95, 0x0e, 0x7f, 0x02, 0xd2, 0x53, 0xd1, 0x5c,
+	0xb2, 0x37, 0xd8, 0x2b, 0xca, 0x5c, 0x34, 0x52, 0xd4, 0x8d, 0x07, 0xbe, 0x19, 0xf4, 0x07, 0x8f,
+	0x42, 0xbd, 0xae, 0xec, 0x30, 0xde, 0x78, 0x51, 0x29, 0xeb, 0xdb, 0x13, 0x7a, 0xf7, 0xfb, 0x09,
+	0x49, 0xb7, 0x8d, 0x83, 0x33, 0xdc, 0xdf, 0x8d, 0x30, 0x17, 0xcd, 0x6b, 0x71, 0xeb, 0x81, 0x0f,
+	0x41, 0x2f, 0x55, 0x92, 0x05, 0x68, 0x7d, 0xcd, 0xbe, 0x4c, 0x85, 0x67, 0xf8, 0x10, 0xf4, 0x07,
+	0xac, 0x9b, 0xdf, 0xd4, 0xd4, 0x99, 0xb4, 0x0b, 0xbc, 0x36, 0x5e, 0x41, 0x42, 0x1d, 0xc3, 0x35,
+	0x0f, 0xbf, 0x03, 0xde, 0xff, 0x3f, 0xc1, 0x18, 0xd2, 0xec, 0xea, 0xaa, 0x5e, 0xef, 0x6a, 0xcd,
+	0x1e, 0x63, 0x4f, 0x16, 0x13, 0xd1, 0xc8, 0x6c, 0x72, 0xa3, 0xc7, 0xcd, 0x74, 0xfb, 0xc0, 0x9e,
+	0xa2, 0xd5, 0xc8, 0x4c, 0x0a, 0xcf, 0xf4, 0x21, 0xd8, 0x1f, 0x3c, 0xd8, 0x3d, 0xfb, 0x51, 0x59,
+	0x69, 0x97, 0x60, 0x0f, 0xd1, 0x96, 0xd5, 0xb5, 0x28, 0x1b, 0xcf, 0xf6, 0xcd, 0x60, 0x2f, 0x5d,
+	0x53, 0x42, 0x1d, 0xea, 0x5a, 0x09, 0x75, 0x2c, 0xd7, 0x7e, 0x36, 0xc4, 0xbd, 0x9d, 0x2e, 0x43,
+	0xb4, 0x8f, 0xdf, 0x9e, 0xc7, 0x17, 0x91, 0x4b, 0x58, 0x1f, 0xef, 0x0d, 0xa3, 0xe3, 0x8b, 0x78,
+	0xf4, 0xde, 0x05, 0x05, 0x67, 0xd1, 0xe8, 0x54, 0x81, 0xa1, 0x20, 0xf9, 0x10, 0x8f, 0x14, 0x98,
+	0xcc, 0x41, 0x3a, 0x8c, 0xde, 0x9d, 0xbb, 0xf4, 0xe4, 0xe5, 0x6c, 0xc1, 0xc9, 0x7c, 0xc1, 0xc9,
+	0x6a, 0xc1, 0xe1, 0x5b, 0xcb, 0xe1, 0x57, 0xcb, 0xe1, 0xae, 0xe5, 0x30, 0x6b, 0x39, 0xfc, 0x69,
+	0x39, 0xfc, 0x6d, 0x39, 0x59, 0xb5, 0x1c, 0x7e, 0x2c, 0x39, 0x99, 0x2d, 0x39, 0x99, 0x2f, 0x39,
+	0x19, 0xdb, 0xfa, 0xf3, 0x5e, 0xfc, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x33, 0x18, 0xb8, 0xad, 0xff,
+	0x01, 0x00, 0x00,
 }
 
 func (x IngesterState) String() string {
@@ -304,14 +242,6 @@ func (this *Desc) Equal(that interface{}) bool {
 			return false
 		}
 	}
-	if len(this.Tokens) != len(that1.Tokens) {
-		return false
-	}
-	for i := range this.Tokens {
-		if !this.Tokens[i].Equal(&that1.Tokens[i]) {
-			return false
-		}
-	}
 	return true
 }
 func (this *IngesterDesc) Equal(that interface{}) bool {
@@ -352,38 +282,11 @@ func (this *IngesterDesc) Equal(that interface{}) bool {
 	}
 	return true
 }
-func (this *TokenDesc) Equal(that interface{}) bool {
-	if that == nil {
-		return this == nil
-	}
-
-	that1, ok := that.(*TokenDesc)
-	if !ok {
-		that2, ok := that.(TokenDesc)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return this == nil
-	} else if this == nil {
-		return false
-	}
-	if this.Token != that1.Token {
-		return false
-	}
-	if this.Ingester != that1.Ingester {
-		return false
-	}
-	return true
-}
 func (this *Desc) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 6)
+	s := make([]string, 0, 5)
 	s = append(s, "&ring.Desc{")
 	keysForIngesters := make([]string, 0, len(this.Ingesters))
 	for k, _ := range this.Ingesters {
@@ -398,13 +301,6 @@ func (this *Desc) GoString() string {
 	if this.Ingesters != nil {
 		s = append(s, "Ingesters: "+mapStringForIngesters+",\n")
 	}
-	if this.Tokens != nil {
-		vs := make([]*TokenDesc, len(this.Tokens))
-		for i := range vs {
-			vs[i] = &this.Tokens[i]
-		}
-		s = append(s, "Tokens: "+fmt.Sprintf("%#v", vs)+",\n")
-	}
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -421,17 +317,6 @@ func (this *IngesterDesc) GoString() string {
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
-func (this *TokenDesc) GoString() string {
-	if this == nil {
-		return "nil"
-	}
-	s := make([]string, 0, 6)
-	s = append(s, "&ring.TokenDesc{")
-	s = append(s, "Token: "+fmt.Sprintf("%#v", this.Token)+",\n")
-	s = append(s, "Ingester: "+fmt.Sprintf("%#v", this.Ingester)+",\n")
-	s = append(s, "}")
-	return strings.Join(s, "")
-}
 func valueToGoStringRing(v interface{}, typ string) string {
 	rv := reflect.ValueOf(v)
 	if rv.IsNil() {
@@ -481,18 +366,6 @@ func (m *Desc) MarshalTo(dAtA []byte) (int, error) {
 			i += n1
 		}
 	}
-	if len(m.Tokens) > 0 {
-		for _, msg := range m.Tokens {
-			dAtA[i] = 0x12
-			i++
-			i = encodeVarintRing(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
-			}
-			i += n
-		}
-	}
 	return i, nil
 }
 
@@ -547,35 +420,6 @@ func (m *IngesterDesc) MarshalTo(dAtA []byte) (int, error) {
 	return i, nil
 }
 
-func (m *TokenDesc) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-func (m *TokenDesc) MarshalTo(dAtA []byte) (int, error) {
-	var i int
-	_ = i
-	var l int
-	_ = l
-	if m.Token != 0 {
-		dAtA[i] = 0x8
-		i++
-		i = encodeVarintRing(dAtA, i, uint64(m.Token))
-	}
-	if len(m.Ingester) > 0 {
-		dAtA[i] = 0x12
-		i++
-		i = encodeVarintRing(dAtA, i, uint64(len(m.Ingester)))
-		i += copy(dAtA[i:], m.Ingester)
-	}
-	return i, nil
-}
-
 func encodeVarintRing(dAtA []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
@@ -600,12 +444,6 @@ func (m *Desc) Size() (n int) {
 			n += mapEntrySize + 1 + sovRing(uint64(mapEntrySize))
 		}
 	}
-	if len(m.Tokens) > 0 {
-		for _, e := range m.Tokens {
-			l = e.Size()
-			n += 1 + l + sovRing(uint64(l))
-		}
-	}
 	return n
 }
 
@@ -635,22 +473,6 @@ func (m *IngesterDesc) Size() (n int) {
 	return n
 }
 
-func (m *TokenDesc) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	if m.Token != 0 {
-		n += 1 + sovRing(uint64(m.Token))
-	}
-	l = len(m.Ingester)
-	if l > 0 {
-		n += 1 + l + sovRing(uint64(l))
-	}
-	return n
-}
-
 func sovRing(x uint64) (n int) {
 	for {
 		n++
@@ -680,7 +502,6 @@ func (this *Desc) String() string {
 	mapStringForIngesters += "}"
 	s := strings.Join([]string{`&Desc{`,
 		`Ingesters:` + mapStringForIngesters + `,`,
-		`Tokens:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.Tokens), "TokenDesc", "TokenDesc", 1), `&`, ``, 1) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -698,17 +519,6 @@ func (this *IngesterDesc) String() string {
 	}, "")
 	return s
 }
-func (this *TokenDesc) String() string {
-	if this == nil {
-		return "nil"
-	}
-	s := strings.Join([]string{`&TokenDesc{`,
-		`Token:` + fmt.Sprintf("%v", this.Token) + `,`,
-		`Ingester:` + fmt.Sprintf("%v", this.Ingester) + `,`,
-		`}`,
-	}, "")
-	return s
-}
 func valueToStringRing(v interface{}) string {
 	rv := reflect.ValueOf(v)
 	if rv.IsNil() {
@@ -875,40 +685,6 @@ func (m *Desc) Unmarshal(dAtA []byte) error {
 			}
 			m.Ingesters[mapkey] = *mapvalue
 			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Tokens", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowRing
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthRing
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthRing
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Tokens = append(m.Tokens, TokenDesc{})
-			if err := m.Tokens[len(m.Tokens)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipRing(dAtA[iNdEx:])
@@ -1132,110 +908,6 @@ func (m *IngesterDesc) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
-func (m *TokenDesc) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowRing
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: TokenDesc: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: TokenDesc: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Token", wireType)
-			}
-			m.Token = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowRing
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Token |= uint32(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Ingester", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowRing
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthRing
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthRing
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Ingester = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		default:
-			iNdEx = preIndex
-			skippy, err := skipRing(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthRing
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthRing
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
 func skipRing(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
diff --git a/pkg/ring/ring.proto b/pkg/ring/ring.proto
index 1c96830930..9670b01c09 100644
--- a/pkg/ring/ring.proto
+++ b/pkg/ring/ring.proto
@@ -9,7 +9,7 @@ option (gogoproto.unmarshaler_all) = true;
 
 message Desc {
 	map<string,IngesterDesc> ingesters = 1 [(gogoproto.nullable) = false];
-	repeated TokenDesc tokens = 2 [(gogoproto.nullable) = false];
+	reserved 2;
 }
 
 message IngesterDesc {
@@ -21,11 +21,6 @@ message IngesterDesc {
 	repeated uint32 tokens = 6;
 }
 
-message TokenDesc {
-	uint32 token = 1;
-	string ingester = 2;
-}
-
 enum IngesterState {
 	ACTIVE = 0;
 	LEAVING = 1;
diff --git a/pkg/util/metrics_helper.go b/pkg/util/metrics_helper.go
index 4957c40ded..234b154ebd 100644
--- a/pkg/util/metrics_helper.go
+++ b/pkg/util/metrics_helper.go
@@ -4,77 +4,147 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"sync"
 
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
 )
 
-// MetricFamiliesPerUser is a collection of metrics gathered via calling Gatherer.Gather() method on different
-// gatherers, one per user.
-// First key = userID, second key = metric name.
-// Value = slice of gathered values with the same metric name.
-type MetricFamiliesPerUser map[string]map[string]*dto.MetricFamily
+// Data for single value (counter/gauge) with labels.
+type singleValueWithLabels struct {
+	Value       float64
+	LabelValues []string
+}
 
-func BuildMetricFamiliesPerUserFromUserRegistries(regs map[string]*prometheus.Registry) MetricFamiliesPerUser {
-	data := MetricFamiliesPerUser{}
-	for userID, r := range regs {
-		m, err := r.Gather()
-		if err == nil {
-			err = data.AddGatheredDataForUser(userID, m)
-		}
+// Key to this map is value unique to label values (generated by getLabelsString function)
+type singleValueWithLabelsMap map[string]singleValueWithLabels
 
-		if err != nil {
-			level.Warn(Logger).Log("msg", "failed to gather metrics from registry", "user", userID, "err", err)
-			continue
-		}
+// This function is used to aggregate results with different labels into a map. Values for same labels are added together.
+func (m singleValueWithLabelsMap) aggregateFn(labelsKey string, labelValues []string, value float64) {
+	r := m[labelsKey]
+	if r.LabelValues == nil {
+		r.LabelValues = labelValues
+	}
+
+	r.Value += value
+	m[labelsKey] = r
+}
+
+func (m singleValueWithLabelsMap) WriteToMetricChannel(out chan<- prometheus.Metric, desc *prometheus.Desc, valueType prometheus.ValueType) {
+	for _, cr := range m {
+		out <- prometheus.MustNewConstMetric(desc, valueType, cr.Value, cr.LabelValues...)
 	}
-	return data
 }
 
-// AddGatheredDataForUser adds user-specific output of Gatherer.Gather method.
+// MetricFamilyMap is a map of metric names to their family (metrics with same name, but different labels)
+// Keeping map of metric name to its family makes it easier to do searches later.
+type MetricFamilyMap map[string]*dto.MetricFamily
+
+// NewMetricFamilyMap sorts output from Gatherer.Gather method into a map.
 // Gatherer.Gather specifies that there metric families are uniquely named, and we use that fact here.
 // If they are not, this method returns error.
-func (d MetricFamiliesPerUser) AddGatheredDataForUser(userID string, metrics []*dto.MetricFamily) error {
-	// Keeping map of metric name to its family makes it easier to do searches later.
-	perMetricName := map[string]*dto.MetricFamily{}
+func NewMetricFamilyMap(metrics []*dto.MetricFamily) (MetricFamilyMap, error) {
+	perMetricName := MetricFamilyMap{}
 
 	for _, m := range metrics {
 		name := m.GetName()
 		// these errors should never happen when passing Gatherer.Gather() output.
 		if name == "" {
-			return errors.New("empty name for metric family")
+			return nil, errors.New("empty name for metric family")
 		}
 		if perMetricName[name] != nil {
-			return fmt.Errorf("non-unique name for metric family: %q", name)
+			return nil, fmt.Errorf("non-unique name for metric family: %q", name)
 		}
 
 		perMetricName[name] = m
 	}
 
-	d[userID] = perMetricName
-	return nil
+	return perMetricName, nil
+}
+
+func (mfm MetricFamilyMap) SumCounters(name string) float64 {
+	return sum(mfm[name], counterValue)
+}
+
+func (mfm MetricFamilyMap) SumGauges(name string) float64 {
+	return sum(mfm[name], gaugeValue)
+}
+
+func (mfm MetricFamilyMap) SumHistograms(name string) HistogramData {
+	hd := HistogramData{}
+	mfm.SumHistogramsTo(name, &hd)
+	return hd
+}
+
+func (mfm MetricFamilyMap) SumHistogramsTo(name string, output *HistogramData) {
+	for _, m := range mfm[name].GetMetric() {
+		output.AddHistogram(m.GetHistogram())
+	}
+}
+
+func (mfm MetricFamilyMap) SumSummaries(name string) SummaryData {
+	sd := SummaryData{}
+	mfm.SumSummariesTo(name, &sd)
+	return sd
+}
+
+func (mfm MetricFamilyMap) SumSummariesTo(name string, output *SummaryData) {
+	for _, m := range mfm[name].GetMetric() {
+		output.AddSummary(m.GetSummary())
+	}
+}
+
+func (mfm MetricFamilyMap) sumOfSingleValuesWithLabels(metric string, labelNames []string, extractFn func(*dto.Metric) float64, aggregateFn func(labelsKey string, labelValues []string, value float64)) {
+	metricsPerLabelValue := getMetricsWithLabelNames(mfm[metric], labelNames)
+
+	for key, mlv := range metricsPerLabelValue {
+		for _, m := range mlv.metrics {
+			val := extractFn(m)
+			aggregateFn(key, mlv.labelValues, val)
+		}
+	}
+}
+
+// MetricFamiliesPerUser is a collection of metrics gathered via calling Gatherer.Gather() method on different
+// gatherers, one per user.
+type MetricFamiliesPerUser map[string]MetricFamilyMap
+
+func BuildMetricFamiliesPerUserFromUserRegistries(regs map[string]*prometheus.Registry) MetricFamiliesPerUser {
+	data := MetricFamiliesPerUser{}
+	for userID, r := range regs {
+		m, err := r.Gather()
+		if err == nil {
+			var mfm MetricFamilyMap // := would shadow err from outer block, and single err check will not work
+			mfm, err = NewMetricFamilyMap(m)
+			if err == nil {
+				data[userID] = mfm
+			}
+		}
+
+		if err != nil {
+			level.Warn(Logger).Log("msg", "failed to gather metrics from registry", "user", userID, "err", err)
+			continue
+		}
+	}
+	return data
 }
 
 func (d MetricFamiliesPerUser) SendSumOfCounters(out chan<- prometheus.Metric, desc *prometheus.Desc, counter string) {
 	result := float64(0)
-	for _, perMetric := range d {
-		result += sum(perMetric[counter], counterValue)
+	for _, userMetrics := range d {
+		result += userMetrics.SumCounters(counter)
 	}
-
 	out <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, result)
 }
 
 func (d MetricFamiliesPerUser) SendSumOfCountersWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, counter string, labelNames ...string) {
-	result := d.sumOfSingleValuesWithLabels(counter, counterValue, labelNames)
-	for _, cr := range result {
-		out <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, cr.value, cr.labelValues...)
-	}
+	d.sumOfSingleValuesWithLabels(counter, counterValue, labelNames).WriteToMetricChannel(out, desc, prometheus.CounterValue)
 }
 
 func (d MetricFamiliesPerUser) SendSumOfCountersPerUser(out chan<- prometheus.Metric, desc *prometheus.Desc, counter string) {
-	for user, perMetric := range d {
-		v := sum(perMetric[counter], counterValue)
+	for user, userMetrics := range d {
+		v := userMetrics.SumCounters(counter)
 
 		out <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, v, user)
 	}
@@ -82,70 +152,35 @@ func (d MetricFamiliesPerUser) SendSumOfCountersPerUser(out chan<- prometheus.Me
 
 func (d MetricFamiliesPerUser) SendSumOfGauges(out chan<- prometheus.Metric, desc *prometheus.Desc, gauge string) {
 	result := float64(0)
-	for _, perMetric := range d {
-		result += sum(perMetric[gauge], gaugeValue)
+	for _, userMetrics := range d {
+		result += userMetrics.SumGauges(gauge)
 	}
 	out <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, result)
 }
 
 func (d MetricFamiliesPerUser) SendSumOfGaugesWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, gauge string, labelNames ...string) {
-	result := d.sumOfSingleValuesWithLabels(gauge, gaugeValue, labelNames)
-	for _, cr := range result {
-		out <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, cr.value, cr.labelValues...)
-	}
+	d.sumOfSingleValuesWithLabels(gauge, gaugeValue, labelNames).WriteToMetricChannel(out, desc, prometheus.GaugeValue)
 }
 
-type singleResult struct {
-	value       float64
-	labelValues []string
-}
-
-func (d MetricFamiliesPerUser) sumOfSingleValuesWithLabels(metric string, fn func(*dto.Metric) float64, labelNames []string) map[string]singleResult {
-	result := map[string]singleResult{}
-
+func (d MetricFamiliesPerUser) sumOfSingleValuesWithLabels(metric string, fn func(*dto.Metric) float64, labelNames []string) singleValueWithLabelsMap {
+	result := singleValueWithLabelsMap{}
 	for _, userMetrics := range d {
-		metricsPerLabelValue := getMetricsWithLabelNames(userMetrics[metric], labelNames)
-
-		for key, mlv := range metricsPerLabelValue {
-			for _, m := range mlv.metrics {
-				r := result[key]
-				if r.labelValues == nil {
-					r.labelValues = mlv.labelValues
-				}
-
-				r.value += fn(m)
-				result[key] = r
-			}
-		}
+		userMetrics.sumOfSingleValuesWithLabels(metric, labelNames, fn, result.aggregateFn)
 	}
-
 	return result
 }
 
 func (d MetricFamiliesPerUser) SendSumOfSummaries(out chan<- prometheus.Metric, desc *prometheus.Desc, summaryName string) {
-	var (
-		sampleCount uint64
-		sampleSum   float64
-		quantiles   map[float64]float64
-	)
-
+	summaryData := SummaryData{}
 	for _, userMetrics := range d {
-		for _, m := range userMetrics[summaryName].GetMetric() {
-			summary := m.GetSummary()
-			sampleCount += summary.GetSampleCount()
-			sampleSum += summary.GetSampleSum()
-			quantiles = mergeSummaryQuantiles(quantiles, summary.GetQuantile())
-		}
+		userMetrics.SumSummariesTo(summaryName, &summaryData)
 	}
-
-	out <- prometheus.MustNewConstSummary(desc, sampleCount, sampleSum, quantiles)
+	out <- summaryData.Metric(desc)
 }
 
 func (d MetricFamiliesPerUser) SendSumOfSummariesWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, summaryName string, labelNames ...string) {
 	type summaryResult struct {
-		sampleCount uint64
-		sampleSum   float64
-		quantiles   map[float64]float64
+		data        SummaryData
 		labelValues []string
 	}
 
@@ -161,74 +196,26 @@ func (d MetricFamiliesPerUser) SendSumOfSummariesWithLabels(out chan<- prometheu
 					r.labelValues = mwl.labelValues
 				}
 
-				summary := m.GetSummary()
-				r.sampleCount += summary.GetSampleCount()
-				r.sampleSum += summary.GetSampleSum()
-				r.quantiles = mergeSummaryQuantiles(r.quantiles, summary.GetQuantile())
-
+				r.data.AddSummary(m.GetSummary())
 				result[key] = r
 			}
 		}
 	}
 
 	for _, sr := range result {
-		out <- prometheus.MustNewConstSummary(desc, sr.sampleCount, sr.sampleSum, sr.quantiles, sr.labelValues...)
+		out <- sr.data.Metric(desc, sr.labelValues...)
 	}
 }
 
 func (d MetricFamiliesPerUser) SendSumOfHistograms(out chan<- prometheus.Metric, desc *prometheus.Desc, histogramName string) {
-	var (
-		sampleCount uint64
-		sampleSum   float64
-		buckets     map[float64]uint64
-	)
-
+	hd := HistogramData{}
 	for _, userMetrics := range d {
-		for _, m := range userMetrics[histogramName].GetMetric() {
-			histo := m.GetHistogram()
-			sampleCount += histo.GetSampleCount()
-			sampleSum += histo.GetSampleSum()
-			buckets = mergeHistogramBuckets(buckets, histo.GetBucket())
-		}
+		userMetrics.SumHistogramsTo(histogramName, &hd)
 	}
-
-	out <- prometheus.MustNewConstHistogram(desc, sampleCount, sampleSum, buckets)
-}
-
-func mergeSummaryQuantiles(quantiles map[float64]float64, summaryQuantiles []*dto.Quantile) map[float64]float64 {
-	if len(summaryQuantiles) == 0 {
-		return quantiles
-	}
-
-	out := quantiles
-	if out == nil {
-		out = map[float64]float64{}
-	}
-
-	for _, q := range summaryQuantiles {
-		// we assume that all summaries have same quantiles
-		out[q.GetQuantile()] += q.GetValue()
-	}
-	return out
-}
-
-func mergeHistogramBuckets(buckets map[float64]uint64, histogramBuckets []*dto.Bucket) map[float64]uint64 {
-	if len(histogramBuckets) == 0 {
-		return buckets
-	}
-
-	out := buckets
-	if out == nil {
-		out = map[float64]uint64{}
-	}
-
-	for _, q := range histogramBuckets {
-		// we assume that all histograms have same buckets
-		out[q.GetUpperBound()] += q.GetCumulativeCount()
-	}
-	return out
+	out <- hd.Metric(desc)
 }
 
+// struct for holding metrics with same label values
 type metricsWithLabels struct {
 	labelValues []string
 	metrics     []*dto.Metric
@@ -294,3 +281,107 @@ func sum(mf *dto.MetricFamily, fn func(*dto.Metric) float64) float64 {
 // This works even if m is nil, m.Counter is nil or m.Counter.Value is nil (it returns 0 in those cases)
 func counterValue(m *dto.Metric) float64 { return m.GetCounter().GetValue() }
 func gaugeValue(m *dto.Metric) float64   { return m.GetGauge().GetValue() }
+
+// SummaryData keeps all data needed to create summary metric
+type SummaryData struct {
+	sampleCount uint64
+	sampleSum   float64
+	quantiles   map[float64]float64
+}
+
+func (s *SummaryData) AddSummary(sum *dto.Summary) {
+	s.sampleCount += sum.GetSampleCount()
+	s.sampleSum += sum.GetSampleSum()
+
+	qs := sum.GetQuantile()
+	if len(qs) > 0 && s.quantiles == nil {
+		s.quantiles = map[float64]float64{}
+	}
+
+	for _, q := range qs {
+		// we assume that all summaries have same quantiles
+		s.quantiles[q.GetQuantile()] += q.GetValue()
+	}
+}
+
+func (s *SummaryData) Metric(desc *prometheus.Desc, labelValues ...string) prometheus.Metric {
+	return prometheus.MustNewConstSummary(desc, s.sampleCount, s.sampleSum, s.quantiles, labelValues...)
+}
+
+// HistogramData keeps data required to build histogram Metric
+type HistogramData struct {
+	sampleCount uint64
+	sampleSum   float64
+	buckets     map[float64]uint64
+}
+
+// Adds histogram from gathered metrics to this histogram data.
+func (d *HistogramData) AddHistogram(histo *dto.Histogram) {
+	d.sampleCount += histo.GetSampleCount()
+	d.sampleSum += histo.GetSampleSum()
+
+	histoBuckets := histo.GetBucket()
+	if len(histoBuckets) > 0 && d.buckets == nil {
+		d.buckets = map[float64]uint64{}
+	}
+
+	for _, b := range histoBuckets {
+		// we assume that all histograms have same buckets
+		d.buckets[b.GetUpperBound()] += b.GetCumulativeCount()
+	}
+}
+
+// Merged another histogram data into this one.
+func (d *HistogramData) AddHistogramData(histo HistogramData) {
+	d.sampleCount += histo.sampleCount
+	d.sampleSum += histo.sampleSum
+
+	if len(histo.buckets) > 0 && d.buckets == nil {
+		d.buckets = map[float64]uint64{}
+	}
+
+	for bound, count := range histo.buckets {
+		// we assume that all histograms have same buckets
+		d.buckets[bound] += count
+	}
+}
+
+// Return prometheus metric from this histogram data.
+func (d *HistogramData) Metric(desc *prometheus.Desc) prometheus.Metric {
+	return prometheus.MustNewConstHistogram(desc, d.sampleCount, d.sampleSum, d.buckets)
+}
+
+// Creates new histogram data collector.
+func NewHistogramDataCollector(desc *prometheus.Desc) *HistogramDataCollector {
+	return &HistogramDataCollector{
+		desc: desc,
+		data: &HistogramData{},
+	}
+}
+
+// HistogramDataCollector combines histogram data, with prometheus descriptor. It can be registered
+// into prometheus to report histogram with stored data. Data can be updated via Add method.
+type HistogramDataCollector struct {
+	desc *prometheus.Desc
+
+	dataMu sync.RWMutex
+	data   *HistogramData
+}
+
+func (h *HistogramDataCollector) Describe(out chan<- *prometheus.Desc) {
+	out <- h.desc
+}
+
+func (h *HistogramDataCollector) Collect(out chan<- prometheus.Metric) {
+	h.dataMu.RLock()
+	defer h.dataMu.RUnlock()
+
+	out <- h.data.Metric(h.desc)
+}
+
+func (h *HistogramDataCollector) Add(hd HistogramData) {
+	h.dataMu.Lock()
+	defer h.dataMu.Unlock()
+
+	h.data.AddHistogramData(hd)
+}
diff --git a/tools/release/check-changelog.sh b/tools/release/check-changelog.sh
new file mode 100755
index 0000000000..6ba9c30e18
--- /dev/null
+++ b/tools/release/check-changelog.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Expect as input parameter the commits range to analyze.
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 range"
+    echo ""
+    echo "  range   The commit range to compare as documented at:"
+    echo "          https://git-scm.com/docs/gitrevisions"
+    echo ""
+    echo "Example:"
+    echo "  $0 v0.4.0...master"
+    echo ""
+    exit 1
+fi
+
+# Find all merged PRs.
+GIT_LOG=$(git log --pretty=format:"%s" $1)
+PR_LIST=$(echo "$GIT_LOG" | grep -Eo '#[0-9]+')
+PR_LIST_COUNT=$(echo "$PR_LIST" | wc -l | grep -Eo '[0-9]+')
+PR_AUTHORS_COUNT=$(git log --pretty=format:"%an" $1 | sort | uniq -i | wc -l | grep -Eo '[0-9]+')
+echo "Found ${PR_LIST_COUNT} PRs from ${PR_AUTHORS_COUNT} authors."
+echo ""
+
+# For each PR check if it's mentioned in the changelog.
+echo "List of missing PR in the CHANGELOG.md:"
+for PR in $PR_LIST; do
+    grep -q "$PR" CHANGELOG.md
+    if [ $? -eq 0 ]; then
+        continue
+    fi
+
+    # Print 1 line for the missing PR
+    echo -n "- ${PR}: "
+    echo "$GIT_LOG" | grep "$PR"
+done


Test patch:
diff --git a/pkg/compactor/compactor_test.go b/pkg/compactor/compactor_test.go
index 59e14a8e2c..99bf38f51c 100644
--- a/pkg/compactor/compactor_test.go
+++ b/pkg/compactor/compactor_test.go
@@ -99,6 +99,58 @@ func TestCompactor_ShouldDoNothingOnNoUserBlocks(t *testing.T) {
 		# TYPE cortex_compactor_runs_failed_total counter
 		# HELP cortex_compactor_runs_failed_total Total number of compaction runs failed.
 		cortex_compactor_runs_failed_total 0
+
+		# HELP cortex_compactor_garbage_collected_blocks_total TSDB Syncer: Total number of deleted blocks by compactor.
+		# TYPE cortex_compactor_garbage_collected_blocks_total counter
+		cortex_compactor_garbage_collected_blocks_total 0
+
+		# HELP cortex_compactor_garbage_collection_duration_seconds TSDB Syncer: Time it took to perform garbage collection iteration.
+		# TYPE cortex_compactor_garbage_collection_duration_seconds histogram
+		cortex_compactor_garbage_collection_duration_seconds_bucket{le="+Inf"} 0
+		cortex_compactor_garbage_collection_duration_seconds_sum 0
+		cortex_compactor_garbage_collection_duration_seconds_count 0
+
+		# HELP cortex_compactor_garbage_collection_failures_total TSDB Syncer: Total number of failed garbage collection operations.
+		# TYPE cortex_compactor_garbage_collection_failures_total counter
+		cortex_compactor_garbage_collection_failures_total 0
+
+		# HELP cortex_compactor_garbage_collection_total TSDB Syncer: Total number of garbage collection operations.
+		# TYPE cortex_compactor_garbage_collection_total counter
+		cortex_compactor_garbage_collection_total 0
+
+		# HELP cortex_compactor_sync_meta_duration_seconds TSDB Syncer: Time it took to sync meta files.
+		# TYPE cortex_compactor_sync_meta_duration_seconds histogram
+		cortex_compactor_sync_meta_duration_seconds_bucket{le="+Inf"} 0
+		cortex_compactor_sync_meta_duration_seconds_sum 0
+		cortex_compactor_sync_meta_duration_seconds_count 0
+
+		# HELP cortex_compactor_sync_meta_failures_total TSDB Syncer: Total number of failed sync meta operations.
+		# TYPE cortex_compactor_sync_meta_failures_total counter
+		cortex_compactor_sync_meta_failures_total 0
+
+		# HELP cortex_compactor_sync_meta_total TSDB Syncer: Total number of sync meta operations.
+		# TYPE cortex_compactor_sync_meta_total counter
+		cortex_compactor_sync_meta_total 0
+
+		# HELP cortex_compactor_group_compaction_runs_completed_total TSDB Syncer: Total number of group completed compaction runs. This also includes compactor group runs that resulted with no compaction.
+		# TYPE cortex_compactor_group_compaction_runs_completed_total counter
+		cortex_compactor_group_compaction_runs_completed_total 0
+
+		# HELP cortex_compactor_group_compaction_runs_started_total TSDB Syncer: Total number of group compaction attempts.
+		# TYPE cortex_compactor_group_compaction_runs_started_total counter
+		cortex_compactor_group_compaction_runs_started_total 0
+
+		# HELP cortex_compactor_group_compactions_failures_total TSDB Syncer: Total number of failed group compactions.
+		# TYPE cortex_compactor_group_compactions_failures_total counter
+		cortex_compactor_group_compactions_failures_total 0
+
+		# HELP cortex_compactor_group_compactions_total TSDB Syncer: Total number of group compaction attempts that resulted in a new block.
+		# TYPE cortex_compactor_group_compactions_total counter
+		cortex_compactor_group_compactions_total 0
+
+		# HELP cortex_compactor_group_vertical_compactions_total TSDB Syncer: Total number of group compaction attempts that resulted in a new block based on overlapping blocks.
+		# TYPE cortex_compactor_group_vertical_compactions_total counter
+		cortex_compactor_group_vertical_compactions_total 0
 	`)))
 }
 
@@ -142,6 +194,58 @@ func TestCompactor_ShouldRetryOnFailureWhileDiscoveringUsersFromBucket(t *testin
 		# TYPE cortex_compactor_runs_failed_total counter
 		# HELP cortex_compactor_runs_failed_total Total number of compaction runs failed.
 		cortex_compactor_runs_failed_total 1
+
+		# HELP cortex_compactor_garbage_collected_blocks_total TSDB Syncer: Total number of deleted blocks by compactor.
+		# TYPE cortex_compactor_garbage_collected_blocks_total counter
+		cortex_compactor_garbage_collected_blocks_total 0
+
+		# HELP cortex_compactor_garbage_collection_duration_seconds TSDB Syncer: Time it took to perform garbage collection iteration.
+		# TYPE cortex_compactor_garbage_collection_duration_seconds histogram
+		cortex_compactor_garbage_collection_duration_seconds_bucket{le="+Inf"} 0
+		cortex_compactor_garbage_collection_duration_seconds_sum 0
+		cortex_compactor_garbage_collection_duration_seconds_count 0
+
+		# HELP cortex_compactor_garbage_collection_failures_total TSDB Syncer: Total number of failed garbage collection operations.
+		# TYPE cortex_compactor_garbage_collection_failures_total counter
+		cortex_compactor_garbage_collection_failures_total 0
+
+		# HELP cortex_compactor_garbage_collection_total TSDB Syncer: Total number of garbage collection operations.
+		# TYPE cortex_compactor_garbage_collection_total counter
+		cortex_compactor_garbage_collection_total 0
+
+		# HELP cortex_compactor_sync_meta_duration_seconds TSDB Syncer: Time it took to sync meta files.
+		# TYPE cortex_compactor_sync_meta_duration_seconds histogram
+		cortex_compactor_sync_meta_duration_seconds_bucket{le="+Inf"} 0
+		cortex_compactor_sync_meta_duration_seconds_sum 0
+		cortex_compactor_sync_meta_duration_seconds_count 0
+
+		# HELP cortex_compactor_sync_meta_failures_total TSDB Syncer: Total number of failed sync meta operations.
+		# TYPE cortex_compactor_sync_meta_failures_total counter
+		cortex_compactor_sync_meta_failures_total 0
+
+		# HELP cortex_compactor_sync_meta_total TSDB Syncer: Total number of sync meta operations.
+		# TYPE cortex_compactor_sync_meta_total counter
+		cortex_compactor_sync_meta_total 0
+
+		# HELP cortex_compactor_group_compaction_runs_completed_total TSDB Syncer: Total number of group completed compaction runs. This also includes compactor group runs that resulted with no compaction.
+		# TYPE cortex_compactor_group_compaction_runs_completed_total counter
+		cortex_compactor_group_compaction_runs_completed_total 0
+
+		# HELP cortex_compactor_group_compaction_runs_started_total TSDB Syncer: Total number of group compaction attempts.
+		# TYPE cortex_compactor_group_compaction_runs_started_total counter
+		cortex_compactor_group_compaction_runs_started_total 0
+
+		# HELP cortex_compactor_group_compactions_failures_total TSDB Syncer: Total number of failed group compactions.
+		# TYPE cortex_compactor_group_compactions_failures_total counter
+		cortex_compactor_group_compactions_failures_total 0
+
+		# HELP cortex_compactor_group_compactions_total TSDB Syncer: Total number of group compaction attempts that resulted in a new block.
+		# TYPE cortex_compactor_group_compactions_total counter
+		cortex_compactor_group_compactions_total 0
+
+		# HELP cortex_compactor_group_vertical_compactions_total TSDB Syncer: Total number of group compaction attempts that resulted in a new block based on overlapping blocks.
+		# TYPE cortex_compactor_group_vertical_compactions_total counter
+		cortex_compactor_group_vertical_compactions_total 0
 	`)))
 }
 
@@ -191,6 +295,9 @@ func TestCompactor_ShouldIterateOverUsersAndRunCompaction(t *testing.T) {
 		`level=info msg="successfully compacted user blocks" user=user-2`,
 	}, strings.Split(strings.TrimSpace(logs.String()), "\n"))
 
+	// Instead of testing for shipper metrics, we only check our metrics here.
+	// Real shipper metrics are too variable to embed into a test.
+	testedMetrics := []string{"cortex_compactor_runs_started_total", "cortex_compactor_runs_completed_total", "cortex_compactor_runs_failed_total"}
 	assert.NoError(t, prom_testutil.GatherAndCompare(registry, strings.NewReader(`
 		# TYPE cortex_compactor_runs_started_total counter
 		# HELP cortex_compactor_runs_started_total Total number of compaction runs started.
@@ -203,7 +310,7 @@ func TestCompactor_ShouldIterateOverUsersAndRunCompaction(t *testing.T) {
 		# TYPE cortex_compactor_runs_failed_total counter
 		# HELP cortex_compactor_runs_failed_total Total number of compaction runs failed.
 		cortex_compactor_runs_failed_total 0
-	`)))
+	`), testedMetrics...))
 }
 
 func prepare(t *testing.T, bucketClient *cortex_tsdb.BucketClientMock) (*Compactor, *tsdbCompactorMock, *bytes.Buffer, prometheus.Gatherer) {
diff --git a/pkg/compactor/syncer_metrics_test.go b/pkg/compactor/syncer_metrics_test.go
new file mode 100644
index 0000000000..4103bbb610
--- /dev/null
+++ b/pkg/compactor/syncer_metrics_test.go
@@ -0,0 +1,232 @@
+package compactor
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSyncerMetrics(t *testing.T) {
+	reg := prometheus.NewPedanticRegistry()
+
+	sm := newSyncerMetrics(reg)
+	sm.gatherThanosSyncerMetrics(generateTestData(12345))
+	sm.gatherThanosSyncerMetrics(generateTestData(76543))
+	sm.gatherThanosSyncerMetrics(generateTestData(22222))
+	// total base = 111110
+
+	err := testutil.GatherAndCompare(reg, bytes.NewBufferString(`
+			# HELP cortex_compactor_sync_meta_total TSDB Syncer: Total number of sync meta operations.
+			# TYPE cortex_compactor_sync_meta_total counter
+			cortex_compactor_sync_meta_total 111110
+
+			# HELP cortex_compactor_sync_meta_failures_total TSDB Syncer: Total number of failed sync meta operations.
+			# TYPE cortex_compactor_sync_meta_failures_total counter
+			cortex_compactor_sync_meta_failures_total 222220
+
+			# HELP cortex_compactor_sync_meta_duration_seconds TSDB Syncer: Time it took to sync meta files.
+			# TYPE cortex_compactor_sync_meta_duration_seconds histogram
+			# Observed values: 3.7035, 22.9629, 6.6666 (seconds)
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="0.01"} 0
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="0.1"} 0
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="0.3"} 0
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="0.6"} 0
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="1"} 0
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="3"} 0
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="6"} 1
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="9"} 2
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="20"} 2
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="30"} 3
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="60"} 3
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="90"} 3
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="120"} 3
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="240"} 3
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="360"} 3
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="720"} 3
+			cortex_compactor_sync_meta_duration_seconds_bucket{le="+Inf"} 3
+			# rounding error
+			cortex_compactor_sync_meta_duration_seconds_sum 33.333000000000006
+			cortex_compactor_sync_meta_duration_seconds_count 3
+
+			# HELP cortex_compactor_garbage_collected_blocks_total TSDB Syncer: Total number of deleted blocks by compactor.
+			# TYPE cortex_compactor_garbage_collected_blocks_total counter
+			cortex_compactor_garbage_collected_blocks_total 444440
+
+			# HELP cortex_compactor_garbage_collection_total TSDB Syncer: Total number of garbage collection operations.
+			# TYPE cortex_compactor_garbage_collection_total counter
+			cortex_compactor_garbage_collection_total 555550
+
+			# HELP cortex_compactor_garbage_collection_failures_total TSDB Syncer: Total number of failed garbage collection operations.
+			# TYPE cortex_compactor_garbage_collection_failures_total counter
+			cortex_compactor_garbage_collection_failures_total 666660
+
+			# HELP cortex_compactor_garbage_collection_duration_seconds TSDB Syncer: Time it took to perform garbage collection iteration.
+			# TYPE cortex_compactor_garbage_collection_duration_seconds histogram
+			# Observed values: 8.6415, 53.5801, 15.5554
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="0.01"} 0
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="0.1"} 0
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="0.3"} 0
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="0.6"} 0
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="1"} 0
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="3"} 0
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="6"} 0
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="9"} 1
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="20"} 2
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="30"} 2
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="60"} 3
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="90"} 3
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="120"} 3
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="240"} 3
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="360"} 3
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="720"} 3
+			cortex_compactor_garbage_collection_duration_seconds_bucket{le="+Inf"} 3
+			cortex_compactor_garbage_collection_duration_seconds_sum 77.777
+			cortex_compactor_garbage_collection_duration_seconds_count 3
+
+			# HELP cortex_compactor_group_compactions_total TSDB Syncer: Total number of group compaction attempts that resulted in a new block.
+			# TYPE cortex_compactor_group_compactions_total counter
+			# Sum across all groups
+			cortex_compactor_group_compactions_total 2999970
+
+			# HELP cortex_compactor_group_compaction_runs_started_total TSDB Syncer: Total number of group compaction attempts.
+			# TYPE cortex_compactor_group_compaction_runs_started_total counter
+			# Sum across all groups
+			cortex_compactor_group_compaction_runs_started_total 3999960
+
+			# HELP cortex_compactor_group_compaction_runs_completed_total TSDB Syncer: Total number of group completed compaction runs. This also includes compactor group runs that resulted with no compaction.
+			# TYPE cortex_compactor_group_compaction_runs_completed_total counter
+			# Sum across all groups
+			cortex_compactor_group_compaction_runs_completed_total 4999950
+
+			# HELP cortex_compactor_group_compactions_failures_total TSDB Syncer: Total number of failed group compactions.
+			# TYPE cortex_compactor_group_compactions_failures_total counter
+			cortex_compactor_group_compactions_failures_total 5999940
+
+			# HELP cortex_compactor_group_vertical_compactions_total TSDB Syncer: Total number of group compaction attempts that resulted in a new block based on overlapping blocks.
+			# TYPE cortex_compactor_group_vertical_compactions_total counter
+			cortex_compactor_group_vertical_compactions_total 6999930
+	`))
+	require.NoError(t, err)
+}
+
+func generateTestData(base float64) *prometheus.Registry {
+	r := prometheus.NewRegistry()
+	m := newTestSyncerMetrics(r)
+	m.syncMetas.Add(1 * base)
+	m.syncMetaFailures.Add(2 * base)
+	m.syncMetaDuration.Observe(3 * base / 10000)
+	m.garbageCollectedBlocks.Add(4 * base)
+	m.garbageCollections.Add(5 * base)
+	m.garbageCollectionFailures.Add(6 * base)
+	m.garbageCollectionDuration.Observe(7 * base / 10000)
+	m.compactions.WithLabelValues("aaa").Add(8 * base)
+	m.compactions.WithLabelValues("bbb").Add(9 * base)
+	m.compactions.WithLabelValues("ccc").Add(10 * base)
+	m.compactionRunsStarted.WithLabelValues("aaa").Add(11 * base)
+	m.compactionRunsStarted.WithLabelValues("bbb").Add(12 * base)
+	m.compactionRunsStarted.WithLabelValues("ccc").Add(13 * base)
+	m.compactionRunsCompleted.WithLabelValues("aaa").Add(14 * base)
+	m.compactionRunsCompleted.WithLabelValues("bbb").Add(15 * base)
+	m.compactionRunsCompleted.WithLabelValues("ccc").Add(16 * base)
+	m.compactionFailures.WithLabelValues("aaa").Add(17 * base)
+	m.compactionFailures.WithLabelValues("bbb").Add(18 * base)
+	m.compactionFailures.WithLabelValues("ccc").Add(19 * base)
+	m.verticalCompactions.WithLabelValues("aaa").Add(20 * base)
+	m.verticalCompactions.WithLabelValues("bbb").Add(21 * base)
+	m.verticalCompactions.WithLabelValues("ccc").Add(22 * base)
+	return r
+}
+
+// directly copied from Thanos (and renamed syncerMetrics to testSyncerMetrics to avoid conflict)
+type testSyncerMetrics struct {
+	syncMetas                 prometheus.Counter
+	syncMetaFailures          prometheus.Counter
+	syncMetaDuration          prometheus.Histogram
+	garbageCollectedBlocks    prometheus.Counter
+	garbageCollections        prometheus.Counter
+	garbageCollectionFailures prometheus.Counter
+	garbageCollectionDuration prometheus.Histogram
+	compactions               *prometheus.CounterVec
+	compactionRunsStarted     *prometheus.CounterVec
+	compactionRunsCompleted   *prometheus.CounterVec
+	compactionFailures        *prometheus.CounterVec
+	verticalCompactions       *prometheus.CounterVec
+}
+
+func newTestSyncerMetrics(reg prometheus.Registerer) *testSyncerMetrics {
+	var m testSyncerMetrics
+
+	m.syncMetas = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "thanos_compact_sync_meta_total",
+		Help: "Total number of sync meta operations.",
+	})
+	m.syncMetaFailures = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "thanos_compact_sync_meta_failures_total",
+		Help: "Total number of failed sync meta operations.",
+	})
+	m.syncMetaDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    "thanos_compact_sync_meta_duration_seconds",
+		Help:    "Time it took to sync meta files.",
+		Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720},
+	})
+
+	m.garbageCollectedBlocks = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "thanos_compact_garbage_collected_blocks_total",
+		Help: "Total number of deleted blocks by compactor.",
+	})
+	m.garbageCollections = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "thanos_compact_garbage_collection_total",
+		Help: "Total number of garbage collection operations.",
+	})
+	m.garbageCollectionFailures = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "thanos_compact_garbage_collection_failures_total",
+		Help: "Total number of failed garbage collection operations.",
+	})
+	m.garbageCollectionDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    "thanos_compact_garbage_collection_duration_seconds",
+		Help:    "Time it took to perform garbage collection iteration.",
+		Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720},
+	})
+
+	m.compactions = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "thanos_compact_group_compactions_total",
+		Help: "Total number of group compaction attempts that resulted in a new block.",
+	}, []string{"group"})
+	m.compactionRunsStarted = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "thanos_compact_group_compaction_runs_started_total",
+		Help: "Total number of group compaction attempts.",
+	}, []string{"group"})
+	m.compactionRunsCompleted = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "thanos_compact_group_compaction_runs_completed_total",
+		Help: "Total number of group completed compaction runs. This also includes compactor group runs that resulted with no compaction.",
+	}, []string{"group"})
+	m.compactionFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "thanos_compact_group_compactions_failures_total",
+		Help: "Total number of failed group compactions.",
+	}, []string{"group"})
+	m.verticalCompactions = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "thanos_compact_group_vertical_compactions_total",
+		Help: "Total number of group compaction attempts that resulted in a new block based on overlapping blocks.",
+	}, []string{"group"})
+
+	if reg != nil {
+		reg.MustRegister(
+			m.syncMetas,
+			m.syncMetaFailures,
+			m.syncMetaDuration,
+			m.garbageCollectedBlocks,
+			m.garbageCollections,
+			m.garbageCollectionFailures,
+			m.garbageCollectionDuration,
+			m.compactions,
+			m.compactionRunsStarted,
+			m.compactionRunsCompleted,
+			m.compactionFailures,
+			m.verticalCompactions,
+		)
+	}
+	return &m
+}
diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go
index ee363f3c66..914be0f6a4 100644
--- a/pkg/querier/querier_test.go
+++ b/pkg/querier/querier_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"sync"
 	"testing"
 	"time"
 
@@ -149,45 +150,45 @@ func TestQuerier(t *testing.T) {
 
 func TestNoHistoricalQueryToIngester(t *testing.T) {
 	testCases := []struct {
-		name                     string
-		mint, maxt               time.Time
-		hitIngester              bool
-		ingesterMaxQueryLookback time.Duration
+		name                 string
+		mint, maxt           time.Time
+		hitIngester          bool
+		queryIngestersWithin time.Duration
 	}{
 		{
-			name:                     "hit-test1",
-			mint:                     time.Now().Add(-5 * time.Hour),
-			maxt:                     time.Now().Add(1 * time.Hour),
-			hitIngester:              true,
-			ingesterMaxQueryLookback: 1 * time.Hour,
+			name:                 "hit-test1",
+			mint:                 time.Now().Add(-5 * time.Hour),
+			maxt:                 time.Now().Add(1 * time.Hour),
+			hitIngester:          true,
+			queryIngestersWithin: 1 * time.Hour,
 		},
 		{
-			name:                     "hit-test2",
-			mint:                     time.Now().Add(-5 * time.Hour),
-			maxt:                     time.Now().Add(-59 * time.Minute),
-			hitIngester:              true,
-			ingesterMaxQueryLookback: 1 * time.Hour,
+			name:                 "hit-test2",
+			mint:                 time.Now().Add(-5 * time.Hour),
+			maxt:                 time.Now().Add(-59 * time.Minute),
+			hitIngester:          true,
+			queryIngestersWithin: 1 * time.Hour,
 		},
 		{ // Skipping ingester is disabled.
-			name:                     "hit-test2",
-			mint:                     time.Now().Add(-5 * time.Hour),
-			maxt:                     time.Now().Add(-50 * time.Minute),
-			hitIngester:              true,
-			ingesterMaxQueryLookback: 0,
+			name:                 "hit-test2",
+			mint:                 time.Now().Add(-5 * time.Hour),
+			maxt:                 time.Now().Add(-50 * time.Minute),
+			hitIngester:          true,
+			queryIngestersWithin: 0,
 		},
 		{
-			name:                     "dont-hit-test1",
-			mint:                     time.Now().Add(-5 * time.Hour),
-			maxt:                     time.Now().Add(-100 * time.Minute),
-			hitIngester:              false,
-			ingesterMaxQueryLookback: 1 * time.Hour,
+			name:                 "dont-hit-test1",
+			mint:                 time.Now().Add(-5 * time.Hour),
+			maxt:                 time.Now().Add(-100 * time.Minute),
+			hitIngester:          false,
+			queryIngestersWithin: 1 * time.Hour,
 		},
 		{
-			name:                     "dont-hit-test2",
-			mint:                     time.Now().Add(-5 * time.Hour),
-			maxt:                     time.Now().Add(-61 * time.Minute),
-			hitIngester:              false,
-			ingesterMaxQueryLookback: 1 * time.Hour,
+			name:                 "dont-hit-test2",
+			mint:                 time.Now().Add(-5 * time.Hour),
+			maxt:                 time.Now().Add(-61 * time.Minute),
+			hitIngester:          false,
+			queryIngestersWithin: 1 * time.Hour,
 		},
 	}
 
@@ -201,7 +202,7 @@ func TestNoHistoricalQueryToIngester(t *testing.T) {
 	for _, ingesterStreaming := range []bool{true, false} {
 		cfg.IngesterStreaming = ingesterStreaming
 		for _, c := range testCases {
-			cfg.IngesterMaxQueryLookback = c.ingesterMaxQueryLookback
+			cfg.QueryIngestersWithin = c.queryIngestersWithin
 			t.Run(fmt.Sprintf("IngesterStreaming=%t,test=%s", cfg.IngesterStreaming, c.name), func(t *testing.T) {
 				chunkStore, _ := makeMockChunkStore(t, 24, encodings[0].e)
 				distributor := &errDistributor{}
@@ -300,3 +301,101 @@ func (m *errDistributor) LabelNames(context.Context) ([]string, error) {
 func (m *errDistributor) MetricsForLabelMatchers(ctx context.Context, from, through model.Time, matchers ...*labels.Matcher) ([]metric.Metric, error) {
 	return nil, errDistributorError
 }
+
+type emptyChunkStore struct {
+	sync.Mutex
+	called bool
+}
+
+func (c *emptyChunkStore) Get(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) ([]chunk.Chunk, error) {
+	c.Lock()
+	defer c.Unlock()
+	c.called = true
+	return nil, nil
+}
+
+func (c *emptyChunkStore) IsCalled() bool {
+	c.Lock()
+	defer c.Unlock()
+	return c.called
+}
+
+func TestShortTermQueryToLTS(t *testing.T) {
+	testCases := []struct {
+		name                 string
+		mint, maxt           time.Time
+		hitIngester          bool
+		hitLTS               bool
+		queryIngestersWithin time.Duration
+		queryStoreAfter      time.Duration
+	}{
+		{
+			name:                 "hit only ingester",
+			mint:                 time.Now().Add(-5 * time.Minute),
+			maxt:                 time.Now(),
+			hitIngester:          true,
+			hitLTS:               false,
+			queryIngestersWithin: 1 * time.Hour,
+			queryStoreAfter:      time.Hour,
+		},
+		{
+			name:                 "hit both",
+			mint:                 time.Now().Add(-5 * time.Hour),
+			maxt:                 time.Now(),
+			hitIngester:          true,
+			hitLTS:               true,
+			queryIngestersWithin: 1 * time.Hour,
+			queryStoreAfter:      time.Hour,
+		},
+		{
+			name:                 "hit only storage",
+			mint:                 time.Now().Add(-5 * time.Hour),
+			maxt:                 time.Now().Add(-2 * time.Hour),
+			hitIngester:          false,
+			hitLTS:               true,
+			queryIngestersWithin: 1 * time.Hour,
+			queryStoreAfter:      0,
+		},
+	}
+
+	engine := promql.NewEngine(promql.EngineOpts{
+		Logger:        util.Logger,
+		MaxConcurrent: 10,
+		MaxSamples:    1e6,
+		Timeout:       1 * time.Minute,
+	})
+	cfg := Config{}
+	for _, ingesterStreaming := range []bool{true, false} {
+		cfg.IngesterStreaming = ingesterStreaming
+		for _, c := range testCases {
+			cfg.QueryIngestersWithin = c.queryIngestersWithin
+			cfg.QueryStoreAfter = c.queryStoreAfter
+			t.Run(fmt.Sprintf("IngesterStreaming=%t,test=%s", cfg.IngesterStreaming, c.name), func(t *testing.T) {
+				chunkStore := &emptyChunkStore{}
+				distributor := &errDistributor{}
+
+				queryable, _ := New(cfg, distributor, chunkStore)
+				query, err := engine.NewRangeQuery(queryable, "dummy", c.mint, c.maxt, 1*time.Minute)
+				require.NoError(t, err)
+
+				ctx := user.InjectOrgID(context.Background(), "0")
+				r := query.Exec(ctx)
+				_, err = r.Matrix()
+
+				if c.hitIngester {
+					// If the ingester was hit, the distributor always returns errDistributorError.
+					require.Error(t, err)
+					require.Equal(t, errDistributorError.Error(), err.Error())
+				} else {
+					// If the ingester was hit, there would have been an error from errDistributor.
+					require.NoError(t, err)
+				}
+
+				// Verify if the test did/did not hit the LTS
+				time.Sleep(30 * time.Millisecond) // NOTE: Since this is a lazy querier there is a race condition between the response and chunk store being called
+				require.Equal(t, c.hitLTS, chunkStore.IsCalled())
+			})
+		}
+	}
+
+}
diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go
index 3f27560488..cf41e1c842 100644
--- a/pkg/ring/lifecycler_test.go
+++ b/pkg/ring/lifecycler_test.go
@@ -43,92 +43,12 @@ func testLifecyclerConfig(ringConfig Config, id string) LifecyclerConfig {
 	return lifecyclerConfig
 }
 
-func checkDenormalisedLeaving(d interface{}, id string) bool {
-	desc, ok := d.(*Desc)
-	return ok &&
-		len(desc.Ingesters) == 1 &&
-		desc.Ingesters[id].State == LEAVING &&
-		len(desc.Ingesters[id].Tokens) == 0 &&
-		len(desc.Tokens) == 1
-}
-
 func checkNormalised(d interface{}, id string) bool {
 	desc, ok := d.(*Desc)
 	return ok &&
 		len(desc.Ingesters) == 1 &&
 		desc.Ingesters[id].State == ACTIVE &&
-		len(desc.Ingesters[id].Tokens) == 1 &&
-		len(desc.Tokens) == 0
-}
-
-func TestRingNormaliseMigration(t *testing.T) {
-	var ringConfig Config
-	flagext.DefaultValues(&ringConfig)
-	ringConfig.KVStore.Mock = consul.NewInMemoryClient(GetCodec())
-
-	r, err := New(ringConfig, "ingester", IngesterRingKey)
-	require.NoError(t, err)
-	defer r.Stop()
-
-	// Add an 'ingester' with denormalised tokens.
-	lifecyclerConfig1 := testLifecyclerConfig(ringConfig, "ing1")
-
-	// Since code to insert ingester with denormalised tokens into ring was removed,
-	// instead of running lifecycler, we do it manually here.
-	token := uint32(0)
-	err = r.KVClient.CAS(context.Background(), IngesterRingKey, func(in interface{}) (out interface{}, retry bool, err error) {
-		require.Nil(t, in)
-		r := NewDesc()
-		tks := GenerateTokens(lifecyclerConfig1.NumTokens, nil)
-		r.Ingesters[lifecyclerConfig1.ID] = IngesterDesc{
-			Addr:      lifecyclerConfig1.Addr,
-			Timestamp: time.Now().Unix(),
-			State:     LEAVING, // expected by second ingester`
-		}
-		for _, t := range tks {
-			r.Tokens = append(r.Tokens, TokenDesc{
-				Token:    t,
-				Ingester: lifecyclerConfig1.ID,
-			})
-		}
-		token = tks[0]
-		return r, true, nil
-	})
-	require.NoError(t, err)
-
-	// Check this ingester joined, is active, and has one token.
-	test.Poll(t, 1000*time.Millisecond, true, func() interface{} {
-		d, err := r.KVClient.Get(context.Background(), IngesterRingKey)
-		require.NoError(t, err)
-		return checkDenormalisedLeaving(d, "ing1")
-	})
-
-	// Add a second ingester with normalised tokens.
-	var lifecyclerConfig2 = testLifecyclerConfig(ringConfig, "ing2")
-	lifecyclerConfig2.JoinAfter = 100 * time.Second
-
-	l2, err := NewLifecycler(lifecyclerConfig2, &flushTransferer{}, "ingester", IngesterRingKey, true)
-	require.NoError(t, err)
-	l2.Start()
-
-	// Since there is nothing that would make l2 to claim tokens from l1 (normally done on transfer)
-	// we do it manually.
-	require.NoError(t, l2.ClaimTokensFor(context.Background(), "ing1"))
-	require.NoError(t, l2.ChangeState(context.Background(), ACTIVE))
-
-	// Check the new ingester joined, has the same token, and is active.
-	test.Poll(t, 1000*time.Millisecond, true, func() interface{} {
-		d, err := r.KVClient.Get(context.Background(), IngesterRingKey)
-		require.NoError(t, err)
-
-		if desc, ok := d.(*Desc); ok {
-			// lifecycler for ingester 1 isn't running, so we need to delete it manually
-			// (to make checkNormalised happy)
-			delete(desc.Ingesters, lifecyclerConfig1.ID)
-		}
-		return checkNormalised(d, "ing2") &&
-			d.(*Desc).Ingesters["ing2"].Tokens[0] == token
-	})
+		len(desc.Ingesters[id].Tokens) == 1
 }
 
 func TestLifecycler_HealthyInstancesCount(t *testing.T) {
@@ -381,8 +301,7 @@ func TestTokensOnDisk(t *testing.T) {
 		return ok &&
 			len(desc.Ingesters) == 1 &&
 			desc.Ingesters["ing1"].State == ACTIVE &&
-			len(desc.Ingesters["ing1"].Tokens) == 512 &&
-			len(desc.Tokens) == 0
+			len(desc.Ingesters["ing1"].Tokens) == 512
 	})
 
 	l1.Shutdown()
@@ -406,8 +325,7 @@ func TestTokensOnDisk(t *testing.T) {
 		return ok &&
 			len(desc.Ingesters) == 1 &&
 			desc.Ingesters["ing2"].State == ACTIVE &&
-			len(desc.Ingesters["ing2"].Tokens) == 512 &&
-			len(desc.Tokens) == 0
+			len(desc.Ingesters["ing2"].Tokens) == 512
 	})
 
 	// Check for same tokens.
@@ -441,15 +359,8 @@ func TestJoinInLeavingState(t *testing.T) {
 					State:  LEAVING,
 					Tokens: []uint32{1, 4},
 				},
-			},
-			Tokens: []TokenDesc{
-				{
-					Ingester: "ing2",
-					Token:    2,
-				},
-				{
-					Ingester: "ing2",
-					Token:    3,
+				"ing2": {
+					Tokens: []uint32{2, 3},
 				},
 			},
 		}
@@ -468,9 +379,9 @@ func TestJoinInLeavingState(t *testing.T) {
 		require.NoError(t, err)
 		desc, ok := d.(*Desc)
 		return ok &&
-			len(desc.Ingesters) == 1 &&
+			len(desc.Ingesters) == 2 &&
 			desc.Ingesters["ing1"].State == ACTIVE &&
 			len(desc.Ingesters["ing1"].Tokens) == cfg.NumTokens &&
-			len(desc.Tokens) == 2
+			len(desc.Ingesters["ing2"].Tokens) == 2
 	})
 }
diff --git a/pkg/ring/merge_test.go b/pkg/ring/merge_test.go
index 480e16331c..d050deb799 100644
--- a/pkg/ring/merge_test.go
+++ b/pkg/ring/merge_test.go
@@ -12,22 +12,11 @@ func TestNormalizationAndConflictResolution(t *testing.T) {
 
 	first := &Desc{
 		Ingesters: map[string]IngesterDesc{
-			"Ing 1": {Addr: "addr1", Timestamp: now, State: ACTIVE, Tokens: nil},
-			"Ing 2": {Addr: "addr2", Timestamp: 123456, State: LEAVING, Tokens: []uint32{100, 5, 5, 100, 100, 200}},
-			"Ing 3": {Addr: "addr3", Timestamp: now, State: LEFT, Tokens: []uint32{100, 200, 300}},
-			"Ing 4": {Addr: "addr4", Timestamp: now, State: LEAVING, Tokens: []uint32{30, 40, 50}},
-		},
-
-		Tokens: []TokenDesc{
-			{Token: 50, Ingester: "Ing 1"},
-			{Token: 40, Ingester: "Ing 1"},
-			{Token: 40, Ingester: "Ing 1"}, // dup
-			{Token: 30, Ingester: "Ing 1"},
-			{Token: 20, Ingester: "Ing 2"},
-			{Token: 10, Ingester: "Ing 2"},
-			{Token: 100, Ingester: "Ing 3"}, // LEFT, will be ignored
-			{Token: 200, Ingester: "Ing 3"},
-			{Token: 100, Ingester: "Unknown"},
+			"Ing 1":   {Addr: "addr1", Timestamp: now, State: ACTIVE, Tokens: []uint32{50, 40, 40, 30}},
+			"Ing 2":   {Addr: "addr2", Timestamp: 123456, State: LEAVING, Tokens: []uint32{100, 5, 5, 100, 100, 200, 20, 10}},
+			"Ing 3":   {Addr: "addr3", Timestamp: now, State: LEFT, Tokens: []uint32{100, 200, 300}},
+			"Ing 4":   {Addr: "addr4", Timestamp: now, State: LEAVING, Tokens: []uint32{30, 40, 50}},
+			"Unknown": {Tokens: []uint32{100}},
 		},
 	}
 
@@ -35,12 +24,9 @@ func TestNormalizationAndConflictResolution(t *testing.T) {
 		Ingesters: map[string]IngesterDesc{
 			"Unknown": {
 				Timestamp: now + 10,
+				Tokens:    []uint32{1000, 2000},
 			},
 		},
-		Tokens: []TokenDesc{
-			{Token: 1000, Ingester: "Unknown"},
-			{Token: 2000, Ingester: "Unknown"},
-		},
 	}
 
 	change, err := first.Merge(second, false)
@@ -60,19 +46,6 @@ func TestNormalizationAndConflictResolution(t *testing.T) {
 			"Ing 4":   {Addr: "addr4", Timestamp: now, State: LEAVING},
 			"Unknown": {Timestamp: now + 10, Tokens: []uint32{1000, 2000}},
 		},
-		// // Since the ring wasn't normalized before the merge, it will be denormalized after the merge
-		// Tokens: []TokenDesc{
-		// 	{5, "Ing 2"},
-		// 	{10, "Ing 2"},
-		// 	{20, "Ing 2"},
-		// 	{30, "Ing 1"},
-		// 	{40, "Ing 1"},
-		// 	{50, "Ing 1"},
-		// 	{100, "Ing 2"},
-		// 	{200, "Ing 2"},
-		// 	{1000, "Unknown"},
-		// 	{2000, "Unknown"},
-		// },
 	}, first)
 
 	assert.Equal(t, &Desc{
@@ -80,7 +53,6 @@ func TestNormalizationAndConflictResolution(t *testing.T) {
 		Ingesters: map[string]IngesterDesc{
 			"Unknown": {Timestamp: now + 10, Tokens: []uint32{1000, 2000}},
 		},
-		Tokens: nil,
 	}, changeRing)
 }
 
@@ -107,7 +79,6 @@ func TestMerge(t *testing.T) {
 				"Ing 1": {Addr: "addr1", Timestamp: now, State: ACTIVE, Tokens: []uint32{30, 40, 50}},
 				"Ing 2": {Addr: "addr2", Timestamp: now, State: JOINING, Tokens: []uint32{5, 10, 20, 100, 200}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -117,7 +88,6 @@ func TestMerge(t *testing.T) {
 				"Ing 3": {Addr: "addr3", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{150, 250, 350}},
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -127,7 +97,6 @@ func TestMerge(t *testing.T) {
 				"Ing 1": {Addr: "addr1", Timestamp: now + 10, State: LEAVING, Tokens: []uint32{30, 40, 50}},
 				"Ing 3": {Addr: "addr3", Timestamp: now + 10, State: ACTIVE, Tokens: []uint32{150, 250, 350}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -138,7 +107,6 @@ func TestMerge(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 				"Ing 3": {Addr: "addr3", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{150, 250, 350}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -149,7 +117,6 @@ func TestMerge(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 				"Ing 3": {Addr: "addr3", Timestamp: now + 10, State: ACTIVE, Tokens: []uint32{150, 250, 350}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -158,7 +125,6 @@ func TestMerge(t *testing.T) {
 			Ingesters: map[string]IngesterDesc{
 				"Ing 1": {Addr: "addr1", Timestamp: now + 10, State: LEFT, Tokens: []uint32{30, 40, 50}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -169,7 +135,6 @@ func TestMerge(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 				"Ing 3": {Addr: "addr3", Timestamp: now + 10, State: ACTIVE, Tokens: []uint32{150, 250, 350}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -193,7 +158,6 @@ func TestMerge(t *testing.T) {
 			Ingesters: map[string]IngesterDesc{
 				"Ing 1": {Addr: "addr1", Timestamp: now, State: ACTIVE, Tokens: []uint32{30, 40, 50}},
 			},
-			Tokens: nil,
 		}, ch)
 	}
 
@@ -215,7 +179,6 @@ func TestMerge(t *testing.T) {
 			Ingesters: map[string]IngesterDesc{
 				"Ing 1": {Addr: "addr1", Timestamp: now + 10, State: LEFT, Tokens: nil},
 			},
-			Tokens: nil,
 		}, ch)
 	}
 }
@@ -229,7 +192,6 @@ func TestTokensTakeover(t *testing.T) {
 				"Ing 1": {Addr: "addr1", Timestamp: now, State: ACTIVE, Tokens: []uint32{30, 40, 50}},
 				"Ing 2": {Addr: "addr2", Timestamp: now, State: JOINING, Tokens: []uint32{5, 10, 20}}, // partially migrated from Ing 3
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -239,7 +201,6 @@ func TestTokensTakeover(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20}},
 				"Ing 3": {Addr: "addr3", Timestamp: now + 5, State: LEAVING, Tokens: []uint32{5, 10, 20, 100, 200}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -250,7 +211,6 @@ func TestTokensTakeover(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20}},
 				"Ing 3": {Addr: "addr3", Timestamp: now + 5, State: LEAVING, Tokens: []uint32{100, 200}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -293,7 +253,6 @@ func TestMergeLeft(t *testing.T) {
 				"Ing 1": {Addr: "addr1", Timestamp: now, State: ACTIVE, Tokens: []uint32{30, 40, 50}},
 				"Ing 2": {Addr: "addr2", Timestamp: now, State: JOINING, Tokens: []uint32{5, 10, 20, 100, 200}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -302,7 +261,6 @@ func TestMergeLeft(t *testing.T) {
 			Ingesters: map[string]IngesterDesc{
 				"Ing 2": {Addr: "addr2", Timestamp: now, State: LEFT, Tokens: []uint32{5, 10, 20, 100, 200}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -312,7 +270,6 @@ func TestMergeLeft(t *testing.T) {
 				"Ing 1": {Addr: "addr1", Timestamp: now, State: ACTIVE, Tokens: []uint32{30, 40, 50}},
 				"Ing 2": {Addr: "addr2", Timestamp: now, State: LEFT},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -322,7 +279,6 @@ func TestMergeLeft(t *testing.T) {
 				"Ing 1": {Addr: "addr1", Timestamp: now + 10, State: LEAVING, Tokens: []uint32{30, 40, 50}},
 				"Ing 2": {Addr: "addr2", Timestamp: now, State: JOINING, Tokens: []uint32{5, 10, 20, 100, 200}}, // from firstRing
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -332,7 +288,6 @@ func TestMergeLeft(t *testing.T) {
 				"Ing 1": {Addr: "addr1", Timestamp: now + 10, State: LEAVING, Tokens: []uint32{30, 40, 50}},
 				"Ing 2": {Addr: "addr2", Timestamp: now, State: LEFT},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -343,7 +298,6 @@ func TestMergeLeft(t *testing.T) {
 			Ingesters: map[string]IngesterDesc{
 				"Ing 2": {Addr: "addr2", Timestamp: now, State: LEFT},
 			},
-			Tokens: nil,
 		}, ch)
 	}
 
@@ -361,7 +315,6 @@ func TestMergeLeft(t *testing.T) {
 			Ingesters: map[string]IngesterDesc{
 				"Ing 1": {Addr: "addr1", Timestamp: now, State: ACTIVE, Tokens: []uint32{30, 40, 50}},
 			},
-			Tokens: nil,
 		}, ch)
 	}
 
@@ -386,7 +339,6 @@ func TestMergeRemoveMissing(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now, State: JOINING, Tokens: []uint32{5, 10, 20, 100, 200}},
 				"Ing 3": {Addr: "addr3", Timestamp: now, State: LEAVING, Tokens: []uint32{5, 10, 20, 100, 200}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -396,7 +348,6 @@ func TestMergeRemoveMissing(t *testing.T) {
 				"Ing 1": {Addr: "addr1", Timestamp: now, State: ACTIVE, Tokens: []uint32{30, 40, 50}},
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -407,7 +358,6 @@ func TestMergeRemoveMissing(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 				"Ing 3": {Addr: "addr3", Timestamp: now, State: LEFT},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -419,7 +369,6 @@ func TestMergeRemoveMissing(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 				"Ing 3": {Addr: "addr3", Timestamp: now, State: LEFT},
 			},
-			Tokens: nil,
 		}, ch) // entire second ring is new
 	}
 
@@ -437,14 +386,12 @@ func TestMergeRemoveMissing(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 				"Ing 3": {Addr: "addr3", Timestamp: now, State: LEAVING},
 			},
-			Tokens: nil,
 		}, our)
 
 		assert.Equal(t, &Desc{
 			Ingesters: map[string]IngesterDesc{
 				"Ing 3": {Addr: "addr3", Timestamp: now, State: LEAVING},
 			},
-			Tokens: nil,
 		}, ch)
 	}
 }
@@ -459,7 +406,6 @@ func TestMergeMissingIntoLeft(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 5, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 				"Ing 3": {Addr: "addr3", Timestamp: now, State: LEFT},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -469,7 +415,6 @@ func TestMergeMissingIntoLeft(t *testing.T) {
 				"Ing 1": {Addr: "addr1", Timestamp: now + 10, State: ACTIVE, Tokens: []uint32{30, 40, 50}},
 				"Ing 2": {Addr: "addr2", Timestamp: now + 10, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 			},
-			Tokens: nil,
 		}
 	}
 
@@ -481,7 +426,6 @@ func TestMergeMissingIntoLeft(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 10, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 				"Ing 3": {Addr: "addr3", Timestamp: now, State: LEFT},
 			},
-			Tokens: nil,
 		}, our)
 
 		assert.Equal(t, &Desc{
@@ -490,7 +434,6 @@ func TestMergeMissingIntoLeft(t *testing.T) {
 				"Ing 2": {Addr: "addr2", Timestamp: now + 10, State: ACTIVE, Tokens: []uint32{5, 10, 20, 100, 200}},
 				// Ing 3 is not changed, it was already LEFT
 			},
-			Tokens: nil,
 		}, ch)
 	}
 }
diff --git a/pkg/ring/model_test.go b/pkg/ring/model_test.go
index 7ad1ef8a3e..80588fc331 100644
--- a/pkg/ring/model_test.go
+++ b/pkg/ring/model_test.go
@@ -72,25 +72,12 @@ func normalizedSource() *Desc {
 	return r
 }
 
-func unnormalizedSource() *Desc {
-	r := NewDesc()
-	r.Ingesters["first"] = IngesterDesc{}
-	r.Ingesters["second"] = IngesterDesc{}
-	r.Tokens = []TokenDesc{
-		{Token: 100, Ingester: "first"},
-		{Token: 200, Ingester: "first"},
-		{Token: 300, Ingester: "first"},
-	}
-	return r
-}
-
 func normalizedOutput() *Desc {
 	return &Desc{
 		Ingesters: map[string]IngesterDesc{
 			"first":  {},
 			"second": {Tokens: []uint32{100, 200, 300}},
 		},
-		Tokens: nil,
 	}
 }
 
@@ -102,15 +89,6 @@ func TestClaimTokensFromNormalizedToNormalized(t *testing.T) {
 	assert.Equal(t, normalizedOutput(), r)
 }
 
-func TestClaimTokensFromUnnormalizedToNormalized(t *testing.T) {
-	r := unnormalizedSource()
-
-	result := r.ClaimTokens("first", "second")
-
-	assert.Equal(t, Tokens{100, 200, 300}, result)
-	assert.Equal(t, normalizedOutput(), r)
-}
-
 func TestReady(t *testing.T) {
 	now := time.Now()
 
@@ -145,8 +123,9 @@ func TestReady(t *testing.T) {
 		t.Fatal("expected !ready (no tokens), but got no error")
 	}
 
-	r.Tokens = []TokenDesc{
-		{Token: 12345, Ingester: "some ingester"},
+	r.Ingesters["some ingester"] = IngesterDesc{
+		Tokens:    []uint32{12345},
+		Timestamp: now.Unix(),
 	}
 
 	if err := r.Ready(now, 10*time.Second); err != nil {
diff --git a/pkg/ring/ring_test.go b/pkg/ring/ring_test.go
index cf80ab94a8..f850c7a327 100644
--- a/pkg/ring/ring_test.go
+++ b/pkg/ring/ring_test.go
@@ -92,32 +92,14 @@ func TestDoBatchZeroIngesters(t *testing.T) {
 func TestAddIngester(t *testing.T) {
 	r := NewDesc()
 
-	const (
-		ing1Name = "ing1"
-		ing2Name = "ing2"
-	)
+	const ingName = "ing1"
 
 	ing1Tokens := GenerateTokens(128, nil)
-	ing2Tokens := GenerateTokens(128, ing1Tokens)
 
-	// store tokens to r.Tokens
-	for _, t := range ing1Tokens {
-		r.Tokens = append(r.Tokens, TokenDesc{Token: t, Ingester: ing1Name})
-	}
-
-	for _, t := range ing2Tokens {
-		r.Tokens = append(r.Tokens, TokenDesc{Token: t, Ingester: ing2Name})
-	}
+	r.AddIngester(ingName, "addr", ing1Tokens, ACTIVE)
 
-	r.AddIngester(ing1Name, "addr", ing1Tokens, ACTIVE)
-
-	require.Equal(t, "addr", r.Ingesters[ing1Name].Addr)
-	require.Equal(t, ing1Tokens, r.Ingesters[ing1Name].Tokens)
-
-	require.Equal(t, len(ing2Tokens), len(r.Tokens))
-	for _, tok := range r.Tokens {
-		require.NotEqual(t, "test", tok.Ingester)
-	}
+	require.Equal(t, "addr", r.Ingesters[ingName].Addr)
+	require.Equal(t, ing1Tokens, r.Ingesters[ingName].Tokens)
 }
 
 func TestAddIngesterReplacesExistingTokens(t *testing.T) {
@@ -125,16 +107,14 @@ func TestAddIngesterReplacesExistingTokens(t *testing.T) {
 
 	const ing1Name = "ing1"
 
-	oldTokens := []uint32{11111, 22222, 33333}
 	// old tokens will be replaced
-	for _, t := range oldTokens {
-		r.Tokens = append(r.Tokens, TokenDesc{Token: t, Ingester: ing1Name})
+	r.Ingesters[ing1Name] = IngesterDesc{
+		Tokens: []uint32{11111, 22222, 33333},
 	}
 
-	newTokens := GenerateTokens(128, oldTokens)
+	newTokens := GenerateTokens(128, nil)
 
 	r.AddIngester(ing1Name, "addr", newTokens, ACTIVE)
 
 	require.Equal(t, newTokens, r.Ingesters[ing1Name].Tokens)
-	require.Equal(t, 0, len(r.Tokens)) // all previous tokens were removed
 }
diff --git a/pkg/ring/testutils/testutils.go b/pkg/ring/testutils/testutils.go
index c043187ee8..d3acf462d3 100644
--- a/pkg/ring/testutils/testutils.go
+++ b/pkg/ring/testutils/testutils.go
@@ -17,12 +17,6 @@ func NumTokens(c kv.Client, name, ringKey string) int {
 		level.Error(util.Logger).Log("msg", "error reading consul", "err", err)
 		return 0
 	}
-	count := 0
 	rd := ringDesc.(*ring.Desc)
-	for _, token := range rd.Tokens {
-		if token.Ingester == name {
-			count++
-		}
-	}
-	return count + len(rd.Ingesters[name].Tokens)
+	return len(rd.Ingesters[name].Tokens)
 }
diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go
index 719e294d7c..6ebb8434bc 100644
--- a/pkg/ruler/ruler_test.go
+++ b/pkg/ruler/ruler_test.go
@@ -48,7 +48,7 @@ func newTestRuler(t *testing.T, cfg Config) *Ruler {
 		MaxConcurrent: 20,
 		Timeout:       2 * time.Minute,
 	})
-	queryable := querier.NewQueryable(nil, nil, nil, 0)
+	queryable := querier.NewQueryable(nil, nil, nil, querier.Config{})
 	ruler, err := NewRuler(cfg, engine, queryable, nil)
 	if err != nil {
 		t.Fatal(err)