Repository: prometheus/prometheus
Pull request #11827 (2023-01-09 11:25:22)

Add 'keep_firing_for' field to alerting rules
This commit adds a new 'keep_firing_for' field to Prometheus alerting rules. The 'keep_firing_for' field specifies the minimum amount of time that an alert should remain firing, even if the expression does not return any results.

This feature was discussed at a previous dev summit, and it was determined that a feature like this would be useful in order to allow the expression time to stabilize and prevent confusing resolved messages from being propagated through Alertmanager.

This approach is simpler than having two PromQL queries, as was sometimes discussed, and it should be easier to implement.

This commit does not include tests for the 'resolve_delay' field.  This is intentional, as the purpose of this commit is to gather comments on the proposed design of the 'resolve_delay' field before implementing tests. Once the design of the 'resolve_delay' field has been finalized, a follow-up commit will be submitted with tests.

See https://github.com/prometheus/prometheus/issues/11570

Signed-off-by: Julien Pivotto <roidelapluie@o11y.eu>

<!--
    Don't forget!

    - Please sign CNCF's Developer Certificate of Origin and sign-off your commits by adding the -s / --sign-off flag to `git commit`. See https://github.com/apps/dco for more information.

    - If the PR adds or changes a behaviour or fixes a bug of an exported API it would need a unit/e2e test.

    - Where possible use only exported APIs for tests to simplify the review and make it as close as possible to an actual library usage.

    - No tests are needed for internal implementation changes.

    - Performance improvements would need a benchmark test to prove it.

    - All exposed objects should have a comment.

    - All comments should start with a capital letter and end with a full stop.
 -->



Patch:
diff --git a/docs/configuration/recording_rules.md b/docs/configuration/recording_rules.md
index d70ffa0cb..eda0214b3 100644
--- a/docs/configuration/recording_rules.md
+++ b/docs/configuration/recording_rules.md
@@ -123,6 +123,10 @@ expr: <string>
 # Alerts which have not yet fired for long enough are considered pending.
 [ for: <duration> | default = 0s ]
 
+# How long an alert will continue firing after the condition that triggered it
+# has cleared.
+[ keep_firing_for: <duration> | default = 0s ]
+
 # Labels to add or overwrite for each alert.
 labels:
   [ <labelname>: <tmpl_string> ]
diff --git a/model/rulefmt/rulefmt.go b/model/rulefmt/rulefmt.go
index f1d5f3925..30b3face0 100644
--- a/model/rulefmt/rulefmt.go
+++ b/model/rulefmt/rulefmt.go
@@ -143,22 +143,24 @@ type RuleGroup struct {
 
 // Rule describes an alerting or recording rule.
 type Rule struct {
-	Record      string            `yaml:"record,omitempty"`
-	Alert       string            `yaml:"alert,omitempty"`
-	Expr        string            `yaml:"expr"`
-	For         model.Duration    `yaml:"for,omitempty"`
-	Labels      map[string]string `yaml:"labels,omitempty"`
-	Annotations map[string]string `yaml:"annotations,omitempty"`
+	Record        string            `yaml:"record,omitempty"`
+	Alert         string            `yaml:"alert,omitempty"`
+	Expr          string            `yaml:"expr"`
+	For           model.Duration    `yaml:"for,omitempty"`
+	KeepFiringFor model.Duration    `yaml:"keep_firing_for,omitempty"`
+	Labels        map[string]string `yaml:"labels,omitempty"`
+	Annotations   map[string]string `yaml:"annotations,omitempty"`
 }
 
 // RuleNode adds yaml.v3 layer to support line and column outputs for invalid rules.
 type RuleNode struct {
-	Record      yaml.Node         `yaml:"record,omitempty"`
-	Alert       yaml.Node         `yaml:"alert,omitempty"`
-	Expr        yaml.Node         `yaml:"expr"`
-	For         model.Duration    `yaml:"for,omitempty"`
-	Labels      map[string]string `yaml:"labels,omitempty"`
-	Annotations map[string]string `yaml:"annotations,omitempty"`
+	Record        yaml.Node         `yaml:"record,omitempty"`
+	Alert         yaml.Node         `yaml:"alert,omitempty"`
+	Expr          yaml.Node         `yaml:"expr"`
+	For           model.Duration    `yaml:"for,omitempty"`
+	KeepFiringFor model.Duration    `yaml:"keep_firing_for,omitempty"`
+	Labels        map[string]string `yaml:"labels,omitempty"`
+	Annotations   map[string]string `yaml:"annotations,omitempty"`
 }
 
 // Validate the rule and return a list of encountered errors.
@@ -208,6 +210,12 @@ func (r *RuleNode) Validate() (nodes []WrappedError) {
 				node: &r.Record,
 			})
 		}
+		if r.KeepFiringFor != 0 {
+			nodes = append(nodes, WrappedError{
+				err:  fmt.Errorf("invalid field 'keep_firing_for' in recording rule"),
+				node: &r.Record,
+			})
+		}
 		if !model.IsValidMetricName(model.LabelValue(r.Record.Value)) {
 			nodes = append(nodes, WrappedError{
 				err:  fmt.Errorf("invalid recording rule name: %s", r.Record.Value),
diff --git a/rules/alerting.go b/rules/alerting.go
index d45666266..a7c94f7a7 100644
--- a/rules/alerting.go
+++ b/rules/alerting.go
@@ -83,11 +83,12 @@ type Alert struct {
 	Value float64
 	// The interval during which the condition of this alert held true.
 	// ResolvedAt will be 0 to indicate a still active alert.
-	ActiveAt   time.Time
-	FiredAt    time.Time
-	ResolvedAt time.Time
-	LastSentAt time.Time
-	ValidUntil time.Time
+	ActiveAt        time.Time
+	FiredAt         time.Time
+	ResolvedAt      time.Time
+	LastSentAt      time.Time
+	ValidUntil      time.Time
+	KeepFiringSince time.Time
 }
 
 func (a *Alert) needsSending(ts time.Time, resendDelay time.Duration) bool {
@@ -112,6 +113,9 @@ type AlertingRule struct {
 	// The duration for which a labelset needs to persist in the expression
 	// output vector before an alert transitions from Pending to Firing state.
 	holdDuration time.Duration
+	// The amount of time that the alert should remain firing after the
+	// resolution.
+	keepFiringFor time.Duration
 	// Extra labels to attach to the resulting alert sample vectors.
 	labels labels.Labels
 	// Non-identifying key/value pairs.
@@ -142,7 +146,7 @@ type AlertingRule struct {
 
 // NewAlertingRule constructs a new AlertingRule.
 func NewAlertingRule(
-	name string, vec parser.Expr, hold time.Duration,
+	name string, vec parser.Expr, hold, keepFiringFor time.Duration,
 	labels, annotations, externalLabels labels.Labels, externalURL string,
 	restored bool, logger log.Logger,
 ) *AlertingRule {
@@ -152,6 +156,7 @@ func NewAlertingRule(
 		name:                name,
 		vector:              vec,
 		holdDuration:        hold,
+		keepFiringFor:       keepFiringFor,
 		labels:              labels,
 		annotations:         annotations,
 		externalLabels:      el,
@@ -201,6 +206,12 @@ func (r *AlertingRule) HoldDuration() time.Duration {
 	return r.holdDuration
 }
 
+// KeepFiringFor returns the duration an alerting rule should keep firing for
+// after resolution.
+func (r *AlertingRule) KeepFiringFor() time.Duration {
+	return r.keepFiringFor
+}
+
 // Labels returns the labels of the alerting rule.
 func (r *AlertingRule) Labels() labels.Labels {
 	return r.labels
@@ -404,16 +415,36 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc,
 	// Check if any pending alerts should be removed or fire now. Write out alert timeseries.
 	for fp, a := range r.active {
 		if _, ok := resultFPs[fp]; !ok {
+			// There is no firing alerts for this fingerprint. The alert is no
+			// longer firing.
+
+			// Use keepFiringFor value to determine if the alert should keep
+			// firing.
+			var keepFiring bool
+			if a.State == StateFiring && r.keepFiringFor > 0 {
+				if a.KeepFiringSince.IsZero() {
+					a.KeepFiringSince = ts
+				}
+				if ts.Sub(a.KeepFiringSince) < r.keepFiringFor {
+					keepFiring = true
+				}
+			}
+
 			// If the alert was previously firing, keep it around for a given
 			// retention time so it is reported as resolved to the AlertManager.
 			if a.State == StatePending || (!a.ResolvedAt.IsZero() && ts.Sub(a.ResolvedAt) > resolvedRetention) {
 				delete(r.active, fp)
 			}
-			if a.State != StateInactive {
+			if a.State != StateInactive && !keepFiring {
 				a.State = StateInactive
 				a.ResolvedAt = ts
 			}
-			continue
+			if !keepFiring {
+				continue
+			}
+		} else {
+			// The alert is firing, reset keepFiringSince.
+			a.KeepFiringSince = time.Time{}
 		}
 		numActivePending++
 
@@ -512,11 +543,12 @@ func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay
 
 func (r *AlertingRule) String() string {
 	ar := rulefmt.Rule{
-		Alert:       r.name,
-		Expr:        r.vector.String(),
-		For:         model.Duration(r.holdDuration),
-		Labels:      r.labels.Map(),
-		Annotations: r.annotations.Map(),
+		Alert:         r.name,
+		Expr:          r.vector.String(),
+		For:           model.Duration(r.holdDuration),
+		KeepFiringFor: model.Duration(r.keepFiringFor),
+		Labels:        r.labels.Map(),
+		Annotations:   r.annotations.Map(),
 	}
 
 	byt, err := yaml.Marshal(ar)
diff --git a/rules/manager.go b/rules/manager.go
index d1ad8afdc..6f6ce2cfe 100644
--- a/rules/manager.go
+++ b/rules/manager.go
@@ -1119,6 +1119,7 @@ func (m *Manager) LoadGroups(
 						r.Alert.Value,
 						expr,
 						time.Duration(r.For),
+						time.Duration(r.KeepFiringFor),
 						labels.FromMap(r.Labels),
 						labels.FromMap(r.Annotations),
 						externalLabels,
diff --git a/storage/remote/codec.go b/storage/remote/codec.go
index 10f24efec..023780b48 100644
--- a/storage/remote/codec.go
+++ b/storage/remote/codec.go
@@ -525,7 +525,7 @@ func exemplarProtoToExemplar(ep prompb.Exemplar) exemplar.Exemplar {
 
 // HistogramProtoToHistogram extracts a (normal integer) Histogram from the
 // provided proto message. The caller has to make sure that the proto message
-// represents an integer histogram and not a float histogram.
+// represents an interger histogram and not a float histogram.
 func HistogramProtoToHistogram(hp prompb.Histogram) *histogram.Histogram {
 	return &histogram.Histogram{
 		Schema:          hp.Schema,
@@ -540,23 +540,6 @@ func HistogramProtoToHistogram(hp prompb.Histogram) *histogram.Histogram {
 	}
 }
 
-// HistogramProtoToFloatHistogram extracts a (normal integer) Histogram from the
-// provided proto message to a Float Histogram. The caller has to make sure that
-// the proto message represents an float histogram and not a integer histogram.
-func HistogramProtoToFloatHistogram(hp prompb.Histogram) *histogram.FloatHistogram {
-	return &histogram.FloatHistogram{
-		Schema:          hp.Schema,
-		ZeroThreshold:   hp.ZeroThreshold,
-		ZeroCount:       hp.GetZeroCountFloat(),
-		Count:           hp.GetCountFloat(),
-		Sum:             hp.Sum,
-		PositiveSpans:   spansProtoToSpans(hp.GetPositiveSpans()),
-		PositiveBuckets: hp.GetPositiveCounts(),
-		NegativeSpans:   spansProtoToSpans(hp.GetNegativeSpans()),
-		NegativeBuckets: hp.GetNegativeCounts(),
-	}
-}
-
 func spansProtoToSpans(s []*prompb.BucketSpan) []histogram.Span {
 	spans := make([]histogram.Span, len(s))
 	for i := 0; i < len(s); i++ {
@@ -581,21 +564,6 @@ func HistogramToHistogramProto(timestamp int64, h *histogram.Histogram) prompb.H
 	}
 }
 
-func FloatHistogramToHistogramProto(timestamp int64, fh *histogram.FloatHistogram) prompb.Histogram {
-	return prompb.Histogram{
-		Count:          &prompb.Histogram_CountFloat{CountFloat: fh.Count},
-		Sum:            fh.Sum,
-		Schema:         fh.Schema,
-		ZeroThreshold:  fh.ZeroThreshold,
-		ZeroCount:      &prompb.Histogram_ZeroCountFloat{ZeroCountFloat: fh.ZeroCount},
-		NegativeSpans:  spansToSpansProto(fh.NegativeSpans),
-		NegativeCounts: fh.NegativeBuckets,
-		PositiveSpans:  spansToSpansProto(fh.PositiveSpans),
-		PositiveCounts: fh.PositiveBuckets,
-		Timestamp:      timestamp,
-	}
-}
-
 func spansToSpansProto(s []histogram.Span) []*prompb.BucketSpan {
 	spans := make([]*prompb.BucketSpan, len(s))
 	for i := 0; i < len(s); i++ {
diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go
index 30c0750a1..f907615fc 100644
--- a/storage/remote/queue_manager.go
+++ b/storage/remote/queue_manager.go
@@ -716,53 +716,6 @@ outer:
 	return true
 }
 
-func (t *QueueManager) AppendFloatHistograms(floatHistograms []record.RefFloatHistogramSample) bool {
-	if !t.sendNativeHistograms {
-		return true
-	}
-
-outer:
-	for _, h := range floatHistograms {
-		t.seriesMtx.Lock()
-		lbls, ok := t.seriesLabels[h.Ref]
-		if !ok {
-			t.metrics.droppedHistogramsTotal.Inc()
-			t.dataDropped.incr(1)
-			if _, ok := t.droppedSeries[h.Ref]; !ok {
-				level.Info(t.logger).Log("msg", "Dropped histogram for series that was not explicitly dropped via relabelling", "ref", h.Ref)
-			}
-			t.seriesMtx.Unlock()
-			continue
-		}
-		t.seriesMtx.Unlock()
-
-		backoff := model.Duration(5 * time.Millisecond)
-		for {
-			select {
-			case <-t.quit:
-				return false
-			default:
-			}
-			if t.shards.enqueue(h.Ref, timeSeries{
-				seriesLabels:   lbls,
-				timestamp:      h.T,
-				floatHistogram: h.FH,
-				sType:          tFloatHistogram,
-			}) {
-				continue outer
-			}
-
-			t.metrics.enqueueRetriesTotal.Inc()
-			time.Sleep(time.Duration(backoff))
-			backoff = backoff * 2
-			if backoff > t.cfg.MaxBackoff {
-				backoff = t.cfg.MaxBackoff
-			}
-		}
-	}
-	return true
-}
-
 // Start the queue manager sending samples to the remote storage.
 // Does not block.
 func (t *QueueManager) Start() {
@@ -1176,7 +1129,7 @@ func (s *shards) enqueue(ref chunks.HeadSeriesRef, data timeSeries) bool {
 		case tExemplar:
 			s.qm.metrics.pendingExemplars.Inc()
 			s.enqueuedExemplars.Inc()
-		case tHistogram, tFloatHistogram:
+		case tHistogram:
 			s.qm.metrics.pendingHistograms.Inc()
 			s.enqueuedHistograms.Inc()
 		}
@@ -1201,7 +1154,6 @@ type timeSeries struct {
 	seriesLabels   labels.Labels
 	value          float64
 	histogram      *histogram.Histogram
-	floatHistogram *histogram.FloatHistogram
 	timestamp      int64
 	exemplarLabels labels.Labels
 	// The type of series: sample, exemplar, or histogram.
@@ -1214,7 +1166,6 @@ const (
 	tSample seriesType = iota
 	tExemplar
 	tHistogram
-	tFloatHistogram
 )
 
 func newQueue(batchSize, capacity int) *queue {
@@ -1402,8 +1353,7 @@ func (s *shards) runShard(ctx context.Context, shardID int, queue *queue) {
 			if len(batch) > 0 {
 				nPendingSamples, nPendingExemplars, nPendingHistograms := s.populateTimeSeries(batch, pendingData)
 				n := nPendingSamples + nPendingExemplars + nPendingHistograms
-				level.Debug(s.qm.logger).Log("msg", "runShard timer ticked, sending buffered data", "samples", nPendingSamples,
-					"exemplars", nPendingExemplars, "shard", shardNum, "histograms", nPendingHistograms)
+				level.Debug(s.qm.logger).Log("msg", "runShard timer ticked, sending buffered data", "samples", nPendingSamples, "exemplars", nPendingExemplars, "shard", shardNum)
 				s.sendSamples(ctx, pendingData[:n], nPendingSamples, nPendingExemplars, nPendingHistograms, pBuf, &buf)
 			}
 			queue.ReturnForReuse(batch)
@@ -1444,9 +1394,6 @@ func (s *shards) populateTimeSeries(batch []timeSeries, pendingData []prompb.Tim
 		case tHistogram:
 			pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, HistogramToHistogramProto(d.timestamp, d.histogram))
 			nPendingHistograms++
-		case tFloatHistogram:
-			pendingData[nPending].Histograms = append(pendingData[nPending].Histograms, FloatHistogramToHistogramProto(d.timestamp, d.floatHistogram))
-			nPendingHistograms++
 		}
 	}
 	return nPendingSamples, nPendingExemplars, nPendingHistograms
diff --git a/storage/remote/write_handler.go b/storage/remote/write_handler.go
index 45304c43c..4a8beeedc 100644
--- a/storage/remote/write_handler.go
+++ b/storage/remote/write_handler.go
@@ -124,20 +124,16 @@ func (h *writeHandler) write(ctx context.Context, req *prompb.WriteRequest) (err
 			}
 		}
 
+		// TODO(codesome): support float histograms.
 		for _, hp := range ts.Histograms {
-			if hp.GetCountFloat() > 0 || hp.GetZeroCountFloat() > 0 { // It is a float histogram.
-				fhs := HistogramProtoToFloatHistogram(hp)
-				_, err = app.AppendHistogram(0, labels, hp.Timestamp, nil, fhs)
-			} else {
-				hs := HistogramProtoToHistogram(hp)
-				_, err = app.AppendHistogram(0, labels, hp.Timestamp, hs, nil)
-			}
+			hs := HistogramProtoToHistogram(hp)
+			_, err = app.AppendHistogram(0, labels, hp.Timestamp, hs, nil)
 			if err != nil {
 				unwrappedErr := errors.Unwrap(err)
 				if unwrappedErr == nil {
 					unwrappedErr = err
 				}
-				// Although AppendHistogram does not currently return ErrDuplicateSampleForTimestamp there is
+				// Althogh AppendHistogram does not currently return ErrDuplicateSampleForTimestamp there is
 				// a note indicating its inclusion in the future.
 				if errors.Is(unwrappedErr, storage.ErrOutOfOrderSample) || errors.Is(unwrappedErr, storage.ErrOutOfBounds) || errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp) {
 					level.Error(h.logger).Log("msg", "Out of order histogram from remote write", "err", err.Error(), "series", labels.String(), "timestamp", hp.Timestamp)
diff --git a/tsdb/wlog/watcher.go b/tsdb/wlog/watcher.go
index 72121283d..5d7c84d34 100644
--- a/tsdb/wlog/watcher.go
+++ b/tsdb/wlog/watcher.go
@@ -50,7 +50,6 @@ type WriteTo interface {
 	Append([]record.RefSample) bool
 	AppendExemplars([]record.RefExemplar) bool
 	AppendHistograms([]record.RefHistogramSample) bool
-	AppendFloatHistograms([]record.RefFloatHistogramSample) bool
 	StoreSeries([]record.RefSeries, int)
 
 	// Next two methods are intended for garbage-collection: first we call
@@ -477,15 +476,13 @@ func (w *Watcher) garbageCollectSeries(segmentNum int) error {
 // Also used with readCheckpoint - implements segmentReadFn.
 func (w *Watcher) readSegment(r *LiveReader, segmentNum int, tail bool) error {
 	var (
-		dec                   record.Decoder
-		series                []record.RefSeries
-		samples               []record.RefSample
-		samplesToSend         []record.RefSample
-		exemplars             []record.RefExemplar
-		histograms            []record.RefHistogramSample
-		histogramsToSend      []record.RefHistogramSample
-		floatHistograms       []record.RefFloatHistogramSample
-		floatHistogramsToSend []record.RefFloatHistogramSample
+		dec              record.Decoder
+		series           []record.RefSeries
+		samples          []record.RefSample
+		samplesToSend    []record.RefSample
+		exemplars        []record.RefExemplar
+		histograms       []record.RefHistogramSample
+		histogramsToSend []record.RefHistogramSample
 	)
 	for r.Next() && !isClosed(w.quit) {
 		rec := r.Record()
@@ -570,33 +567,7 @@ func (w *Watcher) readSegment(r *LiveReader, segmentNum int, tail bool) error {
 				w.writer.AppendHistograms(histogramsToSend)
 				histogramsToSend = histogramsToSend[:0]
 			}
-		case record.FloatHistogramSamples:
-			// Skip if experimental "histograms over remote write" is not enabled.
-			if !w.sendHistograms {
-				break
-			}
-			if !tail {
-				break
-			}
-			floatHistograms, err := dec.FloatHistogramSamples(rec, floatHistograms[:0])
-			if err != nil {
-				w.recordDecodeFailsMetric.Inc()
-				return err
-			}
-			for _, fh := range floatHistograms {
-				if fh.T > w.startTimestamp {
-					if !w.sendSamples {
-						w.sendSamples = true
-						duration := time.Since(w.startTime)
-						level.Info(w.logger).Log("msg", "Done replaying WAL", "duration", duration)
-					}
-					floatHistogramsToSend = append(floatHistogramsToSend, fh)
-				}
-			}
-			if len(floatHistogramsToSend) > 0 {
-				w.writer.AppendFloatHistograms(floatHistogramsToSend)
-				floatHistogramsToSend = floatHistogramsToSend[:0]
-			}
+
 		case record.Tombstones:
 
 		default:
diff --git a/web/api/v1/api.go b/web/api/v1/api.go
index 894a8666a..f560693d4 100644
--- a/web/api/v1/api.go
+++ b/web/api/v1/api.go
@@ -1111,11 +1111,12 @@ type AlertDiscovery struct {
 
 // Alert has info for an alert.
 type Alert struct {
-	Labels      labels.Labels `json:"labels"`
-	Annotations labels.Labels `json:"annotations"`
-	State       string        `json:"state"`
-	ActiveAt    *time.Time    `json:"activeAt,omitempty"`
-	Value       string        `json:"value"`
+	Labels          labels.Labels `json:"labels"`
+	Annotations     labels.Labels `json:"annotations"`
+	State           string        `json:"state"`
+	ActiveAt        *time.Time    `json:"activeAt,omitempty"`
+	KeepFiringSince *time.Time    `json:"keepFiringSince,omitempty"`
+	Value           string        `json:"value"`
 }
 
 func (api *API) alerts(r *http.Request) apiFuncResult {
@@ -1144,6 +1145,9 @@ func rulesAlertsToAPIAlerts(rulesAlerts []*rules.Alert) []*Alert {
 			ActiveAt:    &ruleAlert.ActiveAt,
 			Value:       strconv.FormatFloat(ruleAlert.Value, 'e', -1, 64),
 		}
+		if !ruleAlert.KeepFiringSince.IsZero() {
+			apiAlerts[i].KeepFiringSince = &ruleAlert.KeepFiringSince
+		}
 	}
 
 	return apiAlerts
@@ -1241,6 +1245,7 @@ type AlertingRule struct {
 	Name           string           `json:"name"`
 	Query          string           `json:"query"`
 	Duration       float64          `json:"duration"`
+	KeepFiringFor  float64          `json:"keepFiringFor"`
 	Labels         labels.Labels    `json:"labels"`
 	Annotations    labels.Labels    `json:"annotations"`
 	Alerts         []*Alert         `json:"alerts"`
@@ -1303,6 +1308,7 @@ func (api *API) rules(r *http.Request) apiFuncResult {
 					Name:           rule.Name(),
 					Query:          rule.Query().String(),
 					Duration:       rule.HoldDuration().Seconds(),
+					KeepFiringFor:  rule.KeepFiringFor().Seconds(),
 					Labels:         rule.Labels(),
 					Annotations:    rule.Annotations(),
 					Alerts:         rulesAlertsToAPIAlerts(rule.ActiveAlerts()),
diff --git a/web/ui/react-app/src/pages/alerts/AlertContents.tsx b/web/ui/react-app/src/pages/alerts/AlertContents.tsx
index 2837399f7..a619f69fc 100644
--- a/web/ui/react-app/src/pages/alerts/AlertContents.tsx
+++ b/web/ui/react-app/src/pages/alerts/AlertContents.tsx
@@ -29,6 +29,7 @@ export interface Alert {
   value: string;
   annotations: Record<string, string>;
   activeAt: string;
+  keepFiringSince: string;
 }
 
 interface RuleGroup {
diff --git a/web/ui/react-app/src/pages/alerts/CollapsibleAlertPanel.tsx b/web/ui/react-app/src/pages/alerts/CollapsibleAlertPanel.tsx
index ef45c205f..676046efd 100644
--- a/web/ui/react-app/src/pages/alerts/CollapsibleAlertPanel.tsx
+++ b/web/ui/react-app/src/pages/alerts/CollapsibleAlertPanel.tsx
@@ -43,6 +43,11 @@ const CollapsibleAlertPanel: FC<CollapsibleAlertPanelProps> = ({ rule, showAnnot
                     <div>for: {formatDuration(rule.duration * 1000)}</div>
                   </div>
                 )}
+                {rule.keepFiringFor > 0 && (
+                  <div>
+                    <div>keep_firing_for: {formatDuration(rule.keepFiringFor * 1000)}</div>
+                  </div>
+                )}
                 {rule.labels && Object.keys(rule.labels).length > 0 && (
                   <div>
                     <div>labels:</div>
@@ -91,9 +96,14 @@ const CollapsibleAlertPanel: FC<CollapsibleAlertPanelProps> = ({ rule, showAnnot
                           </td>
                           <td>
                             <h5 className="m-0">
-                              <Badge color={alertColors[alert.state] + ' text-uppercase'} className="px-3">
+                              <Badge color={alertColors[alert.state] + ' text-uppercase'} className="px-3 mr-1">
                                 {alert.state}
                               </Badge>
+                              {alert.keepFiringSince && (
+                                <Badge color="secondary" className="px-3">
+                                  Stabilizing
+                                </Badge>
+                              )}
                             </h5>
                           </td>
                           <td>{alert.activeAt}</td>
diff --git a/web/ui/react-app/src/pages/rules/RulesContent.tsx b/web/ui/react-app/src/pages/rules/RulesContent.tsx
index e7adfee39..ef4a7ad8f 100644
--- a/web/ui/react-app/src/pages/rules/RulesContent.tsx
+++ b/web/ui/react-app/src/pages/rules/RulesContent.tsx
@@ -96,6 +96,11 @@ export const RulesContent: FC<RulesContentProps> = ({ response }) => {
                             <strong>for:</strong> {formatDuration(r.duration * 1000)}
                           </div>
                         )}
+                        {r.keepFiringFor > 0 && (
+                          <div>
+                            <strong>keep_firing_for:</strong> {formatDuration(r.keepFiringFor * 1000)}
+                          </div>
+                        )}
                         {r.labels && Object.keys(r.labels).length > 0 && (
                           <div>
                             <strong>labels:</strong>
diff --git a/web/ui/react-app/src/types/types.ts b/web/ui/react-app/src/types/types.ts
index 21f52b5fa..a30439c77 100644
--- a/web/ui/react-app/src/types/types.ts
+++ b/web/ui/react-app/src/types/types.ts
@@ -26,6 +26,7 @@ export type Rule = {
   alerts: Alert[];
   annotations: Record<string, string>;
   duration: number;
+  keepFiringFor: number;
   evaluationTime: string;
   health: string;
   labels: Record<string, string>;


Test patch:
diff --git a/model/rulefmt/rulefmt_test.go b/model/rulefmt/rulefmt_test.go
index d79ee94e2..d6499538e 100644
--- a/model/rulefmt/rulefmt_test.go
+++ b/model/rulefmt/rulefmt_test.go
@@ -73,12 +73,21 @@ func TestParseFileFailure(t *testing.T) {
 			filename: "invalid_label_name.bad.yaml",
 			errMsg:   "invalid label name",
 		},
+		{
+			filename: "record_and_for.bad.yaml",
+			errMsg:   "invalid field 'for' in recording rule",
+		},
+		{
+			filename: "record_and_keep_firing_for.bad.yaml",
+			errMsg:   "invalid field 'keep_firing_for' in recording rule",
+		},
 	}
 
 	for _, c := range table {
 		_, errs := ParseFile(filepath.Join("testdata", c.filename))
 		require.NotNil(t, errs, "Expected error parsing %s but got none", c.filename)
-		require.Error(t, errs[0], c.errMsg, "Expected error for %s.", c.filename)
+		require.Error(t, errs[0])
+		require.Containsf(t, errs[0].Error(), c.errMsg, "Expected error for %s.", c.filename)
 	}
 }
 
diff --git a/model/rulefmt/testdata/record_and_for.bad.yaml b/model/rulefmt/testdata/record_and_for.bad.yaml
new file mode 100644
index 000000000..a15d428a0
--- /dev/null
+++ b/model/rulefmt/testdata/record_and_for.bad.yaml
@@ -0,0 +1,6 @@
+groups:
+  - name: yolo
+    rules:
+      - record: Hello
+        expr: 1
+        for: 1m
diff --git a/model/rulefmt/testdata/record_and_keep_firing_for.bad.yaml b/model/rulefmt/testdata/record_and_keep_firing_for.bad.yaml
new file mode 100644
index 000000000..eb8192f05
--- /dev/null
+++ b/model/rulefmt/testdata/record_and_keep_firing_for.bad.yaml
@@ -0,0 +1,6 @@
+groups:
+  - name: yolo
+    rules:
+      - record: Hello
+        expr: 1
+        keep_firing_for: 1m
diff --git a/rules/alerting_test.go b/rules/alerting_test.go
index 4f5f5e683..036bfae1c 100644
--- a/rules/alerting_test.go
+++ b/rules/alerting_test.go
@@ -66,7 +66,7 @@ func TestAlertingRuleState(t *testing.T) {
 	}
 
 	for i, test := range tests {
-		rule := NewAlertingRule(test.name, nil, 0, labels.EmptyLabels(), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil)
+		rule := NewAlertingRule(test.name, nil, 0, 0, labels.EmptyLabels(), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil)
 		rule.active = test.active
 		got := rule.State()
 		require.Equal(t, test.want, got, "test case %d unexpected AlertState, want:%d got:%d", i, test.want, got)
@@ -76,7 +76,7 @@ func TestAlertingRuleState(t *testing.T) {
 func TestAlertingRuleLabelsUpdate(t *testing.T) {
 	suite, err := promql.NewTest(t, `
 		load 1m
-			http_requests{job="app-server", instance="0"}	75 85 70 70
+			http_requests{job="app-server", instance="0"}	75 85 70 70 stale
 	`)
 	require.NoError(t, err)
 	defer suite.Close()
@@ -90,6 +90,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
 		"HTTPRequestRateLow",
 		expr,
 		time.Minute,
+		0,
 		// Basing alerting rule labels off of a value that can change is a very bad idea.
 		// If an alert is going back and forth between two label values it will never fire.
 		// Instead, you should write two alerts with constant labels.
@@ -173,6 +174,10 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) {
 
 		require.Equal(t, result, filteredRes)
 	}
+	evalTime := baseTime.Add(time.Duration(len(results)) * time.Minute)
+	res, err := rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(res))
 }
 
 func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
@@ -192,6 +197,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
 		"ExternalLabelDoesNotExist",
 		expr,
 		time.Minute,
+		0,
 		labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."),
 		labels.EmptyLabels(),
 		labels.EmptyLabels(),
@@ -202,6 +208,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) {
 		"ExternalLabelExists",
 		expr,
 		time.Minute,
+		0,
 		labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."),
 		labels.EmptyLabels(),
 		labels.FromStrings("foo", "bar", "dings", "bums"),
@@ -286,6 +293,7 @@ func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
 		"ExternalURLDoesNotExist",
 		expr,
 		time.Minute,
+		0,
 		labels.FromStrings("templated_label", "The external URL is {{ $externalURL }}."),
 		labels.EmptyLabels(),
 		labels.EmptyLabels(),
@@ -296,6 +304,7 @@ func TestAlertingRuleExternalURLInTemplate(t *testing.T) {
 		"ExternalURLExists",
 		expr,
 		time.Minute,
+		0,
 		labels.FromStrings("templated_label", "The external URL is {{ $externalURL }}."),
 		labels.EmptyLabels(),
 		labels.EmptyLabels(),
@@ -380,6 +389,7 @@ func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) {
 		"EmptyLabel",
 		expr,
 		time.Minute,
+		0,
 		labels.FromStrings("empty_label", ""),
 		labels.EmptyLabels(),
 		labels.EmptyLabels(),
@@ -436,6 +446,7 @@ func TestAlertingRuleQueryInTemplate(t *testing.T) {
 		"ruleWithQueryInTemplate",
 		expr,
 		time.Minute,
+		0,
 		labels.FromStrings("label", "value"),
 		labels.FromStrings("templated_label", `{{- with "sort(sum(http_requests) by (instance))" | query -}}
 {{- range $i,$v := . -}}
@@ -480,7 +491,7 @@ instance: {{ $v.Labels.instance }}, value: {{ printf "%.0f" $v.Value }};
 
 func BenchmarkAlertingRuleAtomicField(b *testing.B) {
 	b.ReportAllocs()
-	rule := NewAlertingRule("bench", nil, 0, labels.EmptyLabels(), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil)
+	rule := NewAlertingRule("bench", nil, 0, 0, labels.EmptyLabels(), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil)
 	done := make(chan struct{})
 	go func() {
 		for i := 0; i < b.N; i++ {
@@ -518,6 +529,7 @@ func TestAlertingRuleDuplicate(t *testing.T) {
 		"foo",
 		expr,
 		time.Minute,
+		0,
 		labels.FromStrings("test", "test"),
 		labels.EmptyLabels(),
 		labels.EmptyLabels(),
@@ -564,6 +576,7 @@ func TestAlertingRuleLimit(t *testing.T) {
 		"foo",
 		expr,
 		time.Minute,
+		0,
 		labels.FromStrings("test", "test"),
 		labels.EmptyLabels(),
 		labels.EmptyLabels(),
@@ -636,6 +649,7 @@ func TestQueryForStateSeries(t *testing.T) {
 			"TestRule",
 			nil,
 			time.Minute,
+			0,
 			labels.FromStrings("severity", "critical"),
 			labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil,
 		)
@@ -669,6 +683,7 @@ func TestSendAlertsDontAffectActiveAlerts(t *testing.T) {
 		"TestRule",
 		nil,
 		time.Minute,
+		0,
 		labels.FromStrings("severity", "critical"),
 		labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil,
 	)
@@ -712,3 +727,171 @@ func TestSendAlertsDontAffectActiveAlerts(t *testing.T) {
 	// But the labels with the AlertingRule should not be changed.
 	require.Equal(t, labels.FromStrings("a1", "1"), rule.active[h].Labels)
 }
+
+func TestKeepFiringFor(t *testing.T) {
+	suite, err := promql.NewTest(t, `
+		load 1m
+			http_requests{job="app-server", instance="0"}	75 85 70 70 10x5
+	`)
+	require.NoError(t, err)
+	defer suite.Close()
+
+	require.NoError(t, suite.Run())
+
+	expr, err := parser.ParseExpr(`http_requests > 50`)
+	require.NoError(t, err)
+
+	rule := NewAlertingRule(
+		"HTTPRequestRateHigh",
+		expr,
+		time.Minute,
+		time.Minute,
+		labels.EmptyLabels(),
+		labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil,
+	)
+
+	results := []promql.Vector{
+		{
+			promql.Sample{
+				Metric: labels.FromStrings(
+					"__name__", "ALERTS",
+					"alertname", "HTTPRequestRateHigh",
+					"alertstate", "pending",
+					"instance", "0",
+					"job", "app-server",
+				),
+				Point: promql.Point{V: 1},
+			},
+		},
+		{
+			promql.Sample{
+				Metric: labels.FromStrings(
+					"__name__", "ALERTS",
+					"alertname", "HTTPRequestRateHigh",
+					"alertstate", "firing",
+					"instance", "0",
+					"job", "app-server",
+				),
+				Point: promql.Point{V: 1},
+			},
+		},
+		{
+			promql.Sample{
+				Metric: labels.FromStrings(
+					"__name__", "ALERTS",
+					"alertname", "HTTPRequestRateHigh",
+					"alertstate", "firing",
+					"instance", "0",
+					"job", "app-server",
+				),
+				Point: promql.Point{V: 1},
+			},
+		},
+		{
+			promql.Sample{
+				Metric: labels.FromStrings(
+					"__name__", "ALERTS",
+					"alertname", "HTTPRequestRateHigh",
+					"alertstate", "firing",
+					"instance", "0",
+					"job", "app-server",
+				),
+				Point: promql.Point{V: 1},
+			},
+		},
+		// From now on the alert should keep firing.
+		{
+			promql.Sample{
+				Metric: labels.FromStrings(
+					"__name__", "ALERTS",
+					"alertname", "HTTPRequestRateHigh",
+					"alertstate", "firing",
+					"instance", "0",
+					"job", "app-server",
+				),
+				Point: promql.Point{V: 1},
+			},
+		},
+	}
+
+	baseTime := time.Unix(0, 0)
+	for i, result := range results {
+		t.Logf("case %d", i)
+		evalTime := baseTime.Add(time.Duration(i) * time.Minute)
+		result[0].Point.T = timestamp.FromTime(evalTime)
+		res, err := rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
+		require.NoError(t, err)
+
+		var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples.
+		for _, smpl := range res {
+			smplName := smpl.Metric.Get("__name__")
+			if smplName == "ALERTS" {
+				filteredRes = append(filteredRes, smpl)
+			} else {
+				// If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'.
+				require.Equal(t, "ALERTS_FOR_STATE", smplName)
+			}
+		}
+
+		require.Equal(t, result, filteredRes)
+	}
+	evalTime := baseTime.Add(time.Duration(len(results)) * time.Minute)
+	res, err := rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(res))
+}
+
+func TestPendingAndKeepFiringFor(t *testing.T) {
+	suite, err := promql.NewTest(t, `
+		load 1m
+			http_requests{job="app-server", instance="0"}	75 10x10
+	`)
+	require.NoError(t, err)
+	defer suite.Close()
+
+	require.NoError(t, suite.Run())
+
+	expr, err := parser.ParseExpr(`http_requests > 50`)
+	require.NoError(t, err)
+
+	rule := NewAlertingRule(
+		"HTTPRequestRateHigh",
+		expr,
+		time.Minute,
+		time.Minute,
+		labels.EmptyLabels(),
+		labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil,
+	)
+
+	result := promql.Sample{
+		Metric: labels.FromStrings(
+			"__name__", "ALERTS",
+			"alertname", "HTTPRequestRateHigh",
+			"alertstate", "pending",
+			"instance", "0",
+			"job", "app-server",
+		),
+		Point: promql.Point{V: 1},
+	}
+
+	baseTime := time.Unix(0, 0)
+	result.Point.T = timestamp.FromTime(baseTime)
+	res, err := rule.Eval(suite.Context(), baseTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
+	require.NoError(t, err)
+
+	require.Len(t, res, 2)
+	for _, smpl := range res {
+		smplName := smpl.Metric.Get("__name__")
+		if smplName == "ALERTS" {
+			require.Equal(t, result, smpl)
+		} else {
+			// If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'.
+			require.Equal(t, "ALERTS_FOR_STATE", smplName)
+		}
+	}
+
+	evalTime := baseTime.Add(time.Minute)
+	res, err = rule.Eval(suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, 0)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(res))
+}
diff --git a/rules/manager_test.go b/rules/manager_test.go
index 788aa0af3..6f0dd0dda 100644
--- a/rules/manager_test.go
+++ b/rules/manager_test.go
@@ -66,6 +66,7 @@ func TestAlertingRule(t *testing.T) {
 		"HTTPRequestRateLow",
 		expr,
 		time.Minute,
+		0,
 		labels.FromStrings("severity", "{{\"c\"}}ritical"),
 		labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil,
 	)
@@ -209,6 +210,7 @@ func TestForStateAddSamples(t *testing.T) {
 		"HTTPRequestRateLow",
 		expr,
 		time.Minute,
+		0,
 		labels.FromStrings("severity", "{{\"c\"}}ritical"),
 		labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil,
 	)
@@ -383,6 +385,7 @@ func TestForStateRestore(t *testing.T) {
 		"HTTPRequestRateLow",
 		expr,
 		alertForDuration,
+		0,
 		labels.FromStrings("severity", "critical"),
 		labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil,
 	)
@@ -449,6 +452,7 @@ func TestForStateRestore(t *testing.T) {
 			"HTTPRequestRateLow",
 			expr,
 			alertForDuration,
+			0,
 			labels.FromStrings("severity", "critical"),
 			labels.EmptyLabels(), labels.EmptyLabels(), "", false, nil,
 		)
@@ -615,13 +619,13 @@ func readSeriesSet(ss storage.SeriesSet) (map[string][]promql.Point, error) {
 func TestCopyState(t *testing.T) {
 	oldGroup := &Group{
 		rules: []Rule{
-			NewAlertingRule("alert", nil, 0, labels.EmptyLabels(), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
+			NewAlertingRule("alert", nil, 0, 0, labels.EmptyLabels(), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
 			NewRecordingRule("rule1", nil, labels.EmptyLabels()),
 			NewRecordingRule("rule2", nil, labels.EmptyLabels()),
 			NewRecordingRule("rule3", nil, labels.FromStrings("l1", "v1")),
 			NewRecordingRule("rule3", nil, labels.FromStrings("l1", "v2")),
 			NewRecordingRule("rule3", nil, labels.FromStrings("l1", "v3")),
-			NewAlertingRule("alert2", nil, 0, labels.FromStrings("l2", "v1"), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
+			NewAlertingRule("alert2", nil, 0, 0, labels.FromStrings("l2", "v1"), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
 		},
 		seriesInPreviousEval: []map[string]labels.Labels{
 			{},
@@ -640,10 +644,10 @@ func TestCopyState(t *testing.T) {
 			NewRecordingRule("rule3", nil, labels.FromStrings("l1", "v0")),
 			NewRecordingRule("rule3", nil, labels.FromStrings("l1", "v1")),
 			NewRecordingRule("rule3", nil, labels.FromStrings("l1", "v2")),
-			NewAlertingRule("alert", nil, 0, labels.EmptyLabels(), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
+			NewAlertingRule("alert", nil, 0, 0, labels.EmptyLabels(), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
 			NewRecordingRule("rule1", nil, labels.EmptyLabels()),
-			NewAlertingRule("alert2", nil, 0, labels.FromStrings("l2", "v0"), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
-			NewAlertingRule("alert2", nil, 0, labels.FromStrings("l2", "v1"), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
+			NewAlertingRule("alert2", nil, 0, 0, labels.FromStrings("l2", "v0"), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
+			NewAlertingRule("alert2", nil, 0, 0, labels.FromStrings("l2", "v1"), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
 			NewRecordingRule("rule4", nil, labels.EmptyLabels()),
 		},
 		seriesInPreviousEval: make([]map[string]labels.Labels, 8),
@@ -875,7 +879,7 @@ func TestNotify(t *testing.T) {
 
 	expr, err := parser.ParseExpr("a > 1")
 	require.NoError(t, err)
-	rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, labels.EmptyLabels(), "", true, log.NewNopLogger())
+	rule := NewAlertingRule("aTooHigh", expr, 0, 0, labels.Labels{}, labels.Labels{}, labels.EmptyLabels(), "", true, log.NewNopLogger())
 	group := NewGroup(GroupOptions{
 		Name:          "alert",
 		Interval:      time.Second,
@@ -1147,7 +1151,7 @@ func TestGroupHasAlertingRules(t *testing.T) {
 			group: &Group{
 				name: "HasAlertingRule",
 				rules: []Rule{
-					NewAlertingRule("alert", nil, 0, labels.EmptyLabels(), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
+					NewAlertingRule("alert", nil, 0, 0, labels.EmptyLabels(), labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil),
 					NewRecordingRule("record", nil, labels.EmptyLabels()),
 				},
 			},
diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go
index bf3954c4d..15990b247 100644
--- a/storage/remote/codec_test.go
+++ b/storage/remote/codec_test.go
@@ -55,7 +55,7 @@ var writeRequestFixture = &prompb.WriteRequest{
 			},
 			Samples:    []prompb.Sample{{Value: 1, Timestamp: 0}},
 			Exemplars:  []prompb.Exemplar{{Labels: []prompb.Label{{Name: "f", Value: "g"}}, Value: 1, Timestamp: 0}},
-			Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram), FloatHistogramToHistogramProto(1, testHistogram.ToFloat())},
+			Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram)},
 		},
 		{
 			Labels: []prompb.Label{
@@ -67,7 +67,7 @@ var writeRequestFixture = &prompb.WriteRequest{
 			},
 			Samples:    []prompb.Sample{{Value: 2, Timestamp: 1}},
 			Exemplars:  []prompb.Exemplar{{Labels: []prompb.Label{{Name: "h", Value: "i"}}, Value: 2, Timestamp: 1}},
-			Histograms: []prompb.Histogram{HistogramToHistogramProto(2, &testHistogram), FloatHistogramToHistogramProto(3, testHistogram.ToFloat())},
+			Histograms: []prompb.Histogram{HistogramToHistogramProto(1, &testHistogram)},
 		},
 	},
 }
@@ -368,7 +368,6 @@ func TestNilHistogramProto(t *testing.T) {
 	// This function will panic if it impromperly handles nil
 	// values, causing the test to fail.
 	HistogramProtoToHistogram(prompb.Histogram{})
-	HistogramProtoToFloatHistogram(prompb.Histogram{})
 }
 
 func TestStreamResponse(t *testing.T) {
diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go
index 5ec52de6b..686c7c37b 100644
--- a/storage/remote/queue_manager_test.go
+++ b/storage/remote/queue_manager_test.go
@@ -61,17 +61,15 @@ func newHighestTimestampMetric() *maxTimestamp {
 
 func TestSampleDelivery(t *testing.T) {
 	testcases := []struct {
-		name            string
-		samples         bool
-		exemplars       bool
-		histograms      bool
-		floatHistograms bool
+		name       string
+		samples    bool
+		exemplars  bool
+		histograms bool
 	}{
-		{samples: true, exemplars: false, histograms: false, floatHistograms: false, name: "samples only"},
-		{samples: true, exemplars: true, histograms: true, floatHistograms: true, name: "samples, exemplars, and histograms"},
-		{samples: false, exemplars: true, histograms: false, floatHistograms: false, name: "exemplars only"},
-		{samples: false, exemplars: false, histograms: true, floatHistograms: false, name: "histograms only"},
-		{samples: false, exemplars: false, histograms: false, floatHistograms: true, name: "float histograms only"},
+		{samples: true, exemplars: false, histograms: false, name: "samples only"},
+		{samples: true, exemplars: true, histograms: true, name: "samples, exemplars, and histograms"},
+		{samples: false, exemplars: true, histograms: false, name: "exemplars only"},
+		{samples: false, exemplars: false, histograms: true, name: "histograms only"},
 	}
 
 	// Let's create an even number of send batches so we don't run into the
@@ -103,11 +101,10 @@ func TestSampleDelivery(t *testing.T) {
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
 			var (
-				series          []record.RefSeries
-				samples         []record.RefSample
-				exemplars       []record.RefExemplar
-				histograms      []record.RefHistogramSample
-				floatHistograms []record.RefFloatHistogramSample
+				series     []record.RefSeries
+				samples    []record.RefSample
+				exemplars  []record.RefExemplar
+				histograms []record.RefHistogramSample
 			)
 
 			// Generates same series in both cases.
@@ -118,10 +115,7 @@ func TestSampleDelivery(t *testing.T) {
 				exemplars, series = createExemplars(n, n)
 			}
 			if tc.histograms {
-				histograms, _, series = createHistograms(n, n, false)
-			}
-			if tc.floatHistograms {
-				_, floatHistograms, series = createHistograms(n, n, true)
+				histograms, series = createHistograms(n, n)
 			}
 
 			// Apply new config.
@@ -141,22 +135,18 @@ func TestSampleDelivery(t *testing.T) {
 			c.expectSamples(samples[:len(samples)/2], series)
 			c.expectExemplars(exemplars[:len(exemplars)/2], series)
 			c.expectHistograms(histograms[:len(histograms)/2], series)
-			c.expectFloatHistograms(floatHistograms[:len(floatHistograms)/2], series)
 			qm.Append(samples[:len(samples)/2])
 			qm.AppendExemplars(exemplars[:len(exemplars)/2])
 			qm.AppendHistograms(histograms[:len(histograms)/2])
-			qm.AppendFloatHistograms(floatHistograms[:len(floatHistograms)/2])
 			c.waitForExpectedData(t)
 
 			// Send second half of data.
 			c.expectSamples(samples[len(samples)/2:], series)
 			c.expectExemplars(exemplars[len(exemplars)/2:], series)
 			c.expectHistograms(histograms[len(histograms)/2:], series)
-			c.expectFloatHistograms(floatHistograms[len(floatHistograms)/2:], series)
 			qm.Append(samples[len(samples)/2:])
 			qm.AppendExemplars(exemplars[len(exemplars)/2:])
 			qm.AppendHistograms(histograms[len(histograms)/2:])
-			qm.AppendFloatHistograms(floatHistograms[len(floatHistograms)/2:])
 			c.waitForExpectedData(t)
 		})
 	}
@@ -596,50 +586,35 @@ func createExemplars(numExemplars, numSeries int) ([]record.RefExemplar, []recor
 	return exemplars, series
 }
 
-func createHistograms(numSamples, numSeries int, floatHistogram bool) ([]record.RefHistogramSample, []record.RefFloatHistogramSample, []record.RefSeries) {
+func createHistograms(numSamples, numSeries int) ([]record.RefHistogramSample, []record.RefSeries) {
 	histograms := make([]record.RefHistogramSample, 0, numSamples)
-	floatHistograms := make([]record.RefFloatHistogramSample, 0, numSamples)
 	series := make([]record.RefSeries, 0, numSeries)
 	for i := 0; i < numSeries; i++ {
 		name := fmt.Sprintf("test_metric_%d", i)
 		for j := 0; j < numSamples; j++ {
-			hist := &histogram.Histogram{
-				Schema:          2,
-				ZeroThreshold:   1e-128,
-				ZeroCount:       0,
-				Count:           2,
-				Sum:             0,
-				PositiveSpans:   []histogram.Span{{Offset: 0, Length: 1}},
-				PositiveBuckets: []int64{int64(i) + 1},
-				NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
-				NegativeBuckets: []int64{int64(-i) - 1},
-			}
-
-			if floatHistogram {
-				fh := record.RefFloatHistogramSample{
-					Ref: chunks.HeadSeriesRef(i),
-					T:   int64(j),
-					FH:  hist.ToFloat(),
-				}
-				floatHistograms = append(floatHistograms, fh)
-			} else {
-				h := record.RefHistogramSample{
-					Ref: chunks.HeadSeriesRef(i),
-					T:   int64(j),
-					H:   hist,
-				}
-				histograms = append(histograms, h)
+			h := record.RefHistogramSample{
+				Ref: chunks.HeadSeriesRef(i),
+				T:   int64(j),
+				H: &histogram.Histogram{
+					Schema:          2,
+					ZeroThreshold:   1e-128,
+					ZeroCount:       0,
+					Count:           2,
+					Sum:             0,
+					PositiveSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+					PositiveBuckets: []int64{int64(i) + 1},
+					NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+					NegativeBuckets: []int64{int64(-i) - 1},
+				},
 			}
+			histograms = append(histograms, h)
 		}
 		series = append(series, record.RefSeries{
 			Ref:    chunks.HeadSeriesRef(i),
 			Labels: labels.FromStrings("__name__", name),
 		})
 	}
-	if floatHistogram {
-		return nil, floatHistograms, series
-	}
-	return histograms, nil, series
+	return histograms, series
 }
 
 func getSeriesNameFromRef(r record.RefSeries) string {
@@ -647,20 +622,18 @@ func getSeriesNameFromRef(r record.RefSeries) string {
 }
 
 type TestWriteClient struct {
-	receivedSamples         map[string][]prompb.Sample
-	expectedSamples         map[string][]prompb.Sample
-	receivedExemplars       map[string][]prompb.Exemplar
-	expectedExemplars       map[string][]prompb.Exemplar
-	receivedHistograms      map[string][]prompb.Histogram
-	receivedFloatHistograms map[string][]prompb.Histogram
-	expectedHistograms      map[string][]prompb.Histogram
-	expectedFloatHistograms map[string][]prompb.Histogram
-	receivedMetadata        map[string][]prompb.MetricMetadata
-	writesReceived          int
-	withWaitGroup           bool
-	wg                      sync.WaitGroup
-	mtx                     sync.Mutex
-	buf                     []byte
+	receivedSamples    map[string][]prompb.Sample
+	expectedSamples    map[string][]prompb.Sample
+	receivedExemplars  map[string][]prompb.Exemplar
+	expectedExemplars  map[string][]prompb.Exemplar
+	receivedHistograms map[string][]prompb.Histogram
+	expectedHistograms map[string][]prompb.Histogram
+	receivedMetadata   map[string][]prompb.MetricMetadata
+	writesReceived     int
+	withWaitGroup      bool
+	wg                 sync.WaitGroup
+	mtx                sync.Mutex
+	buf                []byte
 }
 
 func NewTestWriteClient() *TestWriteClient {
@@ -731,23 +704,6 @@ func (c *TestWriteClient) expectHistograms(hh []record.RefHistogramSample, serie
 	c.wg.Add(len(hh))
 }
 
-func (c *TestWriteClient) expectFloatHistograms(fhs []record.RefFloatHistogramSample, series []record.RefSeries) {
-	if !c.withWaitGroup {
-		return
-	}
-	c.mtx.Lock()
-	defer c.mtx.Unlock()
-
-	c.expectedFloatHistograms = map[string][]prompb.Histogram{}
-	c.receivedFloatHistograms = map[string][]prompb.Histogram{}
-
-	for _, fh := range fhs {
-		seriesName := getSeriesNameFromRef(series[fh.Ref])
-		c.expectedFloatHistograms[seriesName] = append(c.expectedFloatHistograms[seriesName], FloatHistogramToHistogramProto(fh.T, fh.FH))
-	}
-	c.wg.Add(len(fhs))
-}
-
 func (c *TestWriteClient) waitForExpectedData(tb testing.TB) {
 	if !c.withWaitGroup {
 		return
@@ -764,9 +720,6 @@ func (c *TestWriteClient) waitForExpectedData(tb testing.TB) {
 	for ts, expectedHistogram := range c.expectedHistograms {
 		require.Equal(tb, expectedHistogram, c.receivedHistograms[ts], ts)
 	}
-	for ts, expectedFloatHistogram := range c.expectedFloatHistograms {
-		require.Equal(tb, expectedFloatHistogram, c.receivedFloatHistograms[ts], ts)
-	}
 }
 
 func (c *TestWriteClient) Store(_ context.Context, req []byte) error {
@@ -802,12 +755,7 @@ func (c *TestWriteClient) Store(_ context.Context, req []byte) error {
 
 		for _, histogram := range ts.Histograms {
 			count++
-			if histogram.GetCountFloat() > 0 || histogram.GetZeroCountFloat() > 0 {
-				c.receivedFloatHistograms[seriesName] = append(c.receivedFloatHistograms[seriesName], histogram)
-			} else {
-				c.receivedHistograms[seriesName] = append(c.receivedHistograms[seriesName], histogram)
-			}
-
+			c.receivedHistograms[seriesName] = append(c.receivedHistograms[seriesName], histogram)
 		}
 	}
 	if c.withWaitGroup {
diff --git a/storage/remote/write_handler_test.go b/storage/remote/write_handler_test.go
index 8ca6af805..731e0e3e5 100644
--- a/storage/remote/write_handler_test.go
+++ b/storage/remote/write_handler_test.go
@@ -66,14 +66,8 @@ func TestRemoteWriteHandler(t *testing.T) {
 		}
 
 		for _, hp := range ts.Histograms {
-			if hp.GetCountFloat() > 0 || hp.GetZeroCountFloat() > 0 { // It is a float histogram.
-				fh := HistogramProtoToFloatHistogram(hp)
-				require.Equal(t, mockHistogram{labels, hp.Timestamp, nil, fh}, appendable.histograms[k])
-			} else {
-				h := HistogramProtoToHistogram(hp)
-				require.Equal(t, mockHistogram{labels, hp.Timestamp, h, nil}, appendable.histograms[k])
-			}
-
+			h := HistogramProtoToHistogram(hp)
+			require.Equal(t, mockHistogram{labels, hp.Timestamp, h, nil}, appendable.histograms[k])
 			k++
 		}
 	}
@@ -130,7 +124,7 @@ func TestOutOfOrderExemplar(t *testing.T) {
 func TestOutOfOrderHistogram(t *testing.T) {
 	buf, _, err := buildWriteRequest([]prompb.TimeSeries{{
 		Labels:     []prompb.Label{{Name: "__name__", Value: "test_metric"}},
-		Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram), FloatHistogramToHistogramProto(1, testHistogram.ToFloat())},
+		Histograms: []prompb.Histogram{HistogramToHistogramProto(0, &testHistogram)},
 	}}, nil, nil, nil)
 	require.NoError(t, err)
 
diff --git a/tsdb/wlog/watcher_test.go b/tsdb/wlog/watcher_test.go
index 530d0ffb4..545cc338e 100644
--- a/tsdb/wlog/watcher_test.go
+++ b/tsdb/wlog/watcher_test.go
@@ -52,12 +52,11 @@ func retry(t *testing.T, interval time.Duration, n int, f func() bool) {
 }
 
 type writeToMock struct {
-	samplesAppended         int
-	exemplarsAppended       int
-	histogramsAppended      int
-	floatHistogramsAppended int
-	seriesLock              sync.Mutex
-	seriesSegmentIndexes    map[chunks.HeadSeriesRef]int
+	samplesAppended      int
+	exemplarsAppended    int
+	histogramsAppended   int
+	seriesLock           sync.Mutex
+	seriesSegmentIndexes map[chunks.HeadSeriesRef]int
 }
 
 func (wtm *writeToMock) Append(s []record.RefSample) bool {
@@ -75,11 +74,6 @@ func (wtm *writeToMock) AppendHistograms(h []record.RefHistogramSample) bool {
 	return true
 }
 
-func (wtm *writeToMock) AppendFloatHistograms(fh []record.RefFloatHistogramSample) bool {
-	wtm.floatHistogramsAppended += len(fh)
-	return true
-}
-
 func (wtm *writeToMock) StoreSeries(series []record.RefSeries, index int) {
 	wtm.UpdateSeriesSegment(series, index)
 }
@@ -177,31 +171,22 @@ func TestTailSamples(t *testing.T) {
 
 				for j := 0; j < histogramsCount; j++ {
 					inner := rand.Intn(ref + 1)
-					hist := &histogram.Histogram{
-						Schema:          2,
-						ZeroThreshold:   1e-128,
-						ZeroCount:       0,
-						Count:           2,
-						Sum:             0,
-						PositiveSpans:   []histogram.Span{{Offset: 0, Length: 1}},
-						PositiveBuckets: []int64{int64(i) + 1},
-						NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
-						NegativeBuckets: []int64{int64(-i) - 1},
-					}
-
 					histogram := enc.HistogramSamples([]record.RefHistogramSample{{
 						Ref: chunks.HeadSeriesRef(inner),
 						T:   now.UnixNano() + 1,
-						H:   hist,
+						H: &histogram.Histogram{
+							Schema:          2,
+							ZeroThreshold:   1e-128,
+							ZeroCount:       0,
+							Count:           2,
+							Sum:             0,
+							PositiveSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+							PositiveBuckets: []int64{int64(i) + 1},
+							NegativeSpans:   []histogram.Span{{Offset: 0, Length: 1}},
+							NegativeBuckets: []int64{int64(-i) - 1},
+						},
 					}}, nil)
 					require.NoError(t, w.Log(histogram))
-
-					floatHistogram := enc.FloatHistogramSamples([]record.RefFloatHistogramSample{{
-						Ref: chunks.HeadSeriesRef(inner),
-						T:   now.UnixNano() + 1,
-						FH:  hist.ToFloat(),
-					}}, nil)
-					require.NoError(t, w.Log(floatHistogram))
 				}
 			}
 
@@ -236,7 +221,6 @@ func TestTailSamples(t *testing.T) {
 			require.Equal(t, expectedSamples, wt.samplesAppended, "did not receive the expected number of samples")
 			require.Equal(t, expectedExemplars, wt.exemplarsAppended, "did not receive the expected number of exemplars")
 			require.Equal(t, expectedHistograms, wt.histogramsAppended, "did not receive the expected number of histograms")
-			require.Equal(t, expectedHistograms, wt.floatHistogramsAppended, "did not receive the expected number of float histograms")
 		})
 	}
 }
diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go
index 7e2dcbd8b..919fad34b 100644
--- a/web/api/v1/api_test.go
+++ b/web/api/v1/api_test.go
@@ -209,6 +209,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
 		"test_metric3",
 		expr1,
 		time.Second,
+		0,
 		labels.Labels{},
 		labels.Labels{},
 		labels.Labels{},
@@ -220,6 +221,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
 		"test_metric4",
 		expr2,
 		time.Second,
+		0,
 		labels.Labels{},
 		labels.Labels{},
 		labels.Labels{},