Repository: cortexproject/cortex (Go)

Issue:
Query Frontend: queries with negative offset should check whether it is cacheable or not
**Describe the bug**
We enabled negative offset feature by default in Cortex. However, negative offset might query data in the future timestamp and those latest data response shouldn't be cached. We missed such check in Query Frontend similar to what we have for @ modifier https://github.com/cortexproject/cortex/blob/master/pkg/querier/tripperware/queryrange/results_cache.go#L313.

**Expected behavior**
If a query has negative offset, I expect we won't cache its results if the promql query time range after applying negative offset is later than max cache freshness time.

We can also start simple by not caching queries with negative offset.


Discussion: Hi! I would like to work on this issue, I think I understand what changes to make, will make a PR soon.
> We can also start simple by not caching queries with negative offset.

I would recommend we start simple with this option.



Fix (patch):
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9a5cff31d72..758f0f89b52 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@
 * [ENHANCEMENT] Index Cache: Multi level cache adds config `max_backfill_items` to cap max items to backfill per async operation. #5686
 * [ENHANCEMENT] Query Frontend: Log number of split queries in `query stats` log. #5703
 * [BUGFIX] Distributor: Do not use label with empty values for sharding #5717
+* [BUGFIX] Query Frontend: queries with negative offset should check whether it is cacheable or not. #5719
 
 
 ## 1.16.0 2023-11-20
diff --git a/pkg/querier/tripperware/queryrange/results_cache.go b/pkg/querier/tripperware/queryrange/results_cache.go
index 22ccce82d57..21f195b9cce 100644
--- a/pkg/querier/tripperware/queryrange/results_cache.go
+++ b/pkg/querier/tripperware/queryrange/results_cache.go
@@ -265,12 +265,13 @@ func (s resultsCache) shouldCacheResponse(ctx context.Context, req tripperware.R
 	if !s.isAtModifierCachable(ctx, req, maxCacheTime) {
 		return false
 	}
+	if !s.isOffsetCachable(ctx, req) {
+		return false
+	}
 
 	return true
 }
 
-var errAtModifierAfterEnd = errors.New("at modifier after end")
-
 // isAtModifierCachable returns true if the @ modifier result
 // is safe to cache.
 func (s resultsCache) isAtModifierCachable(ctx context.Context, r tripperware.Request, maxCacheTime int64) bool {
@@ -280,6 +281,7 @@ func (s resultsCache) isAtModifierCachable(ctx context.Context, r tripperware.Re
 	//      below maxCacheTime. In such cases if any tenant is intentionally
 	//      playing with old data, we could cache empty result if we look
 	//      beyond query end.
+	var errAtModifierAfterEnd = errors.New("at modifier after end")
 	query := r.GetQuery()
 	if !strings.Contains(query, "@") {
 		return true
@@ -321,6 +323,46 @@ func (s resultsCache) isAtModifierCachable(ctx context.Context, r tripperware.Re
 	return atModCachable
 }
 
+// isOffsetCachable returns true if the offset is positive, result is safe to cache.
+// and false when offset is negative, result is not cached.
+func (s resultsCache) isOffsetCachable(ctx context.Context, r tripperware.Request) bool {
+	var errNegativeOffset = errors.New("negative offset")
+	query := r.GetQuery()
+	if !strings.Contains(query, "offset") {
+		return true
+	}
+	expr, err := parser.ParseExpr(query)
+	if err != nil {
+		level.Warn(util_log.WithContext(ctx, s.logger)).Log("msg", "failed to parse query, considering offset as not cachable", "query", query, "err", err)
+		return false
+	}
+
+	offsetCachable := true
+	parser.Inspect(expr, func(n parser.Node, _ []parser.Node) error {
+		switch e := n.(type) {
+		case *parser.VectorSelector:
+			if e.OriginalOffset < 0 {
+				offsetCachable = false
+				return errNegativeOffset
+			}
+		case *parser.MatrixSelector:
+			offset := e.VectorSelector.(*parser.VectorSelector).OriginalOffset
+			if offset < 0 {
+				offsetCachable = false
+				return errNegativeOffset
+			}
+		case *parser.SubqueryExpr:
+			if e.OriginalOffset < 0 {
+				offsetCachable = false
+				return errNegativeOffset
+			}
+		}
+		return nil
+	})
+
+	return offsetCachable
+}
+
 func getHeaderValuesWithName(r tripperware.Response, headerName string) (headerValues []string) {
 	for name, hv := range r.HTTPHeaders() {
 		if name != headerName {