Repository: jaegertracing/jaeger

Issue:
[Bug]: Flaky e2e tests for OpenSearch
### What happened?

https://github.com/jaegertracing/jaeger/actions/runs/14332919757/job/40172890887?pr=7000

It appears some system-level index `top_queries-2025.04.08-51429` is being picked up by the script that fails the test / comparison.

### Steps to reproduce

This happens occasionally in CI, spurious failure.

### Expected behavior

Expecting no failure when the relevant code is not touched by a PR.

### Relevant log output

```shell
es_index_cleaner_test.go:166: 
        	Error Trace:	/home/runner/work/jaeger/jaeger/internal/storage/integration/es_index_cleaner_test.go:166
        	            				/home/runner/work/jaeger/jaeger/internal/storage/integration/es_index_cleaner_test.go:140
        	Error:      	elements differ
        	            	
        	            	extra elements in list A:
        	            	([]interface {}) (len=1) {
        	            	 (string) (len=28) "top_queries-2025.04.08-51429"
        	            	}
        	            	
        	            	
        	            	listA:
        	            	([]string) (len=10) {
        	            	 (string) (len=26) "jaeger-span-archive-000002",
        	            	 (string) (len=19) "jaeger-span-archive",
        	            	 (string) (len=26) "jaeger-dependencies-000002",
        	            	 (string) (len=26) "jaeger-span-archive-000001",
        	            	 (string) (len=18) "jaeger-span-000002",
        	            	 (string) (len=21) "jaeger-service-000002",
        	            	 (string) (len=21) "jaeger-service-000001",
        	            	 (string) (len=26) "jaeger-dependencies-000001",
        	            	 (string) (len=18) "jaeger-span-000001",
        	            	 (string) (len=28) "top_queries-2025.04.08-51429"
        	            	}
        	            	
        	            	
        	            	listB:
        	            	([]string) (len=9) {
        	            	 (string) (len=19) "jaeger-span-archive",
        	            	 (string) (len=18) "jaeger-span-000001",
        	            	 (string) (len=21) "jaeger-service-000001",
        	            	 (string) (len=26) "jaeger-dependencies-000001",
        	            	 (string) (len=18) "jaeger-span-000002",
        	            	 (string) (len=21) "jaeger-service-000002",
        	            	 (string) (len=26) "jaeger-dependencies-000002",
        	            	 (string) (len=26) "jaeger-span-archive-000001",
        	            	 (string) (len=26) "jaeger-span-archive-000002"
        	            	}
        	Test:       	TestIndexCleaner/RemoveDailyIndices_no_prefix,_[]
```

### Screenshot

_No response_

### Additional context

_No response_

### Jaeger backend version

_No response_

### SDK

_No response_

### Pipeline

_No response_

### Stogage backend

_No response_

### Operating system

_No response_

### Deployment model

_No response_

### Deployment configs

```shell

```

Discussion:
@0xShubhamSolanki: Hi @yurishkuro @pavolloffay @jaegertracing/maintainers,

I have raised this #7004 PR to resolve issue #7002,
I’d appreciate any feedback or suggestions on the implementation. Let me know if additional changes are needed!


Solution patch:
diff --git a/internal/storage/v1/cassandra/Dockerfile b/internal/storage/v1/cassandra/Dockerfile
index 5345f0a2..910dc870 100644
--- a/internal/storage/v1/cassandra/Dockerfile
+++ b/internal/storage/v1/cassandra/Dockerfile
@@ -1,7 +1,7 @@
 # Copyright (c) 2024 The Jaeger Authors.
 # SPDX-License-Identifier: Apache-2.0
 
-FROM cassandra:5.0@sha256:70b40a2025d450f7865c5ec6f1ebea13108166f81fe41462069690cb4d9690f2
+FROM cassandra:5.0.2@sha256:9945dafdc759800f1e129ee871e45c9d3aa304fb5149148bde8685ae9812b81b
 
 COPY schema/* /cassandra-schema/
 
diff --git a/internal/storage/v1/elasticsearch/spanstore/fixtures/es_01.json b/internal/storage/v1/elasticsearch/spanstore/fixtures/es_01.json
index 8b30c120..9a5fac60 100644
--- a/internal/storage/v1/elasticsearch/spanstore/fixtures/es_01.json
+++ b/internal/storage/v1/elasticsearch/spanstore/fixtures/es_01.json
@@ -32,17 +32,17 @@
     {
       "key": "peer.ipv4",
       "type": "int64",
-      "value": "23456"
+      "value": 23456
     },
     {
       "key": "error",
       "type": "bool",
-      "value": "true"
+      "value": true
     },
     {
       "key": "temperature",
       "type": "float64",
-      "value": "72.5"
+      "value": 72.5
     },
     {
       "key": "blob",
@@ -57,7 +57,7 @@
         {
           "key": "event",
           "type": "int64",
-          "value": "123415"
+          "value": 123415
         }
       ]
     },
@@ -78,12 +78,12 @@
       {
         "key": "peer.ipv4",
         "type": "int64",
-        "value": "23456"
+        "value": 23456
       },
       {
         "key": "error",
         "type": "bool",
-        "value": "true"
+        "value": true
       }
     ]
   }
diff --git a/internal/storage/v1/elasticsearch/spanstore/from_domain.go b/internal/storage/v1/elasticsearch/spanstore/from_domain.go
index b571b9bf..d64c51de 100644
--- a/internal/storage/v1/elasticsearch/spanstore/from_domain.go
+++ b/internal/storage/v1/elasticsearch/spanstore/from_domain.go
@@ -119,9 +119,14 @@ func (fd FromDomain) convertProcess(process *model.Process) dbmodel.Process {
 }
 
 func convertKeyValue(kv model.KeyValue) dbmodel.KeyValue {
-	return dbmodel.KeyValue{
-		Key:   kv.Key,
-		Type:  dbmodel.ValueType(strings.ToLower(kv.VType.String())),
-		Value: kv.AsString(),
+	outKv := dbmodel.KeyValue{
+		Key:  kv.Key,
+		Type: dbmodel.ValueType(strings.ToLower(kv.VType.String())),
 	}
+	if kv.GetVType() == model.BinaryType {
+		outKv.Value = kv.AsString()
+	} else {
+		outKv.Value = kv.Value()
+	}
+	return outKv
 }
diff --git a/internal/storage/v1/elasticsearch/spanstore/to_domain.go b/internal/storage/v1/elasticsearch/spanstore/to_domain.go
index 070672f3..964dfbfb 100644
--- a/internal/storage/v1/elasticsearch/spanstore/to_domain.go
+++ b/internal/storage/v1/elasticsearch/spanstore/to_domain.go
@@ -173,13 +173,28 @@ func (td ToDomain) convertTagField(k string, v any) (model.KeyValue, error) {
 
 // convertKeyValue expects the Value field to be string, because it only works
 // as a reverse transformation after FromDomain() for ElasticSearch model.
-func (ToDomain) convertKeyValue(tag *dbmodel.KeyValue) (model.KeyValue, error) {
+func (td ToDomain) convertKeyValue(tag *dbmodel.KeyValue) (model.KeyValue, error) {
 	if tag.Value == nil {
 		return model.KeyValue{}, fmt.Errorf("invalid nil Value in %v", tag)
 	}
 	tagValue, ok := tag.Value.(string)
 	if !ok {
-		return model.KeyValue{}, fmt.Errorf("non-string Value of type %t in %v", tag.Value, tag)
+		switch tag.Type {
+		case dbmodel.Int64Type, dbmodel.Float64Type:
+			kv, err := td.fromDBNumber(tag)
+			if err != nil {
+				return model.KeyValue{}, err
+			}
+			return kv, nil
+		case dbmodel.BoolType:
+			if boolVal, ok := tag.Value.(bool); ok {
+				return model.Bool(tag.Key, boolVal), nil
+			}
+			return model.KeyValue{}, invalidValueErr(tag)
+		// string and binary values should always be of string type
+		default:
+			return model.KeyValue{}, invalidValueErr(tag)
+		}
 	}
 	switch tag.Type {
 	case dbmodel.StringType:
@@ -212,6 +227,42 @@ func (ToDomain) convertKeyValue(tag *dbmodel.KeyValue) (model.KeyValue, error) {
 	return model.KeyValue{}, fmt.Errorf("not a valid ValueType string %s", string(tag.Type))
 }
 
+func (ToDomain) fromDBNumber(kv *dbmodel.KeyValue) (model.KeyValue, error) {
+	if kv.Type == dbmodel.Int64Type {
+		switch v := kv.Value.(type) {
+		case int64:
+			return model.Int64(kv.Key, v), nil
+		// This case is very much possible as JSON converts every number to float64
+		case float64:
+			return model.Int64(kv.Key, int64(v)), nil
+		case json.Number:
+			n, err := v.Int64()
+			if err == nil {
+				return model.Int64(kv.Key, n), nil
+			}
+		default:
+			return model.KeyValue{}, invalidValueErr(kv)
+		}
+	} else if kv.Type == dbmodel.Float64Type {
+		switch v := kv.Value.(type) {
+		case float64:
+			return model.Float64(kv.Key, v), nil
+		case json.Number:
+			n, err := v.Float64()
+			if err == nil {
+				return model.Float64(kv.Key, n), nil
+			}
+		default:
+			return model.KeyValue{}, invalidValueErr(kv)
+		}
+	}
+	return model.KeyValue{}, fmt.Errorf("not a valid number ValueType %s", string(kv.Type))
+}
+
+func invalidValueErr(kv *dbmodel.KeyValue) error {
+	return fmt.Errorf("invalid %s type in %+v", string(kv.Type), kv.Value)
+}
+
 func (td ToDomain) convertLogs(logs []dbmodel.Log) ([]model.Log, error) {
 	retMe := make([]model.Log, len(logs))
 	for i, l := range logs {



Test patch:
diff --git a/internal/storage/integration/es_index_cleaner_test.go b/internal/storage/integration/es_index_cleaner_test.go
index 461778c2..df6733d9 100644
--- a/internal/storage/integration/es_index_cleaner_test.go
+++ b/internal/storage/integration/es_index_cleaner_test.go
@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net/http"
 	"os/exec"
+	"strings"
 	"testing"
 
 	elasticsearch8 "github.com/elastic/go-elasticsearch/v8"
@@ -154,16 +155,23 @@ func runIndexCleanerTest(t *testing.T, client *elastic.Client, v8Client *elastic
 	require.NoError(t, err)
 	err = runEsCleaner(0, envVars)
 	require.NoError(t, err)
-	indices, err := client.IndexNames()
+	foundIndices, err := client.IndexNames()
 	require.NoError(t, err)
 	if prefix != "" {
 		prefix += "-"
 	}
+	var actual []string
+	for _, index := range foundIndices {
+		// ignore system indices https://github.com/jaegertracing/jaeger/issues/7002
+		if strings.HasPrefix(index, prefix+"jaeger") {
+			actual = append(actual, index)
+		}
+	}
 	var expected []string
 	for _, index := range expectedIndices {
 		expected = append(expected, prefix+index)
 	}
-	assert.ElementsMatch(t, indices, expected, "indices found: %v, expected: %v", indices, expected)
+	assert.ElementsMatch(t, actual, expected, "indices found: %v, expected: %v", foundIndices, expected)
 }
 
 func createAllIndices(client *elastic.Client, prefix string, adaptiveSampling bool) error {
diff --git a/internal/storage/v1/elasticsearch/spanstore/from_domain_test.go b/internal/storage/v1/elasticsearch/spanstore/from_domain_test.go
index 31e56bad..81343d08 100644
--- a/internal/storage/v1/elasticsearch/spanstore/from_domain_test.go
+++ b/internal/storage/v1/elasticsearch/spanstore/from_domain_test.go
@@ -32,12 +32,9 @@ func TestFromDomainEmbedProcess(t *testing.T) {
 			converter := NewFromDomain(false, nil, ":")
 			embeddedSpan := converter.FromDomainEmbedProcess(&span)
 
-			var expectedSpan dbmodel.Span
-			require.NoError(t, json.Unmarshal(jsonStr, &expectedSpan))
-
 			testJSONEncoding(t, i, jsonStr, embeddedSpan)
 
-			CompareJSONSpans(t, &expectedSpan, embeddedSpan)
+			CompareJSONSpans(t, jsonStr, embeddedSpan)
 		})
 	}
 }
@@ -112,19 +109,19 @@ func TestConvertKeyValueValue(t *testing.T) {
 	}{
 		{
 			kv:       model.Bool(key, true),
-			expected: dbmodel.KeyValue{Key: key, Value: "true", Type: "bool"},
+			expected: dbmodel.KeyValue{Key: key, Value: true, Type: "bool"},
 		},
 		{
 			kv:       model.Bool(key, false),
-			expected: dbmodel.KeyValue{Key: key, Value: "false", Type: "bool"},
+			expected: dbmodel.KeyValue{Key: key, Value: false, Type: "bool"},
 		},
 		{
 			kv:       model.Int64(key, int64(1499)),
-			expected: dbmodel.KeyValue{Key: key, Value: "1499", Type: "int64"},
+			expected: dbmodel.KeyValue{Key: key, Value: int64(1499), Type: "int64"},
 		},
 		{
 			kv:       model.Float64(key, float64(15.66)),
-			expected: dbmodel.KeyValue{Key: key, Value: "15.66", Type: "float64"},
+			expected: dbmodel.KeyValue{Key: key, Value: 15.66, Type: "float64"},
 		},
 		{
 			kv:       model.String(key, longString),
@@ -134,10 +131,6 @@ func TestConvertKeyValueValue(t *testing.T) {
 			kv:       model.Binary(key, []byte(longString)),
 			expected: dbmodel.KeyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"},
 		},
-		{
-			kv:       model.KeyValue{VType: 1500, Key: key},
-			expected: dbmodel.KeyValue{Key: key, Value: "unknown type 1500", Type: "1500"},
-		},
 	}
 
 	for _, test := range tests {
diff --git a/internal/storage/v1/elasticsearch/spanstore/json_span_compare_test.go b/internal/storage/v1/elasticsearch/spanstore/json_span_compare_test.go
index 6f393bdf..4f74cb4f 100644
--- a/internal/storage/v1/elasticsearch/spanstore/json_span_compare_test.go
+++ b/internal/storage/v1/elasticsearch/spanstore/json_span_compare_test.go
@@ -5,8 +5,8 @@
 package spanstore
 
 import (
+	"bytes"
 	"encoding/json"
-	"sort"
 	"testing"
 
 	"github.com/kr/pretty"
@@ -16,11 +16,12 @@ import (
 	"github.com/jaegertracing/jaeger/internal/storage/elasticsearch/dbmodel"
 )
 
-func CompareJSONSpans(t *testing.T, expected *dbmodel.Span, actual *dbmodel.Span) {
-	sortJSONSpan(expected)
-	sortJSONSpan(actual)
-
-	if !assert.Equal(t, expected, actual) {
+func CompareJSONSpans(t *testing.T, expected []byte, actual *dbmodel.Span) {
+	buf := &bytes.Buffer{}
+	enc := json.NewEncoder(buf)
+	enc.SetIndent("", "  ")
+	require.NoError(t, enc.Encode(actual))
+	if !assert.Equal(t, string(expected), buf.String()) {
 		for _, err := range pretty.Diff(expected, actual) {
 			t.Log(err)
 		}
@@ -29,36 +30,3 @@ func CompareJSONSpans(t *testing.T, expected *dbmodel.Span, actual *dbmodel.Span
 		t.Logf("Actual trace: %s", string(out))
 	}
 }
-
-func sortJSONSpan(span *dbmodel.Span) {
-	sortJSONTags(span.Tags)
-	sortJSONLogs(span.Logs)
-	sortJSONProcess(span.Process)
-}
-
-type JSONTagByKey []dbmodel.KeyValue
-
-func (t JSONTagByKey) Len() int           { return len(t) }
-func (t JSONTagByKey) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
-func (t JSONTagByKey) Less(i, j int) bool { return t[i].Key < t[j].Key }
-
-func sortJSONTags(tags []dbmodel.KeyValue) {
-	sort.Sort(JSONTagByKey(tags))
-}
-
-type JSONLogByTimestamp []dbmodel.Log
-
-func (t JSONLogByTimestamp) Len() int           { return len(t) }
-func (t JSONLogByTimestamp) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
-func (t JSONLogByTimestamp) Less(i, j int) bool { return t[i].Timestamp < t[j].Timestamp }
-
-func sortJSONLogs(logs []dbmodel.Log) {
-	sort.Sort(JSONLogByTimestamp(logs))
-	for i := range logs {
-		sortJSONTags(logs[i].Fields)
-	}
-}
-
-func sortJSONProcess(process dbmodel.Process) {
-	sortJSONTags(process.Tags)
-}
diff --git a/internal/storage/v1/elasticsearch/spanstore/to_domain_test.go b/internal/storage/v1/elasticsearch/spanstore/to_domain_test.go
index c920ad6e..bf2740f8 100644
--- a/internal/storage/v1/elasticsearch/spanstore/to_domain_test.go
+++ b/internal/storage/v1/elasticsearch/spanstore/to_domain_test.go
@@ -163,10 +163,13 @@ func TestFailureBadLogs(t *testing.T) {
 
 func TestRevertKeyValueOfType(t *testing.T) {
 	tests := []struct {
-		kv  *dbmodel.KeyValue
-		err string
+		name  string
+		kv    *dbmodel.KeyValue
+		err   string
+		outKv model.KeyValue
 	}{
 		{
+			name: "not a valid ValueType string",
 			kv: &dbmodel.KeyValue{
 				Key:   "sneh",
 				Type:  "badType",
@@ -175,22 +178,121 @@ func TestRevertKeyValueOfType(t *testing.T) {
 			err: "not a valid ValueType string",
 		},
 		{
-			kv:  &dbmodel.KeyValue{},
-			err: "invalid nil Value",
+			name: "invalid nil Value",
+			kv:   &dbmodel.KeyValue{},
+			err:  "invalid nil Value",
 		},
 		{
+			name: "right int value",
 			kv: &dbmodel.KeyValue{
+				Key:   "int-val",
+				Type:  dbmodel.Int64Type,
+				Value: int64(123),
+			},
+			outKv: model.KeyValue{
+				Key:    "int-val",
+				VInt64: 123,
+				VType:  2,
+			},
+		},
+		{
+			name: "right int float value",
+			kv: &dbmodel.KeyValue{
+				Key:   "int-val",
+				Type:  dbmodel.Int64Type,
+				Value: float64(123),
+			},
+			outKv: model.KeyValue{
+				Key:    "int-val",
+				VInt64: 123,
+				VType:  2,
+			},
+		},
+		{
+			name: "right int json number",
+			kv: &dbmodel.KeyValue{
+				Key:   "int-val",
+				Type:  dbmodel.Int64Type,
+				Value: json.Number("123"),
+			},
+			outKv: model.KeyValue{
+				Key:    "int-val",
+				VInt64: 123,
+				VType:  2,
+			},
+		},
+		{
+			name: "right float value",
+			kv: &dbmodel.KeyValue{
+				Key:   "float-val",
+				Type:  dbmodel.Float64Type,
+				Value: 123.4,
+			},
+			outKv: model.KeyValue{
+				Key:      "float-val",
+				VFloat64: 123.4,
+				VType:    3,
+			},
+		},
+		{
+			name: "right float json number",
+			kv: &dbmodel.KeyValue{
+				Key:   "float-val",
+				Type:  dbmodel.Float64Type,
+				Value: json.Number("123.4"),
+			},
+			outKv: model.KeyValue{
+				Key:      "float-val",
+				VFloat64: 123.4,
+				VType:    3,
+			},
+		},
+		{
+			name: "wrong int64 value",
+			kv: &dbmodel.KeyValue{
+				Key:   "int-val",
+				Type:  dbmodel.Int64Type,
+				Value: true,
+			},
+			err: "invalid int64 type in true",
+		},
+		{
+			name: "wrong float64 value",
+			kv: &dbmodel.KeyValue{
+				Key:   "float-val",
+				Type:  dbmodel.Float64Type,
+				Value: true,
+			},
+			err: "invalid float64 type in true",
+		},
+		{
+			name: "wrong bool value",
+			kv: &dbmodel.KeyValue{
+				Key:   "bool-val",
+				Type:  dbmodel.BoolType,
+				Value: 1.23,
+			},
+			err: "invalid bool type in 1.23",
+		},
+		{
+			name: "wrong string value",
+			kv: &dbmodel.KeyValue{
+				Key:   "string-val",
+				Type:  dbmodel.StringType,
 				Value: 123,
 			},
-			err: "non-string Value of type",
+			err: "invalid string type in 123",
 		},
 	}
 	td := ToDomain{}
 	for _, test := range tests {
-		t.Run(test.err, func(t *testing.T) {
+		t.Run(test.name, func(t *testing.T) {
 			tag := test.kv
-			_, err := td.convertKeyValue(tag)
-			assert.ErrorContains(t, err, test.err)
+			out, err := td.convertKeyValue(tag)
+			if test.err != "" {
+				require.ErrorContains(t, err, test.err)
+			}
+			assert.Equal(t, test.outKv, out)
 		})
 	}
 }