Repository: jaegertracing/jaeger

Issue:
[W5]: SPM Support for OpenSearch
### Requirement

Part of Native SPM support in Elasticsearch (https://github.com/jaegertracing/jaeger/issues/6641). Leverage ES query to directly derive metric data from traces
Add SPM support for opensearch

### Problem

Current SPM deployment use Prometheus to store metric data. We can simplify this deployment by directly derive metric data from traces data in ES



### Proposal
- [x] Add option for opensearch inside metrics_backend
- [x] Add docker-compose file to enable spm using opensearch
- [x] Add corresponding e2e test

### Open questions

_No response_

Discussion:
@pipiland2612: Hi @yurishkuro , this issue is subissue of https://github.com/jaegertracing/jaeger/issues/6641

Solution patch:
diff --git a/.github/workflows/ci-e2e-spm.yml b/.github/workflows/ci-e2e-spm.yml
index dccef87a..afbeb394 100644
--- a/.github/workflows/ci-e2e-spm.yml
+++ b/.github/workflows/ci-e2e-spm.yml
@@ -32,6 +32,9 @@ jobs:
         - name: v2 with ES
           binary: jaeger
           metricstore: elasticsearch
+        - name: v2 with OS
+          binary: jaeger
+          metricstore: opensearch
 
     steps:
     - name: Harden Runner
diff --git a/cmd/jaeger/config-spm-opensearch.yaml b/cmd/jaeger/config-spm-opensearch.yaml
new file mode 100644
index 00000000..99db87ec
--- /dev/null
+++ b/cmd/jaeger/config-spm-opensearch.yaml
@@ -0,0 +1,35 @@
+service:
+  extensions: [jaeger_storage, jaeger_query]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [jaeger_storage_exporter]
+
+extensions:
+  jaeger_query:
+    storage:
+      traces: opensearch_trace_storage
+      metrics: opensearch_trace_storage
+  jaeger_storage:
+    backends:
+      opensearch_trace_storage: &opensearch_config
+        opensearch:
+          server_urls:
+            - http://opensearch:9200
+    metric_backends:
+      opensearch_trace_storage: *opensearch_config
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+        endpoint: "0.0.0.0:4318"
+
+processors:
+  batch:
+
+exporters:
+  jaeger_storage_exporter:
+    trace_storage: opensearch_trace_storage
\ No newline at end of file
diff --git a/cmd/jaeger/internal/extension/jaegerstorage/config.go b/cmd/jaeger/internal/extension/jaegerstorage/config.go
index 56a10fe8..c0b420b5 100644
--- a/cmd/jaeger/internal/extension/jaegerstorage/config.go
+++ b/cmd/jaeger/internal/extension/jaegerstorage/config.go
@@ -53,6 +53,7 @@ type TraceBackend struct {
 type MetricBackend struct {
 	Prometheus    *promCfg.Configuration `mapstructure:"prometheus"`
 	Elasticsearch *esCfg.Configuration   `mapstructure:"elasticsearch"`
+	Opensearch    *esCfg.Configuration   `mapstructure:"opensearch"`
 }
 
 // Unmarshal implements confmap.Unmarshaler. This allows us to provide
@@ -122,5 +123,11 @@ func (cfg *MetricBackend) Unmarshal(conf *confmap.Conf) error {
 		v := es.DefaultConfig()
 		cfg.Elasticsearch = &v
 	}
+
+	if conf.IsSet("opensearch") {
+		v := es.DefaultConfig()
+		cfg.Opensearch = &v
+	}
+
 	return conf.Unmarshal(cfg)
 }
diff --git a/cmd/jaeger/internal/extension/jaegerstorage/extension.go b/cmd/jaeger/internal/extension/jaegerstorage/extension.go
index 9f17fcbb..3dc5cfab 100644
--- a/cmd/jaeger/internal/extension/jaegerstorage/extension.go
+++ b/cmd/jaeger/internal/extension/jaegerstorage/extension.go
@@ -223,6 +223,15 @@ func (s *storageExt) Start(ctx context.Context, host component.Host) error {
 				*cfg.Elasticsearch,
 				esTelset,
 			)
+
+		case cfg.Opensearch != nil:
+			osTelset := telset
+			osTelset.Metrics = scopedMetricsFactory(metricStorageName, "opensearch", "metricstore")
+			metricStoreFactory, err = esmetrics.NewFactory(
+				ctx,
+				*cfg.Opensearch,
+				osTelset,
+			)
 		}
 		if err != nil {
 			return fmt.Errorf("failed to initialize metrics storage '%s': %w", metricStorageName, err)
diff --git a/docker-compose/monitor/Makefile b/docker-compose/monitor/Makefile
index 9a0e3b9c..6db063e0 100644
--- a/docker-compose/monitor/Makefile
+++ b/docker-compose/monitor/Makefile
@@ -33,6 +33,11 @@ elasticsearch: export JAEGER_VERSION = dev
 elasticsearch:
 	docker compose -f docker-compose-elasticsearch.yml up $(DOCKER_COMPOSE_ARGS)
 
+.PHONY: opensearch
+opensearch: export JAEGER_VERSION = dev
+opensearch:
+	docker compose -f docker-compose-opensearch.yml up $(DOCKER_COMPOSE_ARGS)
+
 .PHONY: clean-jaeger
 clean-jaeger:
 	# Also cleans up intermediate cached containers.
diff --git a/docker-compose/monitor/docker-compose-opensearch.yml b/docker-compose/monitor/docker-compose-opensearch.yml
new file mode 100644
index 00000000..658c66d5
--- /dev/null
+++ b/docker-compose/monitor/docker-compose-opensearch.yml
@@ -0,0 +1,55 @@
+services:
+  opensearch:
+    image: opensearchproject/opensearch:2.19.0@sha256:1f8b88245a6af61e7aa500afe0e87d43401e4b33140bb47230a919428ce3f7cb
+    networks:
+      - backend
+    environment:
+      - discovery.type=single-node
+      - plugins.security.disabled=true
+      - http.host=0.0.0.0
+      - transport.host=127.0.0.1
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=passRT%^#234
+    ports:
+      - "9200:9200"
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f http://localhost:9200 || exit 1" ]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+
+  jaeger:
+    networks:
+      backend:
+        # This is the host name used in Prometheus scrape configuration.
+        aliases: [ spm_metrics_source ]
+    image: jaegertracing/jaeger:${JAEGER_VERSION:-latest}
+    volumes:
+      - "./jaeger-ui.json:/etc/jaeger/jaeger-ui.json"
+      - "../../cmd/jaeger/config-spm-opensearch.yaml:/etc/jaeger/config.yml"
+    command: ["--config", "/etc/jaeger/config.yml"]
+    ports:
+      - "16686:16686" # Jaeger UI http://localhost:16686
+      - "8888:8888"
+      - "8889:8889"
+      - "4317:4317"
+      - "4318:4318"
+    depends_on:
+      opensearch:
+        condition: service_healthy
+
+  microsim:
+    networks:
+      - backend
+    image: yurishkuro/microsim:v0.5.0@sha256:b7ee2dee51d2c9fd94de08a80278cfbf5a144ad0f22efce50f3d3be15cbfa2c7
+    command: "-d 24h -s 500ms"
+    environment:
+      - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://jaeger:4318/v1/traces
+    depends_on:
+      - jaeger
+
+networks:
+  backend:
+
+volumes:
+  esdata:
+    driver: local
diff --git a/scripts/e2e/spm.sh b/scripts/e2e/spm.sh
index 850ae359..bd0e1d4a 100755
--- a/scripts/e2e/spm.sh
+++ b/scripts/e2e/spm.sh
@@ -8,7 +8,7 @@ set -euf -o pipefail
 print_help() {
   echo "Usage: $0 [-b binary] [-m metricstore]"
   echo "-b: Which binary to build: 'all-in-one' or 'jaeger' (v2, default)"
-  echo "-m: Which database to use as metrics store: 'prometheus' (default) or 'elasticsearch'"
+  echo "-m: Which database to use as metrics store: 'prometheus' (default) or 'elasticsearch' or 'opensearch'"
   echo "-h: Print help"
   exit 1
 }
@@ -47,7 +47,7 @@ esac
 
 # Validate metricstore option
 case "$METRICSTORE" in
-  "prometheus"|"elasticsearch")
+  "prometheus"|"elasticsearch"|"opensearch")
     # Valid options
     ;;
   *)
@@ -67,6 +67,11 @@ if [ "$METRICSTORE" == "elasticsearch" ]; then
   make_target="elasticsearch"
 fi
 
+if [ "$METRICSTORE" == "opensearch" ]; then
+  compose_file=docker-compose/monitor/docker-compose-opensearch.yml
+  make_target="opensearch"
+fi
+
 timeout=600
 end_time=$((SECONDS + timeout))
 success="false"
@@ -100,13 +105,18 @@ check_service_health() {
 # Function to check if all services are healthy
 wait_for_services() {
   echo "Waiting for services to be up and running..."
-  if [ "$METRICSTORE" == "elasticsearch" ]; then
-    check_service_health "Elasticsearch" "http://localhost:9200"
-  fi
 
-  if [ "$METRICSTORE" == "prometheus" ]; then
-    check_service_health "Prometheus" "http://localhost:9090/query"
-  fi
+  case "$METRICSTORE" in
+    "elasticsearch")
+      check_service_health "Elasticsearch" "http://localhost:9200"
+      ;;
+    "opensearch")
+      check_service_health "Opensearch" "http://localhost:9200"
+      ;;
+    "prometheus")
+      check_service_health "Prometheus" "http://localhost:9090/query"
+      ;;
+  esac
 
   check_service_health "Jaeger" "http://localhost:16686"
 }



Test patch:
diff --git a/cmd/jaeger/internal/extension/jaegerstorage/config_test.go b/cmd/jaeger/internal/extension/jaegerstorage/config_test.go
index befb5318..cc3320a6 100644
--- a/cmd/jaeger/internal/extension/jaegerstorage/config_test.go
+++ b/cmd/jaeger/internal/extension/jaegerstorage/config_test.go
@@ -129,3 +129,14 @@ metric_backends:
 	require.NoError(t, conf.Unmarshal(cfg))
 	assert.NotEmpty(t, cfg.MetricBackends["some_metrics_storage"].Elasticsearch.Servers)
 }
+
+func TestConfigDefaultOpenSearchAsMetricsBackend(t *testing.T) {
+	conf := loadConf(t, `
+metric_backends:
+  some_metrics_storage:
+    opensearch:
+`)
+	cfg := createDefaultConfig().(*Config)
+	require.NoError(t, conf.Unmarshal(cfg))
+	assert.NotEmpty(t, cfg.MetricBackends["some_metrics_storage"].Opensearch.Servers)
+}
diff --git a/cmd/jaeger/internal/extension/jaegerstorage/extension_test.go b/cmd/jaeger/internal/extension/jaegerstorage/extension_test.go
index 3c41c80b..d86c448c 100644
--- a/cmd/jaeger/internal/extension/jaegerstorage/extension_test.go
+++ b/cmd/jaeger/internal/extension/jaegerstorage/extension_test.go
@@ -310,39 +310,61 @@ func TestGRPC(t *testing.T) {
 	require.NoError(t, ext.Shutdown(ctx))
 }
 
-func TestPrometheus(t *testing.T) {
-	ext := makeStorageExtension(t, &Config{
-		MetricBackends: map[string]MetricBackend{
-			"foo": {
-				Prometheus: &promCfg.Configuration{
-					ServerURL: "localhost:12345",
+func TestMetricBackends(t *testing.T) {
+	mockServer := setupMockServer(t, getVersionResponse(t), http.StatusOK)
+	tests := []struct {
+		name   string
+		config *Config
+	}{
+		{
+			name: "Prometheus",
+			config: &Config{
+				MetricBackends: map[string]MetricBackend{
+					"foo": {
+						Prometheus: &promCfg.Configuration{
+							ServerURL: mockServer.URL,
+						},
+					},
 				},
 			},
 		},
-	})
-	ctx := context.Background()
-	err := ext.Start(ctx, componenttest.NewNopHost())
-	require.NoError(t, err)
-	require.NoError(t, ext.Shutdown(ctx))
-}
-
-func TestElasticsearchAsMetricsBackend(t *testing.T) {
-	server := setupMockServer(t, getVersionResponse(t), http.StatusOK)
-
-	ext := makeStorageExtension(t, &Config{
-		MetricBackends: map[string]MetricBackend{
-			"foo": {
-				Elasticsearch: &esCfg.Configuration{
-					Servers:  []string{server.URL},
-					LogLevel: "info",
+		{
+			name: "Elasticsearch",
+			config: &Config{
+				MetricBackends: map[string]MetricBackend{
+					"foo": {
+						Elasticsearch: &esCfg.Configuration{
+							Servers:  []string{mockServer.URL},
+							LogLevel: "info",
+						},
+					},
 				},
 			},
 		},
-	})
-	ctx := context.Background()
-	err := ext.Start(ctx, componenttest.NewNopHost())
-	require.NoError(t, err)
-	require.NoError(t, ext.Shutdown(ctx))
+		{
+			name: "OpenSearch",
+			config: &Config{
+				MetricBackends: map[string]MetricBackend{
+					"foo": {
+						Opensearch: &esCfg.Configuration{
+							Servers:  []string{mockServer.URL},
+							LogLevel: "info",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ext := makeStorageExtension(t, tt.config)
+			ctx := context.Background()
+			err := ext.Start(ctx, componenttest.NewNopHost())
+			require.NoError(t, err)
+			require.NoError(t, ext.Shutdown(ctx))
+		})
+	}
 }
 
 func TestMetricsBackendCloseError(t *testing.T) {
@@ -393,6 +415,16 @@ func TestMetricStorageStartError(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "OpenSearch backend initialization error",
+			config: &Config{
+				MetricBackends: map[string]MetricBackend{
+					"foo": {
+						Opensearch: &esCfg.Configuration{},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {