Repository: jaegertracing/jaeger

Issue:
Remove dependency on jaeger-lib
There are several issues with the dependency on jaeger-lib:
  * it is shared with jaeger-client-go, which is now deprecated, so the library does not need to be standalone
  * library does not use go modules, migrating it to `go mod` is non-trivial because it's already v2
  * because it cannot be migrated to go modules, it cannot be upgraded to later versions of go-kit. The older 0.11 version of go-kit has dependencies with known vulnerabilities
  * there's not much need to depend on go-kit in the first place, we're only using it for its expvar package, which is relatively trivial to implement (especially without the histograms)

There are two directions we can take:
  1. migrate to OpenTelemetry instead. This is preferable, but will involve non-trivial amount of work. Good news is that work can likely be done piecemeal.
  2. Keep using jaeger-lib API for metrics and pull it inside this repo as `internal`.
    * unfortunately, this does not fully remove dependency on jaeger-lib because we're still using jaeger-client-go (cf. #3381, #3380)
  3. Keep using jaeger-lib API for metrics but implement it with OpenTelemetry SDK. This seems like the least resistance path (need to prototype it to test viability).

(Update 2023-01-29) Current plan:
  * [x] Clone jaeger-lib into Jaeger repo (done in #3767, #3772)
  * [x] Upgrade HotROD to use OTEL SDK (via OT bridge), which removes the need for uber/jaeger-lib (done in #4187)
  * [x] Upgrade tracegen to use OTEL SDK (done in #4189)
  * [x] Upgrade collector/query/all-in-one to use OTEL SDK (via OT bridge)
  * [x] Fix crossdock (maybe we don't need SDK testing there anymore)

Discussion:

Solution patch:
diff --git a/.gitignore b/.gitignore
index 38127a8b..95e336f3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,6 @@
+go.work
+go.work.sum
+
 *.out
 *.test
 *.xml
diff --git a/examples/hotrod/cmd/customer.go b/examples/hotrod/cmd/customer.go
index ebdc5a71..56f77bfe 100644
--- a/examples/hotrod/cmd/customer.go
+++ b/examples/hotrod/cmd/customer.go
@@ -23,7 +23,6 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/log"
-	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/tracing"
 	"github.com/jaegertracing/jaeger/examples/hotrod/services/customer"
 )
 
@@ -37,7 +36,7 @@ var customerCmd = &cobra.Command{
 		logger := log.NewFactory(zapLogger)
 		server := customer.NewServer(
 			net.JoinHostPort("0.0.0.0", strconv.Itoa(customerPort)),
-			tracing.Init("customer", metricsFactory, logger),
+			otelExporter,
 			metricsFactory,
 			logger,
 		)
diff --git a/examples/hotrod/cmd/driver.go b/examples/hotrod/cmd/driver.go
index 5e0aa2b5..ac1bf5ae 100644
--- a/examples/hotrod/cmd/driver.go
+++ b/examples/hotrod/cmd/driver.go
@@ -23,7 +23,6 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/log"
-	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/tracing"
 	"github.com/jaegertracing/jaeger/examples/hotrod/services/driver"
 )
 
@@ -37,7 +36,7 @@ var driverCmd = &cobra.Command{
 		logger := log.NewFactory(zapLogger)
 		server := driver.NewServer(
 			net.JoinHostPort("0.0.0.0", strconv.Itoa(driverPort)),
-			tracing.Init("driver", metricsFactory, logger),
+			otelExporter,
 			metricsFactory,
 			logger,
 		)
diff --git a/examples/hotrod/cmd/frontend.go b/examples/hotrod/cmd/frontend.go
index 273b1561..edf44501 100644
--- a/examples/hotrod/cmd/frontend.go
+++ b/examples/hotrod/cmd/frontend.go
@@ -45,7 +45,7 @@ var frontendCmd = &cobra.Command{
 		logger := log.NewFactory(zapLogger)
 		server := frontend.NewServer(
 			options,
-			tracing.Init("frontend", metricsFactory, logger),
+			tracing.Init("frontend", otelExporter, metricsFactory, logger),
 			logger,
 		)
 		return logError(zapLogger, server.Run())
diff --git a/examples/hotrod/cmd/root.go b/examples/hotrod/cmd/root.go
index 68b363ad..81d5ec86 100644
--- a/examples/hotrod/cmd/root.go
+++ b/examples/hotrod/cmd/root.go
@@ -21,19 +21,20 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
-	"github.com/uber/jaeger-lib/metrics"
-	jexpvar "github.com/uber/jaeger-lib/metrics/expvar"
-	jprom "github.com/uber/jaeger-lib/metrics/prometheus"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
 	"github.com/jaegertracing/jaeger/examples/hotrod/services/config"
+	"github.com/jaegertracing/jaeger/internal/metrics/expvar"
+	"github.com/jaegertracing/jaeger/internal/metrics/prometheus"
+	"github.com/jaegertracing/jaeger/pkg/metrics"
 )
 
 var (
 	metricsBackend string
 	logger         *zap.Logger
 	metricsFactory metrics.Factory
+	otelExporter   string // jaeger, otlp, stdout
 
 	fixDBConnDelay         time.Duration
 	fixDBConnDisableMutex  bool
@@ -66,6 +67,8 @@ func Execute() {
 
 func init() {
 	RootCmd.PersistentFlags().StringVarP(&metricsBackend, "metrics", "m", "expvar", "Metrics backend (expvar|prometheus)")
+	RootCmd.PersistentFlags().StringVarP(&otelExporter, "otel-exporter", "x", "jaeger", "OpenTelemetry exporter (jaeger|otlp|stdout)")
+
 	RootCmd.PersistentFlags().DurationVarP(&fixDBConnDelay, "fix-db-query-delay", "D", 300*time.Millisecond, "Average latency of MySQL DB query")
 	RootCmd.PersistentFlags().BoolVarP(&fixDBConnDisableMutex, "fix-disable-db-conn-mutex", "M", false, "Disables the mutex guarding db connection")
 	RootCmd.PersistentFlags().IntVarP(&fixRouteWorkerPoolSize, "fix-route-worker-pool-size", "W", 3, "Default worker pool size")
@@ -92,10 +95,10 @@ func init() {
 func onInitialize() {
 	switch metricsBackend {
 	case "expvar":
-		metricsFactory = jexpvar.NewFactory(10) // 10 buckets for histograms
+		metricsFactory = expvar.NewFactory(10) // 10 buckets for histograms
 		logger.Info("Using expvar as metrics backend")
 	case "prometheus":
-		metricsFactory = jprom.New().Namespace(metrics.NSOptions{Name: "hotrod", Tags: nil})
+		metricsFactory = prometheus.New().Namespace(metrics.NSOptions{Name: "hotrod", Tags: nil})
 		logger.Info("Using Prometheus as metrics backend")
 	default:
 		logger.Fatal("unsupported metrics backend " + metricsBackend)
diff --git a/examples/hotrod/cmd/route.go b/examples/hotrod/cmd/route.go
index b962de6e..ce54159f 100644
--- a/examples/hotrod/cmd/route.go
+++ b/examples/hotrod/cmd/route.go
@@ -37,7 +37,7 @@ var routeCmd = &cobra.Command{
 		logger := log.NewFactory(zapLogger)
 		server := route.NewServer(
 			net.JoinHostPort("0.0.0.0", strconv.Itoa(routePort)),
-			tracing.Init("route", metricsFactory, logger),
+			tracing.Init("route", otelExporter, metricsFactory, logger),
 			logger,
 		)
 		return logError(zapLogger, server.Run())
diff --git a/examples/hotrod/pkg/log/factory.go b/examples/hotrod/pkg/log/factory.go
index d32a98a9..2c85af90 100644
--- a/examples/hotrod/pkg/log/factory.go
+++ b/examples/hotrod/pkg/log/factory.go
@@ -18,8 +18,8 @@ package log
 import (
 	"context"
 
-	"github.com/opentracing/opentracing-go"
-	"github.com/uber/jaeger-client-go"
+	ot "github.com/opentracing/opentracing-go"
+	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 )
@@ -44,13 +44,13 @@ func (b Factory) Bg() Logger {
 // contains an OpenTracing span, all logging calls are also
 // echo-ed into the span.
 func (b Factory) For(ctx context.Context) Logger {
-	if span := opentracing.SpanFromContext(ctx); span != nil {
-		logger := spanLogger{span: span, logger: b.logger}
+	if otSpan := ot.SpanFromContext(ctx); otSpan != nil {
+		logger := spanLogger{span: otSpan, logger: b.logger}
 
-		if jaegerCtx, ok := span.Context().(jaeger.SpanContext); ok {
+		if otelSpan := trace.SpanFromContext(ctx); otelSpan != nil {
 			logger.spanFields = []zapcore.Field{
-				zap.String("trace_id", jaegerCtx.TraceID().String()),
-				zap.String("span_id", jaegerCtx.SpanID().String()),
+				zap.String("trace_id", otelSpan.SpanContext().TraceID().String()),
+				zap.String("span_id", otelSpan.SpanContext().SpanID().String()),
 			}
 		}
 
diff --git a/examples/hotrod/pkg/tracing/init.go b/examples/hotrod/pkg/tracing/init.go
index ac7eea46..eb201e9d 100644
--- a/examples/hotrod/pkg/tracing/init.go
+++ b/examples/hotrod/pkg/tracing/init.go
@@ -16,56 +16,75 @@
 package tracing
 
 import (
+	"context"
 	"fmt"
-	"time"
+	"sync"
 
 	"github.com/opentracing/opentracing-go"
-	"github.com/uber/jaeger-client-go/config"
-	"github.com/uber/jaeger-client-go/rpcmetrics"
-	"github.com/uber/jaeger-lib/metrics"
+	"go.opentelemetry.io/otel"
+	otbridge "go.opentelemetry.io/otel/bridge/opentracing"
+	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
 	"go.uber.org/zap"
 
 	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/log"
+	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/tracing/rpcmetrics"
+	"github.com/jaegertracing/jaeger/pkg/metrics"
 )
 
-// Init creates a new instance of Jaeger tracer.
-func Init(serviceName string, metricsFactory metrics.Factory, logger log.Factory) opentracing.Tracer {
-	cfg := &config.Configuration{
-		Sampler: &config.SamplerConfig{},
-	}
-	cfg.ServiceName = serviceName
-	cfg.Sampler.Type = "const"
-	cfg.Sampler.Param = 1
+var once sync.Once
 
-	_, err := cfg.FromEnv()
-	if err != nil {
-		logger.Bg().Fatal("cannot parse Jaeger env vars", zap.Error(err))
-	}
+// Init initializes OpenTelemetry SDK and uses OTel-OpenTracing Bridge
+// to return an OpenTracing-compatible tracer.
+func Init(serviceName string, exporterType string, metricsFactory metrics.Factory, logger log.Factory) opentracing.Tracer {
+	once.Do(func() {
+		otel.SetTextMapPropagator(propagation.TraceContext{})
+	})
 
-	// TODO(ys) a quick hack to ensure random generators get different seeds, which are based on current time.
-	time.Sleep(100 * time.Millisecond)
-	jaegerLogger := jaegerLoggerAdapter{logger.Bg()}
-
-	metricsFactory = metricsFactory.Namespace(metrics.NSOptions{Name: serviceName, Tags: nil})
-	tracer, _, err := cfg.NewTracer(
-		config.Logger(jaegerLogger),
-		config.Metrics(metricsFactory),
-		config.Observer(rpcmetrics.NewObserver(metricsFactory, rpcmetrics.DefaultNameNormalizer)),
-	)
+	exp, err := createOtelExporter(exporterType)
 	if err != nil {
-		logger.Bg().Fatal("cannot initialize Jaeger Tracer", zap.Error(err))
+		logger.Bg().Fatal("cannot create exporter", zap.String("exporterType", exporterType), zap.Error(err))
 	}
-	return tracer
-}
+	logger.Bg().Info("using " + exporterType + " trace exporter")
 
-type jaegerLoggerAdapter struct {
-	logger log.Logger
-}
+	rpcmetricsObserver := rpcmetrics.NewObserver(metricsFactory, rpcmetrics.DefaultNameNormalizer)
 
-func (l jaegerLoggerAdapter) Error(msg string) {
-	l.logger.Error(msg)
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(exp),
+		sdktrace.WithSpanProcessor(rpcmetricsObserver),
+		sdktrace.WithResource(resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceNameKey.String(serviceName),
+		)),
+	)
+	otTracer, _ := otbridge.NewTracerPair(tp.Tracer(""))
+	logger.Bg().Info("created OTEL->OT brige", zap.String("service-name", serviceName))
+	return otTracer
 }
 
-func (l jaegerLoggerAdapter) Infof(msg string, args ...interface{}) {
-	l.logger.Info(fmt.Sprintf(msg, args...))
+func createOtelExporter(exporterType string) (sdktrace.SpanExporter, error) {
+	var exporter sdktrace.SpanExporter
+	var err error
+	switch exporterType {
+	case "jaeger":
+		exporter, err = jaeger.New(
+			jaeger.WithCollectorEndpoint(),
+		)
+	case "otlp":
+		client := otlptracehttp.NewClient(
+			otlptracehttp.WithInsecure(),
+		)
+		exporter, err = otlptrace.New(context.Background(), client)
+	case "stdout":
+		exporter, err = stdouttrace.New()
+	default:
+		return nil, fmt.Errorf("unrecognized exporter type %s", exporterType)
+	}
+	return exporter, err
 }
diff --git a/examples/hotrod/pkg/tracing/rpcmetrics/README.md b/examples/hotrod/pkg/tracing/rpcmetrics/README.md
new file mode 100644
index 00000000..34126b69
--- /dev/null
+++ b/examples/hotrod/pkg/tracing/rpcmetrics/README.md
@@ -0,0 +1,3 @@
+Package rpcmetrics implements an OpenTelemetry SpanProcessor that can be used to emit RPC metrics.
+
+This package is copied from jaeger-client-go and adapted to work with OpenTelemtery SDK.
diff --git a/examples/hotrod/pkg/tracing/rpcmetrics/endpoints.go b/examples/hotrod/pkg/tracing/rpcmetrics/endpoints.go
new file mode 100644
index 00000000..7ca73bfa
--- /dev/null
+++ b/examples/hotrod/pkg/tracing/rpcmetrics/endpoints.go
@@ -0,0 +1,63 @@
+// Copyright (c) 2023 The Jaeger Authors.
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpcmetrics
+
+import "sync"
+
+// normalizedEndpoints is a cache for endpointName -> safeName mappings.
+type normalizedEndpoints struct {
+	names      map[string]string
+	maxSize    int
+	normalizer NameNormalizer
+	mux        sync.RWMutex
+}
+
+func newNormalizedEndpoints(maxSize int, normalizer NameNormalizer) *normalizedEndpoints {
+	return &normalizedEndpoints{
+		maxSize:    maxSize,
+		normalizer: normalizer,
+		names:      make(map[string]string, maxSize),
+	}
+}
+
+// normalize looks up the name in the cache, if not found it uses normalizer
+// to convert the name to a safe name. If called with more than maxSize unique
+// names it returns "" for all other names beyond those already cached.
+func (n *normalizedEndpoints) normalize(name string) string {
+	n.mux.RLock()
+	norm, ok := n.names[name]
+	l := len(n.names)
+	n.mux.RUnlock()
+	if ok {
+		return norm
+	}
+	if l >= n.maxSize {
+		return ""
+	}
+	return n.normalizeWithLock(name)
+}
+
+func (n *normalizedEndpoints) normalizeWithLock(name string) string {
+	norm := n.normalizer.Normalize(name)
+	n.mux.Lock()
+	defer n.mux.Unlock()
+	// cache may have grown while we were not holding the lock
+	if len(n.names) >= n.maxSize {
+		return ""
+	}
+	n.names[name] = norm
+	return norm
+}
diff --git a/examples/hotrod/pkg/tracing/rpcmetrics/metrics.go b/examples/hotrod/pkg/tracing/rpcmetrics/metrics.go
new file mode 100644
index 00000000..83739876
--- /dev/null
+++ b/examples/hotrod/pkg/tracing/rpcmetrics/metrics.go
@@ -0,0 +1,125 @@
+// Copyright (c) 2023 The Jaeger Authors.
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpcmetrics
+
+import (
+	"sync"
+
+	"github.com/jaegertracing/jaeger/pkg/metrics"
+)
+
+const (
+	otherEndpointsPlaceholder = "other"
+	endpointNameMetricTag     = "endpoint"
+)
+
+// Metrics is a collection of metrics for an endpoint describing
+// throughput, success, errors, and performance.
+type Metrics struct {
+	// RequestCountSuccess is a counter of the total number of successes.
+	RequestCountSuccess metrics.Counter `metric:"requests" tags:"error=false"`
+
+	// RequestCountFailures is a counter of the number of times any failure has been observed.
+	RequestCountFailures metrics.Counter `metric:"requests" tags:"error=true"`
+
+	// RequestLatencySuccess is a latency histogram of successful requests.
+	RequestLatencySuccess metrics.Timer `metric:"request_latency" tags:"error=false"`
+
+	// RequestLatencyFailures is a latency histogram of failed requests.
+	RequestLatencyFailures metrics.Timer `metric:"request_latency" tags:"error=true"`
+
+	// HTTPStatusCode2xx is a counter of the total number of requests with HTTP status code 200-299
+	HTTPStatusCode2xx metrics.Counter `metric:"http_requests" tags:"status_code=2xx"`
+
+	// HTTPStatusCode3xx is a counter of the total number of requests with HTTP status code 300-399
+	HTTPStatusCode3xx metrics.Counter `metric:"http_requests" tags:"status_code=3xx"`
+
+	// HTTPStatusCode4xx is a counter of the total number of requests with HTTP status code 400-499
+	HTTPStatusCode4xx metrics.Counter `metric:"http_requests" tags:"status_code=4xx"`
+
+	// HTTPStatusCode5xx is a counter of the total number of requests with HTTP status code 500-599
+	HTTPStatusCode5xx metrics.Counter `metric:"http_requests" tags:"status_code=5xx"`
+}
+
+func (m *Metrics) recordHTTPStatusCode(statusCode int64) {
+	if statusCode >= 200 && statusCode < 300 {
+		m.HTTPStatusCode2xx.Inc(1)
+	} else if statusCode >= 300 && statusCode < 400 {
+		m.HTTPStatusCode3xx.Inc(1)
+	} else if statusCode >= 400 && statusCode < 500 {
+		m.HTTPStatusCode4xx.Inc(1)
+	} else if statusCode >= 500 && statusCode < 600 {
+		m.HTTPStatusCode5xx.Inc(1)
+	}
+}
+
+// MetricsByEndpoint is a registry/cache of metrics for each unique endpoint name.
+// Only maxNumberOfEndpoints Metrics are stored, all other endpoint names are mapped
+// to a generic endpoint name "other".
+type MetricsByEndpoint struct {
+	metricsFactory    metrics.Factory
+	endpoints         *normalizedEndpoints
+	metricsByEndpoint map[string]*Metrics
+	mux               sync.RWMutex
+}
+
+func newMetricsByEndpoint(
+	metricsFactory metrics.Factory,
+	normalizer NameNormalizer,
+	maxNumberOfEndpoints int,
+) *MetricsByEndpoint {
+	return &MetricsByEndpoint{
+		metricsFactory:    metricsFactory,
+		endpoints:         newNormalizedEndpoints(maxNumberOfEndpoints, normalizer),
+		metricsByEndpoint: make(map[string]*Metrics, maxNumberOfEndpoints+1), // +1 for "other"
+	}
+}
+
+func (m *MetricsByEndpoint) get(endpoint string) *Metrics {
+	safeName := m.endpoints.normalize(endpoint)
+	if safeName == "" {
+		safeName = otherEndpointsPlaceholder
+	}
+	m.mux.RLock()
+	met := m.metricsByEndpoint[safeName]
+	m.mux.RUnlock()
+	if met != nil {
+		return met
+	}
+
+	return m.getWithWriteLock(safeName)
+}
+
+// split to make easier to test
+func (m *MetricsByEndpoint) getWithWriteLock(safeName string) *Metrics {
+	m.mux.Lock()
+	defer m.mux.Unlock()
+
+	// it is possible that the name has been already registered after we released
+	// the read lock and before we grabbed the write lock, so check for that.
+	if met, ok := m.metricsByEndpoint[safeName]; ok {
+		return met
+	}
+
+	// it would be nice to create the struct before locking, since Init() is somewhat
+	// expensive, however some metrics backends (e.g. expvar) may not like duplicate metrics.
+	met := &Metrics{}
+	tags := map[string]string{endpointNameMetricTag: safeName}
+	metrics.Init(met, m.metricsFactory, tags)
+
+	m.metricsByEndpoint[safeName] = met
+	return met
+}
diff --git a/examples/hotrod/pkg/tracing/rpcmetrics/normalizer.go b/examples/hotrod/pkg/tracing/rpcmetrics/normalizer.go
new file mode 100644
index 00000000..01f5bcd7
--- /dev/null
+++ b/examples/hotrod/pkg/tracing/rpcmetrics/normalizer.go
@@ -0,0 +1,101 @@
+// Copyright (c) 2023 The Jaeger Authors.
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpcmetrics
+
+// NameNormalizer is used to convert the endpoint names to strings
+// that can be safely used as tags in the metrics.
+type NameNormalizer interface {
+	Normalize(name string) string
+}
+
+// DefaultNameNormalizer converts endpoint names so that they contain only characters
+// from the safe charset [a-zA-Z0-9./_]. All other characters are replaced with '_'.
+var DefaultNameNormalizer = &SimpleNameNormalizer{
+	SafeSets: []SafeCharacterSet{
+		&Range{From: 'a', To: 'z'},
+		&Range{From: 'A', To: 'Z'},
+		&Range{From: '0', To: '9'},
+		&Char{'_'},
+		&Char{'/'},
+		&Char{'.'},
+	},
+	Replacement: '_',
+}
+
+// SimpleNameNormalizer uses a set of safe character sets.
+type SimpleNameNormalizer struct {
+	SafeSets    []SafeCharacterSet
+	Replacement byte
+}
+
+// SafeCharacterSet determines if the given character is "safe"
+type SafeCharacterSet interface {
+	IsSafe(c byte) bool
+}
+
+// Range implements SafeCharacterSet
+type Range struct {
+	From, To byte
+}
+
+// IsSafe implements SafeCharacterSet
+func (r *Range) IsSafe(c byte) bool {
+	return c >= r.From && c <= r.To
+}
+
+// Char implements SafeCharacterSet
+type Char struct {
+	Val byte
+}
+
+// IsSafe implements SafeCharacterSet
+func (ch *Char) IsSafe(c byte) bool {
+	return c == ch.Val
+}
+
+// Normalize checks each character in the string against SafeSets,
+// and if it's not safe substitutes it with Replacement.
+func (n *SimpleNameNormalizer) Normalize(name string) string {
+	var retMe []byte
+	nameBytes := []byte(name)
+	for i, b := range nameBytes {
+		if n.safeByte(b) {
+			if retMe != nil {
+				retMe[i] = b
+			}
+		} else {
+			if retMe == nil {
+				retMe = make([]byte, len(nameBytes))
+				copy(retMe[0:i], nameBytes[0:i])
+			}
+			retMe[i] = n.Replacement
+		}
+	}
+	if retMe == nil {
+		return name
+	}
+	return string(retMe)
+}
+
+// safeByte checks if b against all safe charsets.
+func (n *SimpleNameNormalizer) safeByte(b byte) bool {
+	for i := range n.SafeSets {
+		if n.SafeSets[i].IsSafe(b) {
+			return true
+		}
+	}
+	return false
+}
diff --git a/examples/hotrod/pkg/tracing/rpcmetrics/observer.go b/examples/hotrod/pkg/tracing/rpcmetrics/observer.go
new file mode 100644
index 00000000..383660e1
--- /dev/null
+++ b/examples/hotrod/pkg/tracing/rpcmetrics/observer.go
@@ -0,0 +1,92 @@
+// Copyright (c) 2023 The Jaeger Authors.
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpcmetrics
+
+import (
+	"context"
+	"strconv"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/jaegertracing/jaeger/pkg/metrics"
+)
+
+const defaultMaxNumberOfEndpoints = 200
+
+var _ sdktrace.SpanProcessor = (*Observer)(nil)
+
+// Observer is an observer that can emit RPC metrics.
+type Observer struct {
+	metricsByEndpoint *MetricsByEndpoint
+}
+
+// NewObserver creates a new observer that can emit RPC metrics.
+func NewObserver(metricsFactory metrics.Factory, normalizer NameNormalizer) *Observer {
+	return &Observer{
+		metricsByEndpoint: newMetricsByEndpoint(
+			metricsFactory,
+			normalizer,
+			defaultMaxNumberOfEndpoints,
+		),
+	}
+}
+
+func (o *Observer) OnStart(parent context.Context, s sdktrace.ReadWriteSpan) {}
+
+func (o *Observer) OnEnd(sp sdktrace.ReadOnlySpan) {
+	operationName := sp.Name()
+	if operationName == "" {
+		return
+	}
+	if sp.SpanKind() != trace.SpanKindServer {
+		return
+	}
+
+	mets := o.metricsByEndpoint.get(operationName)
+	latency := sp.EndTime().Sub(sp.StartTime())
+
+	if status := sp.Status(); status.Code == codes.Error {
+		mets.RequestCountFailures.Inc(1)
+		mets.RequestLatencyFailures.Record(latency)
+	} else {
+		mets.RequestCountSuccess.Inc(1)
+		mets.RequestLatencySuccess.Record(latency)
+	}
+	for _, attr := range sp.Attributes() {
+		if string(attr.Key) == string(semconv.HTTPStatusCodeKey) {
+			if attr.Value.Type() == attribute.INT64 {
+				mets.recordHTTPStatusCode(attr.Value.AsInt64())
+			} else if attr.Value.Type() == attribute.STRING {
+				s := attr.Value.AsString()
+				if n, err := strconv.Atoi(s); err == nil {
+					mets.recordHTTPStatusCode(int64(n))
+				}
+			}
+		}
+	}
+}
+
+func (o *Observer) Shutdown(ctx context.Context) error {
+	return nil
+}
+
+func (o *Observer) ForceFlush(ctx context.Context) error {
+	return nil
+}
diff --git a/examples/hotrod/services/customer/server.go b/examples/hotrod/services/customer/server.go
index 264bfc74..f35a15c3 100644
--- a/examples/hotrod/services/customer/server.go
+++ b/examples/hotrod/services/customer/server.go
@@ -20,12 +20,12 @@ import (
 	"net/http"
 
 	"github.com/opentracing/opentracing-go"
-	"github.com/uber/jaeger-lib/metrics"
 	"go.uber.org/zap"
 
 	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/httperr"
 	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/log"
 	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/tracing"
+	"github.com/jaegertracing/jaeger/pkg/metrics"
 )
 
 // Server implements Customer service
@@ -37,13 +37,13 @@ type Server struct {
 }
 
 // NewServer creates a new customer.Server
-func NewServer(hostPort string, tracer opentracing.Tracer, metricsFactory metrics.Factory, logger log.Factory) *Server {
+func NewServer(hostPort string, otelExporter string, metricsFactory metrics.Factory, logger log.Factory) *Server {
 	return &Server{
 		hostPort: hostPort,
-		tracer:   tracer,
+		tracer:   tracing.Init("customer", otelExporter, metricsFactory, logger),
 		logger:   logger,
 		database: newDatabase(
-			tracing.Init("mysql", metricsFactory, logger),
+			tracing.Init("mysql", otelExporter, metricsFactory, logger),
 			logger.With(zap.String("component", "mysql")),
 		),
 	}
diff --git a/examples/hotrod/services/driver/redis.go b/examples/hotrod/services/driver/redis.go
index cefef66b..ae297851 100644
--- a/examples/hotrod/services/driver/redis.go
+++ b/examples/hotrod/services/driver/redis.go
@@ -24,13 +24,13 @@ import (
 
 	"github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
-	"github.com/uber/jaeger-lib/metrics"
 	"go.uber.org/zap"
 
 	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/delay"
 	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/log"
 	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/tracing"
 	"github.com/jaegertracing/jaeger/examples/hotrod/services/config"
+	"github.com/jaegertracing/jaeger/pkg/metrics"
 )
 
 // Redis is a simulator of remote Redis cache
@@ -40,9 +40,9 @@ type Redis struct {
 	errorSimulator
 }
 
-func newRedis(metricsFactory metrics.Factory, logger log.Factory) *Redis {
+func newRedis(otelExporter string, metricsFactory metrics.Factory, logger log.Factory) *Redis {
 	return &Redis{
-		tracer: tracing.Init("redis", metricsFactory, logger),
+		tracer: tracing.Init("redis", otelExporter, metricsFactory, logger),
 		logger: logger,
 	}
 }
diff --git a/examples/hotrod/services/driver/server.go b/examples/hotrod/services/driver/server.go
index 99d32524..da630555 100644
--- a/examples/hotrod/services/driver/server.go
+++ b/examples/hotrod/services/driver/server.go
@@ -21,11 +21,12 @@ import (
 
 	otgrpc "github.com/opentracing-contrib/go-grpc"
 	"github.com/opentracing/opentracing-go"
-	"github.com/uber/jaeger-lib/metrics"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 
 	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/log"
+	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/tracing"
+	"github.com/jaegertracing/jaeger/pkg/metrics"
 )
 
 // Server implements jaeger-demo-frontend service
@@ -40,7 +41,8 @@ type Server struct {
 var _ DriverServiceServer = (*Server)(nil)
 
 // NewServer creates a new driver.Server
-func NewServer(hostPort string, tracer opentracing.Tracer, metricsFactory metrics.Factory, logger log.Factory) *Server {
+func NewServer(hostPort string, otelExporter string, metricsFactory metrics.Factory, logger log.Factory) *Server {
+	tracer := tracing.Init("driver", otelExporter, metricsFactory, logger)
 	server := grpc.NewServer(grpc.UnaryInterceptor(
 		otgrpc.OpenTracingServerInterceptor(tracer)),
 		grpc.StreamInterceptor(
@@ -50,7 +52,7 @@ func NewServer(hostPort string, tracer opentracing.Tracer, metricsFactory metric
 		tracer:   tracer,
 		logger:   logger,
 		server:   server,
-		redis:    newRedis(metricsFactory, logger),
+		redis:    newRedis(otelExporter, metricsFactory, logger),
 	}
 }
 
diff --git a/examples/hotrod/services/frontend/web_assets/favicon.ico b/examples/hotrod/services/frontend/web_assets/favicon.ico
new file mode 100644
index 00000000..caa96515
Binary files /dev/null and b/examples/hotrod/services/frontend/web_assets/favicon.ico differ
diff --git a/go.mod b/go.mod
index 27bdd045..be83adcf 100644
--- a/go.mod
+++ b/go.mod
@@ -54,7 +54,14 @@ require (
 	go.opentelemetry.io/collector/receiver/otlpreceiver v0.70.0
 	go.opentelemetry.io/collector/semconv v0.70.0
 	go.opentelemetry.io/otel v1.11.2
+	go.opentelemetry.io/otel/bridge/opentracing v1.11.2
+	go.opentelemetry.io/otel/exporters/jaeger v1.11.2
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2
 	go.opentelemetry.io/otel/metric v0.34.0
+	go.opentelemetry.io/otel/sdk v1.11.2
+	go.opentelemetry.io/otel/trace v1.11.2
 	go.uber.org/atomic v1.10.0
 	go.uber.org/automaxprocs v1.5.1
 	go.uber.org/zap v1.24.0
@@ -70,6 +77,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
@@ -86,10 +94,11 @@ require (
 	github.com/go-openapi/analysis v0.21.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v1.12.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -146,7 +155,8 @@ require (
 	go.opentelemetry.io/collector/featuregate v0.70.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 // indirect
-	go.opentelemetry.io/otel/trace v1.11.2 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
 	golang.org/x/text v0.6.0 // indirect
diff --git a/go.sum b/go.sum
index 41887c8d..d15ec317 100644
--- a/go.sum
+++ b/go.sum
@@ -94,6 +94,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/bsm/sarama-cluster v2.1.13+incompatible h1:bqU3gMJbWZVxLZ9PGWVKP05yOmFXUlfw61RBwuE3PYU=
 github.com/bsm/sarama-cluster v2.1.13+incompatible/go.mod h1:r7ao+4tTNXvWm+VRpRJchr2kQhqxgmAp2iEX5W96gMM=
+github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
+github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -108,6 +110,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
@@ -147,6 +150,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -253,8 +257,9 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -339,6 +344,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
@@ -728,14 +735,29 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 h1:yt2NKzK
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0/go.mod h1:+ARmXlUlc51J7sZeCBkBJNdHGySrdOzgzxp6VWRWM1U=
 go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0=
 go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI=
+go.opentelemetry.io/otel/bridge/opentracing v1.11.2 h1:Wx51zQDSZDNo5wxMPhkPwzgpUZLQYYDtT41LCcl7opg=
+go.opentelemetry.io/otel/bridge/opentracing v1.11.2/go.mod h1:kBrIQ2vqDIqtuS7Np7ALjmm8Tml7yxgsAGQwBhNvuU0=
+go.opentelemetry.io/otel/exporters/jaeger v1.11.2 h1:ES8/j2+aB+3/BUw51ioxa50V9btN1eew/2J7N7n1tsE=
+go.opentelemetry.io/otel/exporters/jaeger v1.11.2/go.mod h1:nwcF/DK4Hk0auZ/a5vw20uMsaJSXbzeeimhN5f9d0Lc=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 h1:htgM8vZIF8oPSCxa341e3IZ4yr/sKxgu8KZYllByiVY=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2/go.mod h1:rqbht/LlhVBgn5+k3M5QK96K5Xb0DvXpMJ5SFQpY6uw=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 h1:fqR1kli93643au1RKo0Uma3d2aPQKT+WBKfTSBaKbOc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2/go.mod h1:5Qn6qvgkMsLDX+sYK64rHb1FPhpn0UtxF+ouX1uhyJE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2 h1:Us8tbCmuN16zAnK5TC69AtODLycKbwnskQzaB6DfFhc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2/go.mod h1:GZWSQQky8AgdJj50r1KJm8oiQiIPaAX7uZCFQX9GzC8=
 go.opentelemetry.io/otel/exporters/prometheus v0.34.0 h1:L5D+HxdaC/ORB47ribbTBbkXRZs9JzPjq0EoIOMWncM=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2 h1:BhEVgvuE1NWLLuMLvC6sif791F45KFHi5GhOs1KunZU=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2/go.mod h1:bx//lU66dPzNT+Y0hHA12ciKoMOH9iixEwCqC1OeQWQ=
 go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8=
 go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8=
 go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU=
+go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU=
 go.opentelemetry.io/otel/sdk/metric v0.34.0 h1:7ElxfQpXCFZlRTvVRTkcUvK8Gt5DC8QzmzsLsO2gdzo=
 go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0=
 go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=
+go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
@@ -868,6 +890,7 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1111,6 +1134,7 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY=
 google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
@@ -1135,6 +1159,7 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5
 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.52.1 h1:2NpOPk5g5Xtb0qebIEs7hNIa++PdtZLo2AQUpc1YnSU=
 google.golang.org/grpc v1.52.1/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
@@ -1150,6 +1175,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=



Test patch:
diff --git a/examples/hotrod/pkg/tracing/rpcmetrics/endpoints_test.go b/examples/hotrod/pkg/tracing/rpcmetrics/endpoints_test.go
new file mode 100644
index 00000000..fc1f84f6
--- /dev/null
+++ b/examples/hotrod/pkg/tracing/rpcmetrics/endpoints_test.go
@@ -0,0 +1,44 @@
+// Copyright (c) 2023 The Jaeger Authors.
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpcmetrics
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNormalizedEndpoints(t *testing.T) {
+	n := newNormalizedEndpoints(1, DefaultNameNormalizer)
+
+	assertLen := func(l int) {
+		n.mux.RLock()
+		defer n.mux.RUnlock()
+		assert.Len(t, n.names, l)
+	}
+
+	assert.Equal(t, "ab_cd", n.normalize("ab^cd"), "one translation")
+	assert.Equal(t, "ab_cd", n.normalize("ab^cd"), "cache hit")
+	assertLen(1)
+	assert.Equal(t, "", n.normalize("xys"), "cache overflow")
+	assertLen(1)
+}
+
+func TestNormalizedEndpointsDoubleLocking(t *testing.T) {
+	n := newNormalizedEndpoints(1, DefaultNameNormalizer)
+	assert.Equal(t, "ab_cd", n.normalize("ab^cd"), "fill out the cache")
+	assert.Equal(t, "", n.normalizeWithLock("xys"), "cache overflow")
+}
diff --git a/examples/hotrod/pkg/tracing/rpcmetrics/metrics_test.go b/examples/hotrod/pkg/tracing/rpcmetrics/metrics_test.go
new file mode 100644
index 00000000..3db61811
--- /dev/null
+++ b/examples/hotrod/pkg/tracing/rpcmetrics/metrics_test.go
@@ -0,0 +1,62 @@
+// Copyright (c) 2023 The Jaeger Authors.
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpcmetrics
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/jaegertracing/jaeger/internal/metricstest"
+)
+
+// E.g. tags("key", "value", "key", "value")
+func tags(kv ...string) map[string]string {
+	m := make(map[string]string)
+	for i := 0; i < len(kv)-1; i += 2 {
+		m[kv[i]] = kv[i+1]
+	}
+	return m
+}
+
+func endpointTags(endpoint string, kv ...string) map[string]string {
+	return tags(append([]string{"endpoint", endpoint}, kv...)...)
+}
+
+func TestMetricsByEndpoint(t *testing.T) {
+	met := metricstest.NewFactory(0)
+	mbe := newMetricsByEndpoint(met, DefaultNameNormalizer, 2)
+
+	m1 := mbe.get("abc1")
+	m2 := mbe.get("abc1")               // from cache
+	m2a := mbe.getWithWriteLock("abc1") // from cache in double-checked lock
+	assert.Equal(t, m1, m2)
+	assert.Equal(t, m1, m2a)
+
+	m3 := mbe.get("abc3")
+	m4 := mbe.get("overflow")
+	m5 := mbe.get("overflow2")
+
+	for _, m := range []*Metrics{m1, m2, m2a, m3, m4, m5} {
+		m.RequestCountSuccess.Inc(1)
+	}
+
+	met.AssertCounterMetrics(t,
+		metricstest.ExpectedMetric{Name: "requests", Tags: endpointTags("abc1", "error", "false"), Value: 3},
+		metricstest.ExpectedMetric{Name: "requests", Tags: endpointTags("abc3", "error", "false"), Value: 1},
+		metricstest.ExpectedMetric{Name: "requests", Tags: endpointTags("other", "error", "false"), Value: 2},
+	)
+}
diff --git a/examples/hotrod/pkg/tracing/rpcmetrics/normalizer_test.go b/examples/hotrod/pkg/tracing/rpcmetrics/normalizer_test.go
new file mode 100644
index 00000000..dac1ff7a
--- /dev/null
+++ b/examples/hotrod/pkg/tracing/rpcmetrics/normalizer_test.go
@@ -0,0 +1,35 @@
+// Copyright (c) 2023 The Jaeger Authors.
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpcmetrics
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSimpleNameNormalizer(t *testing.T) {
+	n := &SimpleNameNormalizer{
+		SafeSets: []SafeCharacterSet{
+			&Range{From: 'a', To: 'z'},
+			&Char{'-'},
+		},
+		Replacement: '-',
+	}
+	assert.Equal(t, "ab-cd", n.Normalize("ab-cd"), "all valid")
+	assert.Equal(t, "ab-cd", n.Normalize("ab.cd"), "single mismatch")
+	assert.Equal(t, "a--cd", n.Normalize("aB-cd"), "range letter mismatch")
+}
diff --git a/examples/hotrod/pkg/tracing/rpcmetrics/observer_test.go b/examples/hotrod/pkg/tracing/rpcmetrics/observer_test.go
new file mode 100644
index 00000000..89f1db38
--- /dev/null
+++ b/examples/hotrod/pkg/tracing/rpcmetrics/observer_test.go
@@ -0,0 +1,162 @@
+// Copyright (c) 2023 The Jaeger Authors.
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpcmetrics
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/stretchr/testify/assert"
+	otbridge "go.opentelemetry.io/otel/bridge/opentracing"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
+	"github.com/opentracing/opentracing-go/ext"
+
+	u "github.com/jaegertracing/jaeger/internal/metricstest"
+)
+
+type testTracer struct {
+	metrics *u.Factory
+	tracer  opentracing.Tracer
+}
+
+func withTestTracer(runTest func(tt *testTracer)) {
+	metrics := u.NewFactory(time.Minute)
+	observer := NewObserver(metrics, DefaultNameNormalizer)
+
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithSpanProcessor(observer),
+		sdktrace.WithResource(resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceNameKey.String("test"),
+		)),
+	)
+	tracer, _ := otbridge.NewTracerPair(tp.Tracer(""))
+	runTest(&testTracer{
+		metrics: metrics,
+		tracer:  tracer,
+	})
+}
+
+func TestObserver(t *testing.T) {
+	withTestTracer(func(testTracer *testTracer) {
+		ts := time.Now()
+		finishOptions := opentracing.FinishOptions{
+			FinishTime: ts.Add(50 * time.Millisecond),
+		}
+
+		testCases := []struct {
+			name           string
+			tag            opentracing.Tag
+			opNameOverride string
+			err            bool
+		}{
+			{name: "local-span", tag: opentracing.Tag{Key: "x", Value: "y"}},
+			{name: "get-user", tag: ext.SpanKindRPCServer},
+			{name: "get-user", tag: ext.SpanKindRPCServer, opNameOverride: "get-user-override"},
+			{name: "get-user", tag: ext.SpanKindRPCServer, err: true},
+			{name: "get-user-client", tag: ext.SpanKindRPCClient},
+		}
+
+		for _, testCase := range testCases {
+			span := testTracer.tracer.StartSpan(
+				testCase.name,
+				testCase.tag,
+				opentracing.StartTime(ts),
+			)
+			if testCase.opNameOverride != "" {
+				span.SetOperationName(testCase.opNameOverride)
+			}
+			if testCase.err {
+				ext.Error.Set(span, true)
+			}
+			span.FinishWithOptions(finishOptions)
+		}
+
+		testTracer.metrics.AssertCounterMetrics(t,
+			u.ExpectedMetric{Name: "requests", Tags: endpointTags("local_span", "error", "false"), Value: 0},
+			u.ExpectedMetric{Name: "requests", Tags: endpointTags("get_user", "error", "false"), Value: 1},
+			u.ExpectedMetric{Name: "requests", Tags: endpointTags("get_user", "error", "true"), Value: 1},
+			u.ExpectedMetric{Name: "requests", Tags: endpointTags("get_user_override", "error", "false"), Value: 1},
+			u.ExpectedMetric{Name: "requests", Tags: endpointTags("get_user_client", "error", "false"), Value: 0},
+		)
+		// TODO something wrong with string generation, .P99 should not be appended to the tag
+		// as a result we cannot use u.AssertGaugeMetrics
+		_, g := testTracer.metrics.Snapshot()
+		assert.EqualValues(t, 51, g["request_latency|endpoint=get_user|error=false.P99"])
+		assert.EqualValues(t, 51, g["request_latency|endpoint=get_user|error=true.P99"])
+	})
+}
+
+func TestTags(t *testing.T) {
+	type tagTestCase struct {
+		key     string
+		variant string
+		value   interface{}
+		metrics []u.ExpectedMetric
+	}
+
+	testCases := []tagTestCase{
+		{key: "something", value: 42, metrics: []u.ExpectedMetric{
+			{Name: "requests", Value: 1, Tags: tags("error", "false")},
+		}},
+		{key: "error", value: true, metrics: []u.ExpectedMetric{
+			{Name: "requests", Value: 1, Tags: tags("error", "true")},
+		}},
+		// OTEL bridge does not interpret string "true" as error status
+		// {key: "error", value: "true", variant: "string", metrics: []u.ExpectedMetric{
+		// 	{Name: "requests", Value: 1, Tags: tags("error", "true")},
+		// }},
+	}
+
+	for i := 200; i <= 500; i += 100 {
+		status_codes := []struct {
+			value   interface{}
+			variant string
+		}{
+			{value: i},
+			{value: uint16(i), variant: "uint16"},
+			{value: fmt.Sprintf("%d", i), variant: "string"},
+		}
+		for _, v := range status_codes {
+			testCases = append(testCases, tagTestCase{
+				key:     "http.status_code",
+				value:   v.value,
+				variant: v.variant,
+				metrics: []u.ExpectedMetric{
+					{Name: "http_requests", Value: 1, Tags: tags("status_code", fmt.Sprintf("%dxx", i/100))},
+				},
+			})
+		}
+	}
+
+	for _, testCase := range testCases {
+		for i := range testCase.metrics {
+			testCase.metrics[i].Tags["endpoint"] = "span"
+		}
+		t.Run(fmt.Sprintf("%s-%v-%s", testCase.key, testCase.value, testCase.variant), func(t *testing.T) {
+			withTestTracer(func(testTracer *testTracer) {
+				span := testTracer.tracer.StartSpan("span", ext.SpanKindRPCServer)
+				span.SetTag(testCase.key, testCase.value)
+				span.Finish()
+				testTracer.metrics.AssertCounterMetrics(t, testCase.metrics...)
+			})
+		})
+	}
+}
diff --git a/internal/metricstest/metricstest.go b/internal/metricstest/metricstest.go
index 954dd44b..530b3985 100644
--- a/internal/metricstest/metricstest.go
+++ b/internal/metricstest/metricstest.go
@@ -48,7 +48,7 @@ func assertMetrics(t *testing.T, actualMetrics map[string]int64, expectedMetrics
 		assert.EqualValues(t,
 			expected.Value,
 			actualMetrics[key],
-			"expected metric name: %s, tags: %+v", expected.Name, expected.Tags,
+			"expected metric name=%s tags: %+v; got: %+v", expected.Name, expected.Tags, actualMetrics,
 		)
 	}
 }