Repository: grpc/grpc-go

Issue:
stats: Update documentation and examples to demonstrate enabling `experimental` metrics
There have been several issues opened recently like [this](https://github.com/grpc/grpc-go/issues/8234) asking about not being able to see experimental metrics from wrr and xds. It is because they are not enabled by default and needs to be added to the DefaultMetrics in the dial and server options.

This is the list of all experimental metrics https://grpc.io/docs/guides/opentelemetry-metrics/#instruments. We should include this part in the example and also sample code for adding experimental metrics.

Update documentation and example client and server here https://github.com/grpc/grpc-go/tree/master/examples/features/opentelemetry

Discussion:
@vinothkumarr227: @purnesh42H I'm looking into this. Please assign the issue to me.@purnesh42H: Noticed that we have go doc examples here https://github.com/grpc/grpc-go/blob/master/stats/opentelemetry/example_test.go. We should add one for appending the experimental metrics here as well. We have for removing a metric from `DefaultMetrics` but not for adding.

Also, the godoc doesn't show all the examples from `example_test.go` https://pkg.go.dev/google.golang.org/grpc/stats/opentelemetry#pkg-examples. We should update the names as per specifications here https://go.dev/blog/examples

@dfawley @vinothkumarr227 cc

Solution patch:
diff --git a/examples/features/opentelemetry/README.md b/examples/features/opentelemetry/README.md
index 14ef9794..5e9db482 100644
--- a/examples/features/opentelemetry/README.md
+++ b/examples/features/opentelemetry/README.md
@@ -2,6 +2,8 @@
 
 This example shows how to configure OpenTelemetry on a client and server, and
 shows what type of telemetry data it can produce for certain RPCs.
+This example shows how to enable experimental gRPC metrics, which are disabled
+by default and must be explicitly configured on the client and/or server.
 
 ## Try it
 
diff --git a/examples/features/opentelemetry/client/main.go b/examples/features/opentelemetry/client/main.go
index 0b392d8d..4f9eea53 100644
--- a/examples/features/opentelemetry/client/main.go
+++ b/examples/features/opentelemetry/client/main.go
@@ -63,8 +63,18 @@ func main() {
 	// Configure W3C Trace Context Propagator for traces
 	textMapPropagator := otelpropagation.TraceContext{}
 	do := opentelemetry.DialOption(opentelemetry.Options{
-		MetricsOptions: opentelemetry.MetricsOptions{MeterProvider: meterProvider},
-		TraceOptions:   oteltracing.TraceOptions{TracerProvider: traceProvider, TextMapPropagator: textMapPropagator},
+		MetricsOptions: opentelemetry.MetricsOptions{
+			MeterProvider: meterProvider,
+			// These are example experimental gRPC metrics, which are disabled
+			// by default and must be explicitly enabled. For the full,
+			// up-to-date list of metrics, see:
+			// https://grpc.io/docs/guides/opentelemetry-metrics/#instruments
+			Metrics: opentelemetry.DefaultMetrics().Add(
+				"grpc.client.attempt.started",
+				"grpc.client.attempt.duration",
+			),
+		},
+		TraceOptions: oteltracing.TraceOptions{TracerProvider: traceProvider, TextMapPropagator: textMapPropagator},
 	})
 
 	go http.ListenAndServe(*prometheusEndpoint, promhttp.Handler())
diff --git a/examples/features/opentelemetry/server/main.go b/examples/features/opentelemetry/server/main.go
index 3c5c0346..5288fe98 100644
--- a/examples/features/opentelemetry/server/main.go
+++ b/examples/features/opentelemetry/server/main.go
@@ -71,8 +71,18 @@ func main() {
 	// Configure W3C Trace Context Propagator for traces
 	textMapPropagator := otelpropagation.TraceContext{}
 	so := opentelemetry.ServerOption(opentelemetry.Options{
-		MetricsOptions: opentelemetry.MetricsOptions{MeterProvider: meterProvider},
-		TraceOptions:   oteltracing.TraceOptions{TracerProvider: traceProvider, TextMapPropagator: textMapPropagator}})
+		MetricsOptions: opentelemetry.MetricsOptions{
+			MeterProvider: meterProvider,
+			// These are example experimental gRPC metrics, which are disabled
+			// by default and must be explicitly enabled. For the full,
+			// up-to-date list of metrics, see:
+			// https://grpc.io/docs/guides/opentelemetry-metrics/#instruments
+			Metrics: opentelemetry.DefaultMetrics().Add(
+				"grpc.server.call.started",
+				"grpc.server.call.duration",
+			),
+		},
+		TraceOptions: oteltracing.TraceOptions{TracerProvider: traceProvider, TextMapPropagator: textMapPropagator}})
 
 	go http.ListenAndServe(*prometheusEndpoint, promhttp.Handler())
 



Test patch:
diff --git a/stats/opentelemetry/example_test.go b/stats/opentelemetry/example_test.go
index e87e4ebb..3e58151b 100644
--- a/stats/opentelemetry/example_test.go
+++ b/stats/opentelemetry/example_test.go
@@ -25,7 +25,7 @@ import (
 	"go.opentelemetry.io/otel/sdk/metric"
 )
 
-func Example_dialOption() {
+func ExampleDialOption_basic() {
 	// This is setting default bounds for a view. Setting these bounds through
 	// meter provider from SDK is recommended, as API calls in this module
 	// provide default bounds, but these calls are not guaranteed to be stable
@@ -64,7 +64,7 @@ func Example_dialOption() {
 	defer cc.Close()
 }
 
-func Example_serverOption() {
+func ExampleServerOption_methodFilter() {
 	reader := metric.NewManualReader()
 	provider := metric.NewMeterProvider(metric.WithReader(reader))
 	opts := opentelemetry.Options{
@@ -128,3 +128,24 @@ func ExampleMetrics_enableSome() {
 	}
 	defer cc.Close()
 }
+
+func ExampleOptions_addExperimentalMetrics() {
+	opts := opentelemetry.Options{
+		MetricsOptions: opentelemetry.MetricsOptions{
+			// These are example experimental gRPC metrics, which are disabled
+			// by default and must be explicitly enabled. For the full,
+			// up-to-date list of metrics, see:
+			// https://grpc.io/docs/guides/opentelemetry-metrics/#instruments
+			Metrics: opentelemetry.DefaultMetrics().Add(
+				"grpc.client.attempt.started",
+				"grpc.client.attempt.duration",
+			),
+		},
+	}
+	do := opentelemetry.DialOption(opts)
+	cc, err := grpc.NewClient("<target string>", do, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		// Handle error.
+	}
+	defer cc.Close()
+}