Repository: awslabs/aws-lambda-rust-runtime (language: rust)

Issue:
Add OpenTelemetryLayer to lambda_runtime layers
Hi, I have started testing OpenTelemetry integration and I found the example given here really useful: https://github.com/awslabs/aws-lambda-rust-runtime/blob/main/examples/opentelemetry-tracing/src/lib.rs

I have copied the code into my project but I am wondering if it could be packaged into the distributed crate, maybe behind an opentelemetry feature flag?

I have also tried using nested layers and I would expect this to create child spans but this is not what I am observing. Is this a feature or a bug?

Given this setup I would expect the span of Tracing-layer to be a child of the OpenTelemetry-layer span:
```rust
 let runtime = Runtime::new(handler)
            .layer(layers::TracingLayer::new())
            .layer(OpenTelemetryLayer::new(|| {
                tracer
                    .provider()
                    .expect("Unable to get tracer provider!")
                    .force_flush();
            }));
```

But this is the result (function_name is customer-lookup):
<img width="1290" alt="Screenshot 2024-05-02 at 08 11 20" src="https://github.com/awslabs/aws-lambda-rust-runtime/assets/488193/caf0c4e9-43fe-4215-800d-d21de19b33fd">



Discussion / hints:
Providing that layer can be a good idea since multiple people seem to use it.

OpenTelemetry and Tracing are two completely separated tracing systems. Adding both layers is not going to do what you expect. You need to use the Tracing / Open Telemetry integration for that: https://docs.rs/tracing-opentelemetry/latest/tracing_opentelemetry/
Both TracingLayer and OpenTelemetryLayer uses `tracing::info_span!` from the tracing crate. So if I created `MyCustomLayer` and added some `tracing::info_span!` statements I would expect them to be nested within spans from an outer layer? (I used TracingLayer in the example just for simplicity)  I have already set up tracing_opentelemetry in order to emit the events to Jaeger where the attached image is taken. 

Fix patch:
diff --git a/examples/opentelemetry-tracing/Cargo.toml b/examples/opentelemetry-tracing/Cargo.toml
index 27a778b5..c80b997d 100644
--- a/examples/opentelemetry-tracing/Cargo.toml
+++ b/examples/opentelemetry-tracing/Cargo.toml
@@ -4,33 +4,15 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-# Library dependencies
-lambda_runtime = { path = "../../lambda-runtime" }
-pin-project = "1"
+lambda_runtime = { path = "../../lambda-runtime", features = ["opentelemetry"] }
 opentelemetry-semantic-conventions = "0.14"
+opentelemetry = "0.22"
+opentelemetry_sdk = { version = "0.22", features = ["rt-tokio"] }
+opentelemetry-stdout = { version = "0.3", features = ["trace"] }
+pin-project = "1"
+serde_json = "1.0"
+tokio = "1"
 tower = "0.4"
 tracing = "0.1"
-
-# Binary dependencies
-opentelemetry = { version = "0.22", optional = true }
-opentelemetry_sdk = { version = "0.22", features = ["rt-tokio"], optional = true }
-opentelemetry-stdout = { version = "0.3", features = ["trace"], optional = true }
-serde_json = { version = "1.0", optional = true }
-tokio = { version = "1", optional = true }
-tracing-opentelemetry = { version = "0.23", optional = true }
-tracing-subscriber = { version = "0.3", optional = true }
-
-[features]
-build-binary = [
-    "opentelemetry",
-    "opentelemetry_sdk",
-    "opentelemetry-stdout",
-    "serde_json",
-    "tokio",
-    "tracing-opentelemetry",
-    "tracing-subscriber",
-]
-
-[[bin]]
-name = "opentelemetry-tracing"
-required-features = ["build-binary"]
+tracing-opentelemetry = "0.23"
+tracing-subscriber = "0.3"
diff --git a/examples/opentelemetry-tracing/src/main.rs b/examples/opentelemetry-tracing/src/main.rs
index 68038366..4020d87c 100644
--- a/examples/opentelemetry-tracing/src/main.rs
+++ b/examples/opentelemetry-tracing/src/main.rs
@@ -1,7 +1,6 @@
-use lambda_runtime::{LambdaEvent, Runtime};
+use lambda_runtime::{layers::OpenTelemetryLayer as OtelLayer, LambdaEvent, Runtime};
 use opentelemetry::trace::TracerProvider;
 use opentelemetry_sdk::{runtime, trace};
-use opentelemetry_tracing::OpenTelemetryLayer;
 use tower::{service_fn, BoxError};
 use tracing_subscriber::prelude::*;
 
@@ -25,7 +24,7 @@ async fn main() -> Result<(), BoxError> {
         .init();
 
     // Initialize the Lambda runtime and add OpenTelemetry tracing
-    let runtime = Runtime::new(service_fn(echo)).layer(OpenTelemetryLayer::new(|| {
+    let runtime = Runtime::new(service_fn(echo)).layer(OtelLayer::new(|| {
         // Make sure that the trace is exported before the Lambda runtime is frozen
         tracer_provider.force_flush();
     }));
diff --git a/lambda-runtime/Cargo.toml b/lambda-runtime/Cargo.toml
index 652381eb..58ecc6b6 100644
--- a/lambda-runtime/Cargo.toml
+++ b/lambda-runtime/Cargo.toml
@@ -16,6 +16,7 @@ readme = "../README.md"
 [features]
 default = ["tracing"]
 tracing = ["lambda_runtime_api_client/tracing"]
+opentelemetry = ["opentelemetry-semantic-conventions"]
 
 [dependencies]
 async-stream = "0.3"
@@ -34,6 +35,7 @@ hyper-util = { workspace = true, features = [
     "tokio",
 ] }
 lambda_runtime_api_client = { version = "0.11.1", path = "../lambda-runtime-api-client", default-features = false }
+opentelemetry-semantic-conventions = { version = "0.14", optional = true }
 pin-project = "1"
 serde = { version = "1", features = ["derive", "rc"] }
 serde_json = "^1"
diff --git a/lambda-runtime/src/layers/mod.rs b/lambda-runtime/src/layers/mod.rs
index 27ce0d68..55fdccd3 100644
--- a/lambda-runtime/src/layers/mod.rs
+++ b/lambda-runtime/src/layers/mod.rs
@@ -10,3 +10,8 @@ pub(crate) use api_client::RuntimeApiClientService;
 pub(crate) use api_response::RuntimeApiResponseService;
 pub(crate) use panic::CatchPanicService;
 pub use trace::TracingLayer;
+
+#[cfg(feature = "opentelemetry")]
+mod otel;
+#[cfg(feature = "opentelemetry")]
+pub use otel::OpenTelemetryLayer;
diff --git a/examples/opentelemetry-tracing/src/lib.rs b/lambda-runtime/src/layers/otel.rs
similarity index 88%
rename from examples/opentelemetry-tracing/src/lib.rs
rename to lambda-runtime/src/layers/otel.rs
index 82f12a16..1a483a39 100644
--- a/examples/opentelemetry-tracing/src/lib.rs
+++ b/lambda-runtime/src/layers/otel.rs
@@ -2,7 +2,7 @@ use std::future::Future;
 use std::pin::Pin;
 use std::task;
 
-use lambda_runtime::LambdaInvocation;
+use crate::LambdaInvocation;
 use opentelemetry_semantic_conventions::trace as traceconv;
 use pin_project::pin_project;
 use tower::{Layer, Service};
@@ -19,6 +19,7 @@ impl<F> OpenTelemetryLayer<F>
 where
     F: Fn() + Clone,
 {
+    /// Create a new [OpenTelemetryLayer] with the provided flush function.
     pub fn new(flush_fn: F) -> Self {
         Self { flush_fn }
     }
@@ -71,9 +72,14 @@ where
         // After the first execution, we can set 'coldstart' to false
         self.coldstart = false;
 
-        let fut = self.inner.call(req).instrument(span);
+        let future = {
+            // Enter the span before calling the inner service
+            // to ensure that it's assigned as parent of the inner spans.
+            let _guard = span.enter();
+            self.inner.call(req)
+        };
         OpenTelemetryFuture {
-            future: Some(fut),
+            future: Some(future.instrument(span)),
             flush_fn: self.flush_fn.clone(),
         }
     }
diff --git a/lambda-runtime/src/layers/trace.rs b/lambda-runtime/src/layers/trace.rs
index 35c74c12..e93927b1 100644
--- a/lambda-runtime/src/layers/trace.rs
+++ b/lambda-runtime/src/layers/trace.rs
@@ -44,6 +44,8 @@ where
     fn call(&mut self, req: LambdaInvocation) -> Self::Future {
         let span = request_span(&req.context);
         let future = {
+            // Enter the span before calling the inner service
+            // to ensure that it's assigned as parent of the inner spans.
             let _guard = span.enter();
             self.inner.call(req)
         };


Test patch: