feat(observability): propagate traceparent for LLM requests

crates/core/src/api/llm.rs

@@ -30,13 +30,14 @@ use crate::api::runtime::{
     LlmSanitizeRequestContext, LlmSanitizeResponseContext, LlmStreamExecutionNextFn,
     MiddlewareContinuationContext, with_active_event_uuid,
 };
-use crate::api::runtime::{ScopeStackHandle, current_scope_stack};
+use crate::api::runtime::{ScopeStackHandle, capture_traceparent, current_scope_stack};
 use crate::api::scope::event;
 use crate::api::scope::{EmitMarkEventParams, ScopeHandle};
 use crate::api::shared::{
-    ensure_runtime_owner, inject_dynamo_session_ids, metadata_with_otel_error,
-    metadata_with_otel_status, resolve_parent_uuid, run_request_intercepts_with_codec_and_recorder,
-    snapshot_event_sanitizers, snapshot_event_subscribers,
+    ensure_runtime_owner, inject_dynamo_session_ids, inject_traceparent, inject_traceparent_value,
+    metadata_with_otel_error, metadata_with_otel_status, resolve_parent_uuid,
+    run_request_intercepts_with_codec_and_recorder, snapshot_event_sanitizers,
+    snapshot_event_subscribers,
 };
 use crate::codec::request::{AnnotatedLlmRequest, Message};
 use crate::codec::response::{AnnotatedLlmResponse, attach_estimated_cost_for_provider};
@@ -141,6 +142,10 @@ pub struct CreateLlmHandleParams<'a> {
     /// Logical provider or model family name. Gateway-managed provider calls
     /// should pass the provider route name, for example `anthropic.messages`.
     pub name: &'a str,
+    /// Optional UUID reserved before request interception so outbound
+    /// propagation can identify the emitted LLM span.
+    #[builder(default)]
+    pub uuid: Option<Uuid>,
     /// Optional parent scope UUID.
     #[builder(default)]
     pub parent_uuid: Option<uuid::Uuid>,
@@ -1441,8 +1446,9 @@ pub async fn llm_call_execute(params: LlmCallExecuteParams) -> Result<Json> {
     }
 
     let request_codec = codec.clone();
+    let llm_uuid = Uuid::now_v7();
     let optimization_recorder = LlmOptimizationRecorder::default();
-    let (intercepted_request, annotated_request, pending_marks, optimization_contributions) =
+    let (mut intercepted_request, annotated_request, pending_marks, optimization_contributions) =
         scope_llm_optimization_recorder(optimization_recorder.clone(), async {
             run_request_intercepts_with_codec_and_recorder(
                 &name,
@@ -1457,6 +1463,7 @@ pub async fn llm_call_execute(params: LlmCallExecuteParams) -> Result<Json> {
     let mut handle = create_llm_handle(
         CreateLlmHandleParams::builder()
             .name(name.as_str())
+            .uuid(llm_uuid)
             .parent_uuid_opt(resolve_parent_uuid(parent.as_ref()))
             .attributes(attributes)
             .data_opt(data.clone())
@@ -1478,6 +1485,7 @@ pub async fn llm_call_execute(params: LlmCallExecuteParams) -> Result<Json> {
         &lifecycle_subscribers,
     )
     .await?;
+    inject_traceparent(&mut intercepted_request, handle.uuid)?;
     emit_pending_request_marks(&handle, pending_marks, &lifecycle_subscribers).await?;
     handle
         .optimization_recorder
@@ -1650,8 +1658,9 @@ pub async fn llm_stream_call_execute(params: LlmStreamCallExecuteParams) -> Resu
     }
 
     let request_codec = codec.clone();
+    let llm_uuid = Uuid::now_v7();
     let optimization_recorder = LlmOptimizationRecorder::default();
-    let (intercepted_request, annotated_request, pending_marks, optimization_contributions) =
+    let (mut intercepted_request, annotated_request, pending_marks, optimization_contributions) =
         scope_llm_optimization_recorder(optimization_recorder.clone(), async {
             run_request_intercepts_with_codec_and_recorder(
                 &name,
@@ -1666,6 +1675,7 @@ pub async fn llm_stream_call_execute(params: LlmStreamCallExecuteParams) -> Resu
     let mut handle = create_llm_handle(
         CreateLlmHandleParams::builder()
             .name(name.as_str())
+            .uuid(llm_uuid)
             .parent_uuid_opt(resolve_parent_uuid(parent.as_ref()))
             .attributes(attributes)
             .data_opt(data.clone())
@@ -1687,6 +1697,7 @@ pub async fn llm_stream_call_execute(params: LlmStreamCallExecuteParams) -> Resu
         &lifecycle_subscribers,
     )
     .await?;
+    inject_traceparent(&mut intercepted_request, handle.uuid)?;
     emit_pending_request_marks(&handle, pending_marks, &lifecycle_subscribers).await?;
     handle
         .optimization_recorder
@@ -1797,6 +1808,9 @@ pub async fn llm_request_intercepts(
     )
     .await?;
     inject_dynamo_session_ids(&mut outcome.request);
+    if let Ok(traceparent) = capture_traceparent() {
+        inject_traceparent_value(&mut outcome.request, traceparent);
+    }
     Ok(outcome)
 }
 

crates/core/src/api/runtime.rs

@@ -26,10 +26,10 @@ pub use global::global_context;
 pub use scope_stack::{
     PropagationContext, ScopeStack, ScopeStackHandle, TASK_SCOPE_STACK, ThreadScopeStackBinding,
     capture_propagation_context, capture_propagation_context_with_root, capture_thread_scope_stack,
-    create_scope_stack, create_scope_stack_from_propagation, current_scope_stack, fork_scope_stack,
-    propagate_scope_to_thread, restore_thread_scope_stack, scope_stack_active,
-    set_thread_scope_stack, sync_thread_scope_stack, task_scope_push, task_scope_remove,
-    task_scope_top, with_active_event_uuid, with_scope_stack,
+    capture_traceparent, create_scope_stack, create_scope_stack_from_propagation,
+    current_scope_stack, fork_scope_stack, propagate_scope_to_thread, restore_thread_scope_stack,
+    scope_stack_active, set_thread_scope_stack, sync_thread_scope_stack, task_scope_push,
+    task_scope_remove, task_scope_top, with_active_event_uuid, with_scope_stack,
 };
 pub use state::NemoRelayContextState;
 #[doc(hidden)]

crates/core/src/api/runtime/scope_stack.rs

@@ -34,6 +34,7 @@ pub struct ScopeStack {
     scope_registries: HashMap<Uuid, ScopeLocalRegistries>,
     fresh_agents: HashSet<Uuid>,
     propagated_parent_uuid: Option<Uuid>,
+    propagated_root_uuid: Option<Uuid>,
 }
 
 /// Versioned, transport-neutral causal context for crossing a Relay boundary.
@@ -66,6 +67,20 @@ impl PropagationContext {
         Ok(serde_json::to_string(self).expect("PropagationContext is always JSON serializable"))
     }
 
+    /// Convert this rooted context to a W3C `traceparent` header value.
+    pub fn to_traceparent(&self) -> Result<String> {
+        self.validate()?;
+        let Some(root_uuid) = self.root_uuid else {
+            return Err(FlowError::InvalidArgument(
+                "rootless propagation context cannot be converted to traceparent".into(),
+            ));
+        };
+        Ok(crate::observability::format_traceparent(
+            root_uuid,
+            self.parent_uuid,
+        ))
+    }
+
     /// Deserialize and validate a context received from application-managed transport.
     pub fn from_json(value: &str) -> Result<Self> {
         let context: Self = serde_json::from_str(value).map_err(|error| {
@@ -106,6 +121,7 @@ impl ScopeStack {
             scope_registries: self.scope_registries.clone(),
             fresh_agents: self.fresh_agents.clone(),
             propagated_parent_uuid: self.propagated_parent_uuid,
+            propagated_root_uuid: self.propagated_root_uuid,
         }
     }
 
@@ -125,6 +141,7 @@ impl ScopeStack {
             scope_registries: HashMap::new(),
             fresh_agents: HashSet::from([root_uuid]),
             propagated_parent_uuid: None,
+            propagated_root_uuid: None,
         }
     }
 
@@ -166,6 +183,7 @@ impl ScopeStack {
             scope_registries: HashMap::new(),
             fresh_agents: HashSet::from([root_uuid]),
             propagated_parent_uuid: context.root_uuid.map(|_| context.parent_uuid),
+            propagated_root_uuid: context.root_uuid,
         })
     }
 
@@ -492,6 +510,44 @@ pub fn capture_propagation_context_with_root(
     Ok(context)
 }
 
+/// Capture the current rooted Relay context as a W3C `traceparent` value.
+pub fn capture_traceparent() -> Result<String> {
+    let active_uuid = active_event_uuid();
+    let parent_uuid = active_uuid.unwrap_or_else(|| task_scope_top().uuid);
+    let stack = current_scope_stack();
+    let stack_guard = stack
+        .read()
+        .map_err(|error| FlowError::Internal(error.to_string()))?;
+    let root_uuid = stack_guard
+        .propagated_root_uuid
+        .or_else(|| stack_guard.scopes().get(1).map(|scope| scope.uuid))
+        .or(active_uuid)
+        .ok_or_else(|| {
+            FlowError::InvalidArgument(
+                "no emitted Relay scope is available for traceparent capture".into(),
+            )
+        })?;
+    Ok(crate::observability::format_traceparent(
+        root_uuid,
+        parent_uuid,
+    ))
+}
+
+pub(crate) fn traceparent_for_llm(parent_uuid: Uuid) -> Result<String> {
+    let stack = current_scope_stack();
+    let stack_guard = stack
+        .read()
+        .map_err(|error| FlowError::Internal(error.to_string()))?;
+    let root_uuid = stack_guard
+        .propagated_root_uuid
+        .or_else(|| stack_guard.scopes().get(1).map(|scope| scope.uuid))
+        .unwrap_or(parent_uuid);
+    Ok(crate::observability::format_traceparent(
+        root_uuid,
+        parent_uuid,
+    ))
+}
+
 tokio::task_local! {
     /// Task-local scope stack handle used by async execution contexts.
     pub static TASK_SCOPE_STACK: ScopeStackHandle;

crates/core/src/api/runtime/state.rs

@@ -610,6 +610,7 @@ impl NemoRelayContextState {
     /// A new [`LlmHandle`] with a fresh UUID.
     pub fn create_llm_handle(&self, params: CreateLlmHandleParams<'_>) -> LlmHandle {
         LlmHandle::builder()
+            .uuid(params.uuid.unwrap_or_else(Uuid::now_v7))
             .name(params.name)
             .started_at(params.timestamp.unwrap_or_else(Utc::now))
             .attributes(params.attributes)

crates/core/src/api/shared.rs

@@ -9,6 +9,7 @@ use crate::api::event::{Event, EventSanitizeFields, ScopeCategory};
 use crate::api::llm::LlmRequest;
 use crate::api::registry::Guardrail;
 use crate::api::runtime::global_context;
+use crate::api::runtime::scope_stack::traceparent_for_llm;
 use crate::api::runtime::{
     EventSanitizeFn, EventSubscriberFn, NemoRelayContextState, ScopeStackHandle,
 };
@@ -25,6 +26,8 @@ use crate::shared_runtime::ensure_process_runtime_owner;
 pub const DYNAMO_SESSION_ID_HEADER_KEY: &str = "x-dynamo-session-id";
 /// Header carrying the parent Dynamo agent session ID.
 pub const DYNAMO_PARENT_SESSION_ID_HEADER_KEY: &str = "x-dynamo-parent-session-id";
+/// Header carrying the W3C trace context for an outbound provider request.
+pub const TRACEPARENT_HEADER_KEY: &str = "traceparent";
 
 pub(crate) fn resolve_parent_uuid(parent: Option<&ScopeHandle>) -> Option<Uuid> {
     Some(
@@ -182,6 +185,21 @@ pub(crate) fn inject_dynamo_session_ids(request: &mut LlmRequest) {
     }
 }
 
+pub(crate) fn inject_traceparent_value(request: &mut LlmRequest, value: String) {
+    request
+        .headers
+        .retain(|key, _| !key.eq_ignore_ascii_case(TRACEPARENT_HEADER_KEY));
+    request
+        .headers
+        .insert(TRACEPARENT_HEADER_KEY.to_string(), Json::String(value));
+}
+
+pub(crate) fn inject_traceparent(request: &mut LlmRequest, parent_uuid: Uuid) -> Result<()> {
+    let value = traceparent_for_llm(parent_uuid)?;
+    inject_traceparent_value(request, value);
+    Ok(())
+}
+
 pub(crate) fn metadata_with_otel_status(
     metadata: Option<Json>,
     status_code: &'static str,

crates/core/src/observability/mod.rs

@@ -97,6 +97,21 @@ pub(crate) fn relay_span_id(uuid: uuid::Uuid) -> opentelemetry::trace::SpanId {
     opentelemetry::trace::SpanId::from_bytes(bytes)
 }
 
+/// Format a W3C traceparent from Relay's deterministic trace and span IDs.
+pub(crate) fn format_traceparent(trace_uuid: uuid::Uuid, span_uuid: uuid::Uuid) -> String {
+    let trace_id = trace_uuid
+        .as_bytes()
+        .iter()
+        .map(|byte| format!("{byte:02x}"))
+        .collect::<String>();
+    let span_id = &span_uuid.as_bytes()[8..];
+    let span_id = span_id
+        .iter()
+        .map(|byte| format!("{byte:02x}"))
+        .collect::<String>();
+    format!("00-{trace_id}-{span_id}-01")
+}
+
 pub(crate) fn push_common_optimization_attributes(
     attributes: &mut Vec<opentelemetry::KeyValue>,
     summary: &crate::codec::optimization::LlmOptimizationSummary,

crates/core/tests/integration/context_isolation_tests.rs

@@ -6,7 +6,7 @@
 use std::sync::Arc;
 
 use nemo_relay::api::runtime::{
-    PropagationContext, ScopeStack, TASK_SCOPE_STACK, create_scope_stack,
+    PropagationContext, ScopeStack, TASK_SCOPE_STACK, capture_traceparent, create_scope_stack,
     create_scope_stack_from_propagation, current_scope_stack, fork_scope_stack,
     propagate_scope_to_thread, scope_stack_active, set_thread_scope_stack, sync_thread_scope_stack,
     task_scope_push, task_scope_remove, task_scope_top, with_scope_stack,
@@ -145,6 +145,35 @@ fn test_propagation_context_json_round_trips_and_validates_input() {
     );
 }
 
+#[test]
+fn test_rooted_propagation_context_formats_traceparent() {
+    let root_uuid = Uuid::from_u128(0x00112233445566778899aabbccddeeff);
+    let parent_uuid = Uuid::from_u128(0xffeeddccbbaa99887766554433221100);
+    let context = PropagationContext {
+        version: PropagationContext::VERSION,
+        root_uuid: Some(root_uuid),
+        parent_uuid,
+    };
+    assert_eq!(
+        context.to_traceparent().unwrap(),
+        "00-00112233445566778899aabbccddeeff-7766554433221100-01"
+    );
+    assert!(
+        PropagationContext {
+            root_uuid: None,
+            ..context
+        }
+        .to_traceparent()
+        .is_err()
+    );
+}
+
+#[test]
+fn test_capture_traceparent_requires_an_emitted_scope() {
+    set_thread_scope_stack(create_scope_stack());
+    assert!(capture_traceparent().is_err());
+}
+
 #[test]
 fn test_pop_scope_rejects_non_top_and_unknown_handles() {
     set_thread_scope_stack(create_scope_stack());

crates/core/tests/integration/middleware_tests.rs

@@ -3549,6 +3549,45 @@ async fn test_tool_middleware_callbacks_run_without_registry_or_scope_locks() {
     .unwrap();
 }
 
+#[tokio::test]
+async fn managed_llm_injects_runtime_owned_traceparent() {
+    let _lock = TEST_MUTEX.lock().unwrap();
+    reset_global();
+    setup_isolated_thread();
+    let captured = Arc::new(Mutex::new(None::<LlmRequest>));
+    let captured_request = captured.clone();
+    let request = LlmRequest {
+        headers: serde_json::Map::from_iter([
+            ("TraceParent".to_string(), json!("user-value")),
+            ("TRACEPARENT".to_string(), json!("duplicate")),
+        ]),
+        content: json!({"prompt": "hello"}),
+    };
+    llm_call_execute(
+        LlmCallExecuteParams::builder()
+            .name("traceparent-test")
+            .request(request)
+            .func(Arc::new(move |request| {
+                *captured_request.lock().unwrap() = Some(request);
+                Box::pin(async { Ok(json!({"ok": true})) })
+            }))
+            .build(),
+    )
+    .await
+    .unwrap();
+    let request = captured.lock().unwrap().take().unwrap();
+    assert_eq!(request.headers.len(), 1);
+    let traceparent = request
+        .headers
+        .get("traceparent")
+        .unwrap()
+        .as_str()
+        .unwrap();
+    assert!(traceparent.starts_with("00-"));
+    assert!(traceparent.ends_with("-01"));
+    assert_eq!(traceparent.len(), 55);
+}
+
 #[tokio::test]
 async fn test_llm_middleware_callbacks_run_without_registry_or_scope_locks() {
     let _lock = TEST_MUTEX.lock().unwrap();