feat(teapot): Extract nvgpu dumps and create GPU events

relay-dynamic-config/src/feature.rs

@@ -107,6 +107,10 @@ pub enum Feature {
     /// See <https://getsentry.github.io/objectstore/rust/objectstore_service/multipart/>.
     #[serde(rename = "projects:relay-upload-multipart")]
     UploadMultipart,
+    /// Split an NVIDIA GPU crash dump (`.nv-gpudmp`) off a minidump/unreal upload
+    /// into its own event, symbolicated by teapot. Shares the Sentry-side flag.
+    #[serde(rename = "organizations:gpu-crash-symbolication")]
+    NvGpuCrashSplit,
     /// Enables OTLP spans to use the Span V2 processing pipeline in Relay.
     ///
     /// This is now the default behaviour of Relay.

relay-server/src/endpoints/minidump.rs

@@ -252,6 +252,7 @@ struct UploadContext<'a> {
     upload_attachments: UploadDecision,
     upload_minidumps: UploadDecision,
     inline_limit: usize,
+    gpu_crash_split: bool,
 }
 
 impl UploadContext<'_> {
@@ -337,6 +338,11 @@ impl<'a> AttachmentStrategy for MinidumpAttachmentStrategy<'a> {
     }
 
     fn infer_type(&self, field: &Field) -> AttachmentType {
+        match field.file_name() {
+            Some(name) if name.ends_with(".nv-gpudmp") => return AttachmentType::NvGpuDump,
+            Some(name) if name.ends_with(".nvdbg") => return AttachmentType::NvShaderDebug,
+            _ => {}
+        }
         match field.name().unwrap_or("") {
             MINIDUMP_FIELD_NAME => AttachmentType::Minidump,
             ITEM_NAME_BREADCRUMBS1 => AttachmentType::Breadcrumbs,
@@ -526,6 +532,7 @@ async fn upload_context<'a>(
         upload_attachments,
         upload_minidumps,
         inline_limit: global_config.options.attachment_inline_limit,
+        gpu_crash_split: project_config.has_feature(Feature::NvGpuCrashSplit),
     }))
 }
 
@@ -657,10 +664,26 @@ async fn handle(
         ));
         return Ok(TextResponse(Some(EventId::new())));
     }
+    // Gated on the org feature: only opted-in orgs split GPU crashes into a
+    // second (billed) event. Captured before `upload_context` is consumed below.
+    let gpu_crash_split = upload_context
+        .as_ref()
+        .is_some_and(|ctx| ctx.gpu_crash_split);
+
     let items = items(upload_context, &state, &meta, content_type, request)
         .await
         .reject(&managed_err)?;
-    let envelope = envelope(items, meta, managed_err)?;
+
+    let mut envelope = envelope(items, meta, managed_err)?;
+    if gpu_crash_split {
+        let (cpu, gpu) = utils::gpu::split_crash(envelope);
+        if let Some(gpu) = gpu {
+            common::handle_managed_envelope(&state, gpu)
+                .await?
+                .ignore_rate_limits();
+        }
+        envelope = cpu;
+    }
 
     let id = envelope.event_id();
 

relay-server/src/endpoints/unreal.rs

@@ -11,13 +11,14 @@ use crate::constants::UNREAL_USER_HEADER;
 use crate::endpoints::common::{self, BadStoreRequest, TextResponse};
 use crate::envelope::{ContentType, Envelope, Item, ItemType};
 use crate::extractors::RequestMeta;
-use crate::middlewares;
+use crate::managed::Managed;
 use crate::service::ServiceState;
 use crate::services::outcome::{DiscardItemType, DiscardReason};
 use crate::services::processor::ProcessingError;
 use crate::services::projects::project::ProjectState;
 use crate::statsd::RelayCounters;
 use crate::utils::extract_items;
+use crate::{middlewares, utils};
 
 #[derive(Debug, Deserialize)]
 struct UnrealQuery {
@@ -35,10 +36,11 @@ struct UnrealParams {
 }
 
 impl UnrealParams {
+    /// Returns the envelope and whether the org may split off a GPU crash event.
     async fn extract_envelope(
         self,
         state: &ServiceState,
-    ) -> Result<Box<Envelope>, BadStoreRequest> {
+    ) -> Result<(Box<Envelope>, bool), BadStoreRequest> {
         let Self { meta, query, data } = self;
 
         if data.is_empty() {
@@ -88,27 +90,40 @@ impl UnrealParams {
                     item.set_unreal_expanded(true);
                     envelope.add_item(item);
                 }
-                return Ok(envelope);
+                let gpu_crash_split = project_config.has_feature(Feature::NvGpuCrashSplit);
+                return Ok((envelope, gpu_crash_split));
             }
         }
 
         let mut item = Item::new(ItemType::UnrealReport);
         item.set_payload(ContentType::OctetStream, data);
         envelope.add_item(item);
 
-        Ok(envelope)
+        Ok((envelope, false))
     }
 }
 
 async fn handle(
     state: ServiceState,
     params: UnrealParams,
 ) -> axum::response::Result<impl IntoResponse> {
-    let envelope = params.extract_envelope(&state).await?;
+    let (envelope, gpu_crash_split) = params.extract_envelope(&state).await?;
+    let mut envelope = Managed::from_envelope(envelope, state.outcome_aggregator().clone());
+
+    if gpu_crash_split {
+        let (cpu, gpu) = utils::gpu::split_crash(envelope);
+        if let Some(gpu) = gpu {
+            common::handle_managed_envelope(&state, gpu)
+                .await?
+                .ignore_rate_limits();
+        }
+        envelope = cpu;
+    }
+
     let id = envelope.event_id();
 
     // Never respond with a 429 since clients often retry these
-    common::handle_envelope(&state, envelope)
+    common::handle_managed_envelope(&state, envelope)
         .await?
         .ignore_rate_limits();
 

relay-server/src/envelope/attachment.rs

@@ -52,6 +52,14 @@ pub enum AttachmentType {
 
     /// An application UI view hierarchy (json payload).
     ViewHierarchy,
+
+    /// An NVIDIA Aftermath GPU crash dump (`.nv-gpudmp`), decoded by teapot.
+    ///
+    /// Carried on its own GPU crash event, split off a minidump/unreal upload.
+    NvGpuDump,
+
+    /// NVIDIA Aftermath shader debug info (`.nvdbg`) accompanying an `NvGpuDump`.
+    NvShaderDebug,
 }
 
 impl fmt::Display for AttachmentType {
@@ -67,6 +75,8 @@ impl fmt::Display for AttachmentType {
             AttachmentType::UnrealContext => write!(f, "unreal.context"),
             AttachmentType::UnrealLogs => write!(f, "unreal.logs"),
             AttachmentType::ViewHierarchy => write!(f, "event.view_hierarchy"),
+            AttachmentType::NvGpuDump => write!(f, "event.nv_gpudmp"),
+            AttachmentType::NvShaderDebug => write!(f, "event.nv_shader_debug"),
         }
     }
 }
@@ -99,6 +109,8 @@ impl std::str::FromStr for AttachmentType {
             "event.view_hierarchy" => AttachmentType::ViewHierarchy,
             "unreal.context" => AttachmentType::UnrealContext,
             "unreal.logs" => AttachmentType::UnrealLogs,
+            "event.nv_gpudmp" => AttachmentType::NvGpuDump,
+            "event.nv_shader_debug" => AttachmentType::NvShaderDebug,
             _ => return Err(UnknownAttachmentType),
         })
     }

relay-server/src/envelope/item.rs

@@ -631,13 +631,15 @@ impl Item {
                         | AttachmentType::EventPayload
                         | AttachmentType::Prosperodump
                         | AttachmentType::Breadcrumbs
-                        | AttachmentType::NintendoSwitchDyingMessage,
+                        | AttachmentType::NintendoSwitchDyingMessage
+                        | AttachmentType::NvGpuDump,
                     ) => true,
                     Some(
                         AttachmentType::Attachment
                         | AttachmentType::UnrealContext
                         | AttachmentType::UnrealLogs
-                        | AttachmentType::ViewHierarchy,
+                        | AttachmentType::ViewHierarchy
+                        | AttachmentType::NvShaderDebug,
                     ) => false,
                     // When an outdated Relay instance forwards an unknown attachment type for compatibility,
                     // we assume that the attachment does not create a new event. This will make it hard

relay-server/src/processing/errors/errors/gpu.rs

@@ -0,0 +1,79 @@
+use relay_quotas::{DataCategory, RateLimits};
+
+use crate::envelope::{AttachmentType, Item, ItemType};
+use crate::managed::{Counted, Quantities, RecordKeeper};
+use crate::processing::ForwardContext;
+use crate::processing::errors::errors::{Context, Expansion, SentryError, utils};
+use crate::processing::errors::{Error, Result};
+
+/// An NVIDIA Aftermath GPU crash dump (`.nv-gpudmp`).
+///
+/// Relay splits the GPU crash onto its own event (see [`crate::utils::gpu`]),
+/// which carries a copy of the CPU event's scope plus the dump and any shader
+/// debug info (`.nvdbg`). Here we turn the copied scope into the event and keep
+/// the dump and shader debug info as attachments; Sentry decodes the dump
+/// out-of-band (via teapot), analogous to how a minidump is symbolicated.
+#[derive(Debug)]
+pub struct GpuCrash(pub Item);
+
+impl SentryError for GpuCrash {
+    fn event_category(&self) -> DataCategory {
+        DataCategory::Error
+    }
+
+    fn try_expand(items: &mut Vec<Item>, ctx: Context<'_>) -> Result<Option<Expansion<Self>>> {
+        let Some(dump) = utils::take_item_by(items, |item| {
+            item.attachment_type() == Some(AttachmentType::NvGpuDump)
+        }) else {
+            return Ok(None);
+        };
+
+        let mut metrics = Default::default();
+        let event = utils::take_event_from_crash_items(items, &mut metrics, ctx)?;
+
+        Ok(Some(Expansion {
+            event: Box::new(event),
+            // The remaining attachments include the shader debug info (`.nvdbg`);
+            // the dump itself is kept via `serialize_into` below.
+            attachments: utils::take_items_of_type(items, ItemType::Attachment),
+            user_reports: utils::take_items_of_type(items, ItemType::UserReport),
+            error: Self(dump),
+            metrics,
+            fully_normalized: false,
+        }))
+    }
+
+    fn apply_rate_limit(
+        &mut self,
+        _category: DataCategory,
+        limits: RateLimits,
+        records: &mut RecordKeeper<'_>,
+    ) -> Result<()> {
+        if !self.0.rate_limited() {
+            self.0.set_rate_limited(true);
+            records.reject_err(Error::RateLimited(limits), &self.0);
+        }
+
+        Ok(())
+    }
+
+    fn serialize_into(self, items: &mut Vec<Item>, _ctx: ForwardContext<'_>) -> Result<()> {
+        items.push(self.0);
+        Ok(())
+    }
+
+    fn minidump_mut(&mut self) -> Option<&mut Item> {
+        None
+    }
+}
+
+impl Counted for GpuCrash {
+    fn quantities(&self) -> Quantities {
+        // A rate limited dump no longer counts as an attachment, but it is still
+        // passed along so Sentry can decode it into the event later.
+        match self.0.rate_limited() {
+            true => Default::default(),
+            false => self.0.quantities(),
+        }
+    }
+}

relay-server/src/processing/errors/errors/mod.rs

@@ -10,6 +10,7 @@ use crate::statsd::RelayCounters;
 
 mod apple_crash_report;
 mod generic;
+mod gpu;
 mod minidump;
 mod nswitch;
 mod playstation;
@@ -21,6 +22,7 @@ mod utils;
 
 pub use self::apple_crash_report::*;
 pub use self::generic::*;
+pub use self::gpu::*;
 pub use self::minidump::*;
 pub use self::nswitch::*;
 pub use self::playstation::*;
@@ -182,6 +184,7 @@ gen_error_kind![
     Nswitch,
     Unreal,
     Minidump,
+    GpuCrash,
     AppleCrashReport,
     Playstation,
     Security,

relay-server/src/services/outcome/mod.rs

@@ -691,6 +691,7 @@ impl From<&AttachmentType> for DiscardAttachmentType {
             AttachmentType::UnrealContext => Self::UnrealContext,
             AttachmentType::UnrealLogs => Self::UnrealLogs,
             AttachmentType::ViewHierarchy => Self::ViewHierarchy,
+            AttachmentType::NvGpuDump | AttachmentType::NvShaderDebug => Self::Attachment,
         }
     }
 }

relay-server/src/utils/gpu.rs

@@ -0,0 +1,87 @@
+use relay_event_schema::protocol::EventId;
+use relay_quotas::DataCategory;
+
+use crate::Envelope;
+use crate::envelope::{AttachmentType, Item, ItemType};
+use crate::managed::Managed;
+
+/// Splits an envelope carrying an error event and a GPU crash into two envelopes.
+///
+/// The GPU crash attachments (`.nv-gpudmp` / `.nvdbg`) are moved onto a second
+/// envelope that also carries a copy of the scope items, so the GPU crash becomes
+/// its own trace-connected, billed event while the CPU crash keeps the original.
+/// The GPU crash processor (see [`crate::processing::errors`]) turns the copied
+/// scope into the event.
+///
+/// Returns the `(cpu, gpu)` envelopes. The GPU envelope is `None` when there is no
+/// GPU crash item, or no scope to copy the GPU event from.
+pub fn split_crash(
+    envelope: Managed<Box<Envelope>>,
+) -> (Managed<Box<Envelope>>, Option<Managed<Box<Envelope>>>) {
+    if !envelope.items().any(is_gpu_crash_item) {
+        return (envelope, None);
+    }
+
+    // The GPU event is a copy of the CPU event's scope, so it inherits the trace,
+    // release and tags. Clone the scope items — an `Event`, or the crashpad
+    // `__sentry-event` / breadcrumb attachments the event is assembled from. With
+    // no scope there is nothing to copy, so leave the crash on the CPU event.
+    let scope: Vec<Item> = envelope
+        .items()
+        .filter(|item| is_scope_item(item))
+        .cloned()
+        .collect();
+    if scope.is_empty() {
+        return (envelope, None);
+    }
+
+    // The GPU event is billed as a duplicated error; cloning the scope also
+    // duplicates any attachment-shaped scope items, so account for those too.
+    let mut duplicated: Vec<(DataCategory, isize)> = vec![(DataCategory::Error, 1)];
+    for item in &scope {
+        for (category, quantity) in item.quantities() {
+            if category != DataCategory::Error {
+                duplicated.push((category, quantity as isize));
+            }
+        }
+    }
+
+    let (cpu, gpu) = envelope.split_once(move |mut envelope, records| {
+        // A fresh id keeps the GPU event distinct from the CPU event it copies;
+        // the envelope header id is authoritative and overwrites the cloned one.
+        let mut gpu = Envelope::from_request(Some(EventId::new()), envelope.meta().clone());
+        for item in scope {
+            gpu.add_item(item);
+        }
+        for item in envelope.take_items_by(is_gpu_crash_item) {
+            gpu.add_item(item);
+        }
+
+        for (category, quantity) in duplicated {
+            records.modify_by(category, quantity);
+        }
+
+        (envelope, gpu)
+    });
+
+    (cpu, Some(gpu))
+}
+
+/// Scope items that carry the event: an [`ItemType::Event`], or the crashpad
+/// `__sentry-event` ([`AttachmentType::EventPayload`]) and breadcrumb attachments
+/// the event is assembled from. Deliberately narrower than [`Item::creates_event`]
+/// (which also matches minidumps, which must stay on the CPU event).
+fn is_scope_item(item: &Item) -> bool {
+    item.ty() == &ItemType::Event
+        || matches!(
+            item.attachment_type(),
+            Some(AttachmentType::EventPayload | AttachmentType::Breadcrumbs)
+        )
+}
+
+fn is_gpu_crash_item(item: &Item) -> bool {
+    matches!(
+        item.attachment_type(),
+        Some(AttachmentType::NvGpuDump) | Some(AttachmentType::NvShaderDebug)
+    )
+}

relay-server/src/utils/mod.rs

@@ -2,6 +2,7 @@ mod api;
 mod debug;
 mod dynamic_sampling;
 mod error;
+pub mod gpu;
 mod multipart;
 mod param_parser;
 mod pick;