From 07f3731e33e8f75b628eac9d3b82cb73767761db Mon Sep 17 00:00:00 2001 From: Amir Mujacic Date: Thu, 16 Jul 2026 14:13:04 +0200 Subject: [PATCH 1/3] feat(teapot): Extract nvgpu dumps and create GPU events --- relay-dynamic-config/src/feature.rs | 4 ++ relay-server/src/endpoints/minidump.rs | 75 ++++++++++++++++++++++++ relay-server/src/endpoints/unreal.rs | 35 +++++++++-- relay-server/src/envelope/attachment.rs | 12 ++++ relay-server/src/envelope/item.rs | 6 +- relay-server/src/services/outcome/mod.rs | 1 + relay-server/src/utils/unreal.rs | 6 ++ 7 files changed, 133 insertions(+), 6 deletions(-) diff --git a/relay-dynamic-config/src/feature.rs b/relay-dynamic-config/src/feature.rs index 338efc5b292..b400ad45dea 100644 --- a/relay-dynamic-config/src/feature.rs +++ b/relay-dynamic-config/src/feature.rs @@ -107,6 +107,10 @@ pub enum Feature { /// See . #[serde(rename = "projects:relay-upload-multipart")] UploadMultipart, + /// Split an NVIDIA GPU crash dump (`.nv-gpudmp`) off a minidump/unreal upload + /// into its own event, symbolicated by teapot. Shares the Sentry-side flag. + #[serde(rename = "organizations:gpu-crash-symbolication")] + NvGpuCrashSplit, /// Enables OTLP spans to use the Span V2 processing pipeline in Relay. /// /// This is now the default behaviour of Relay. diff --git a/relay-server/src/endpoints/minidump.rs b/relay-server/src/endpoints/minidump.rs index dd3d59232a5..bf080eb4c09 100644 --- a/relay-server/src/endpoints/minidump.rs +++ b/relay-server/src/endpoints/minidump.rs @@ -252,6 +252,7 @@ struct UploadContext<'a> { upload_attachments: UploadDecision, upload_minidumps: UploadDecision, inline_limit: usize, + gpu_crash_split: bool, } impl UploadContext<'_> { @@ -337,6 +338,11 @@ impl<'a> AttachmentStrategy for MinidumpAttachmentStrategy<'a> { } fn infer_type(&self, field: &Field) -> AttachmentType { + match field.file_name() { + Some(name) if name.ends_with(".nv-gpudmp") => return AttachmentType::NvGpuDump, + Some(name) if name.ends_with(".nvdbg") => return AttachmentType::NvShaderDebug, + _ => {} + } match field.name().unwrap_or("") { MINIDUMP_FIELD_NAME => AttachmentType::Minidump, ITEM_NAME_BREADCRUMBS1 => AttachmentType::Breadcrumbs, @@ -526,6 +532,7 @@ async fn upload_context<'a>( upload_attachments, upload_minidumps, inline_limit: global_config.options.attachment_inline_limit, + gpu_crash_split: project_config.has_feature(Feature::NvGpuCrashSplit), })) } @@ -628,6 +635,13 @@ fn envelope( Ok(envelope) } +pub(crate) fn is_gpu_crash_item(item: &Item) -> bool { + matches!( + item.attachment_type(), + Some(AttachmentType::NvGpuDump) | Some(AttachmentType::NvShaderDebug) + ) +} + async fn handle( state: ServiceState, meta: RequestMeta, @@ -657,9 +671,70 @@ async fn handle( )); return Ok(TextResponse(Some(EventId::new()))); } + // Gated on the org feature: only opted-in orgs split GPU crashes into a + // second (billed) event. Captured before `upload_context` is consumed below. + let gpu_crash_split = upload_context + .as_ref() + .is_some_and(|ctx| ctx.gpu_crash_split); let items = items(upload_context, &state, &meta, content_type, request) .await .reject(&managed_err)?; + + if gpu_crash_split && items.iter().any(is_gpu_crash_item) { + let cpu_event_id = common::event_id_from_items(&items) + .reject2(&items, &managed_err)? + .unwrap_or_else(EventId::new); + let gpu_event_id = EventId::new(); + + let scope_event = items + .iter() + .find(|item| { + item.ty() == &ItemType::Event + || item.attachment_type() == Some(AttachmentType::EventPayload) + }) + .cloned(); + + let (cpu_items, gpu_items) = items.split_once(|items, _records| { + let mut cpu = Items::new(); + let mut gpu = Items::new(); + for item in items { + if is_gpu_crash_item(&item) { + gpu.push(item); + } else { + cpu.push(item); + } + } + (cpu, gpu) + }); + + let gpu_meta = meta.clone(); + let gpu_envelope = gpu_items.map(move |items, records| { + records.modify_by(DataCategory::Error, 1); + let mut envelope = Envelope::from_request(Some(gpu_event_id), gpu_meta); + if let Some(scope_event) = scope_event { + envelope.add_item(scope_event); + } + for item in items { + envelope.add_item(item); + } + envelope + }); + + let cpu_envelope = cpu_items.map(move |items, records| { + managed_err.accept(|_| ()); + records.modify_by(DataCategory::Error, 1); + Box::new(Envelope::from_request(Some(cpu_event_id), meta).with_items(items)) + }); + + common::handle_managed_envelope(&state, cpu_envelope) + .await? + .ignore_rate_limits(); + if let Ok(gpu) = common::handle_managed_envelope(&state, gpu_envelope).await { + gpu.ignore_rate_limits(); + } + return Ok(TextResponse(Some(cpu_event_id))); + } + let envelope = envelope(items, meta, managed_err)?; let id = envelope.event_id(); diff --git a/relay-server/src/endpoints/unreal.rs b/relay-server/src/endpoints/unreal.rs index b0a409a9d01..99100cc2c5a 100644 --- a/relay-server/src/endpoints/unreal.rs +++ b/relay-server/src/endpoints/unreal.rs @@ -9,6 +9,7 @@ use serde::Deserialize; use crate::constants::UNREAL_USER_HEADER; use crate::endpoints::common::{self, BadStoreRequest, TextResponse}; +use crate::endpoints::minidump::is_gpu_crash_item; use crate::envelope::{ContentType, Envelope, Item, ItemType}; use crate::extractors::RequestMeta; use crate::middlewares; @@ -35,10 +36,11 @@ struct UnrealParams { } impl UnrealParams { + /// Returns the envelope and whether the org may split off a GPU crash event. async fn extract_envelope( self, state: &ServiceState, - ) -> Result, BadStoreRequest> { + ) -> Result<(Box, bool), BadStoreRequest> { let Self { meta, query, data } = self; if data.is_empty() { @@ -88,7 +90,8 @@ impl UnrealParams { item.set_unreal_expanded(true); envelope.add_item(item); } - return Ok(envelope); + let gpu_crash_split = project_config.has_feature(Feature::NvGpuCrashSplit); + return Ok((envelope, gpu_crash_split)); } } @@ -96,7 +99,7 @@ impl UnrealParams { item.set_payload(ContentType::OctetStream, data); envelope.add_item(item); - Ok(envelope) + Ok((envelope, false)) } } @@ -104,9 +107,33 @@ async fn handle( state: ServiceState, params: UnrealParams, ) -> axum::response::Result { - let envelope = params.extract_envelope(&state).await?; + let (mut envelope, gpu_crash_split) = params.extract_envelope(&state).await?; let id = envelope.event_id(); + // Gated on the org feature: only opted-in orgs split the GPU crash off the + // expanded report into a second (billed) event. + if gpu_crash_split { + let scope_event = envelope + .get_item_by(|item| item.ty() == &ItemType::Event) + .cloned(); + let gpu_items = envelope.take_items_by(is_gpu_crash_item); + if !gpu_items.is_empty() { + let gpu_event_id = EventId::new(); + let mut gpu_envelope = + Envelope::from_request(Some(gpu_event_id), envelope.meta().clone()); + if let Some(scope_event) = scope_event { + gpu_envelope.add_item(scope_event); + } + for item in gpu_items { + gpu_envelope.add_item(item); + } + + if let Ok(handled) = common::handle_envelope(&state, gpu_envelope).await { + handled.ignore_rate_limits(); + } + } + } + // Never respond with a 429 since clients often retry these common::handle_envelope(&state, envelope) .await? diff --git a/relay-server/src/envelope/attachment.rs b/relay-server/src/envelope/attachment.rs index eabaa502460..02920fe5f89 100644 --- a/relay-server/src/envelope/attachment.rs +++ b/relay-server/src/envelope/attachment.rs @@ -52,6 +52,14 @@ pub enum AttachmentType { /// An application UI view hierarchy (json payload). ViewHierarchy, + + /// An NVIDIA Aftermath GPU crash dump (`.nv-gpudmp`), decoded by teapot. + /// + /// Carried on its own GPU crash event, split off a minidump/unreal upload. + NvGpuDump, + + /// NVIDIA Aftermath shader debug info (`.nvdbg`) accompanying an `NvGpuDump`. + NvShaderDebug, } impl fmt::Display for AttachmentType { @@ -67,6 +75,8 @@ impl fmt::Display for AttachmentType { AttachmentType::UnrealContext => write!(f, "unreal.context"), AttachmentType::UnrealLogs => write!(f, "unreal.logs"), AttachmentType::ViewHierarchy => write!(f, "event.view_hierarchy"), + AttachmentType::NvGpuDump => write!(f, "event.nv_gpudmp"), + AttachmentType::NvShaderDebug => write!(f, "event.nv_shader_debug"), } } } @@ -99,6 +109,8 @@ impl std::str::FromStr for AttachmentType { "event.view_hierarchy" => AttachmentType::ViewHierarchy, "unreal.context" => AttachmentType::UnrealContext, "unreal.logs" => AttachmentType::UnrealLogs, + "event.nv_gpudmp" => AttachmentType::NvGpuDump, + "event.nv_shader_debug" => AttachmentType::NvShaderDebug, _ => return Err(UnknownAttachmentType), }) } diff --git a/relay-server/src/envelope/item.rs b/relay-server/src/envelope/item.rs index 71b9d4ea2da..cdfb7bb96c7 100644 --- a/relay-server/src/envelope/item.rs +++ b/relay-server/src/envelope/item.rs @@ -631,13 +631,15 @@ impl Item { | AttachmentType::EventPayload | AttachmentType::Prosperodump | AttachmentType::Breadcrumbs - | AttachmentType::NintendoSwitchDyingMessage, + | AttachmentType::NintendoSwitchDyingMessage + | AttachmentType::NvGpuDump, ) => true, Some( AttachmentType::Attachment | AttachmentType::UnrealContext | AttachmentType::UnrealLogs - | AttachmentType::ViewHierarchy, + | AttachmentType::ViewHierarchy + | AttachmentType::NvShaderDebug, ) => false, // When an outdated Relay instance forwards an unknown attachment type for compatibility, // we assume that the attachment does not create a new event. This will make it hard diff --git a/relay-server/src/services/outcome/mod.rs b/relay-server/src/services/outcome/mod.rs index 549ed596574..4e0d455afe5 100644 --- a/relay-server/src/services/outcome/mod.rs +++ b/relay-server/src/services/outcome/mod.rs @@ -691,6 +691,7 @@ impl From<&AttachmentType> for DiscardAttachmentType { AttachmentType::UnrealContext => Self::UnrealContext, AttachmentType::UnrealLogs => Self::UnrealLogs, AttachmentType::ViewHierarchy => Self::ViewHierarchy, + AttachmentType::NvGpuDump | AttachmentType::NvShaderDebug => Self::Attachment, } } } diff --git a/relay-server/src/utils/unreal.rs b/relay-server/src/utils/unreal.rs index 7bd870b5f1e..951da3b41ed 100644 --- a/relay-server/src/utils/unreal.rs +++ b/relay-server/src/utils/unreal.rs @@ -41,6 +41,12 @@ pub fn extract_items(payload: Bytes, config: &Config) -> Result (ContentType::MsgPack, AttachmentType::EventPayload), self::ITEM_NAME_BREADCRUMBS1 => (ContentType::MsgPack, AttachmentType::Breadcrumbs), self::ITEM_NAME_BREADCRUMBS2 => (ContentType::MsgPack, AttachmentType::Breadcrumbs), + name if name.ends_with(".nv-gpudmp") => { + (ContentType::OctetStream, AttachmentType::NvGpuDump) + } + name if name.ends_with(".nvdbg") => { + (ContentType::OctetStream, AttachmentType::NvShaderDebug) + } _ => (ContentType::OctetStream, AttachmentType::Attachment), }, }; From 8a334566373e8ee52c8fcb2e784fd63fb3e3665d Mon Sep 17 00:00:00 2001 From: David Herberth Date: Fri, 17 Jul 2026 13:40:20 +0200 Subject: [PATCH 2/3] simplify splitting --- relay-server/src/endpoints/minidump.rs | 70 ++++---------------------- relay-server/src/endpoints/unreal.rs | 38 +++++--------- relay-server/src/utils/gpu.rs | 47 +++++++++++++++++ relay-server/src/utils/mod.rs | 1 + 4 files changed, 70 insertions(+), 86 deletions(-) create mode 100644 relay-server/src/utils/gpu.rs diff --git a/relay-server/src/endpoints/minidump.rs b/relay-server/src/endpoints/minidump.rs index bf080eb4c09..b649da031b8 100644 --- a/relay-server/src/endpoints/minidump.rs +++ b/relay-server/src/endpoints/minidump.rs @@ -635,13 +635,6 @@ fn envelope( Ok(envelope) } -pub(crate) fn is_gpu_crash_item(item: &Item) -> bool { - matches!( - item.attachment_type(), - Some(AttachmentType::NvGpuDump) | Some(AttachmentType::NvShaderDebug) - ) -} - async fn handle( state: ServiceState, meta: RequestMeta, @@ -676,67 +669,22 @@ async fn handle( let gpu_crash_split = upload_context .as_ref() .is_some_and(|ctx| ctx.gpu_crash_split); + let items = items(upload_context, &state, &meta, content_type, request) .await .reject(&managed_err)?; - if gpu_crash_split && items.iter().any(is_gpu_crash_item) { - let cpu_event_id = common::event_id_from_items(&items) - .reject2(&items, &managed_err)? - .unwrap_or_else(EventId::new); - let gpu_event_id = EventId::new(); - - let scope_event = items - .iter() - .find(|item| { - item.ty() == &ItemType::Event - || item.attachment_type() == Some(AttachmentType::EventPayload) - }) - .cloned(); - - let (cpu_items, gpu_items) = items.split_once(|items, _records| { - let mut cpu = Items::new(); - let mut gpu = Items::new(); - for item in items { - if is_gpu_crash_item(&item) { - gpu.push(item); - } else { - cpu.push(item); - } - } - (cpu, gpu) - }); - - let gpu_meta = meta.clone(); - let gpu_envelope = gpu_items.map(move |items, records| { - records.modify_by(DataCategory::Error, 1); - let mut envelope = Envelope::from_request(Some(gpu_event_id), gpu_meta); - if let Some(scope_event) = scope_event { - envelope.add_item(scope_event); - } - for item in items { - envelope.add_item(item); - } - envelope - }); - - let cpu_envelope = cpu_items.map(move |items, records| { - managed_err.accept(|_| ()); - records.modify_by(DataCategory::Error, 1); - Box::new(Envelope::from_request(Some(cpu_event_id), meta).with_items(items)) - }); - - common::handle_managed_envelope(&state, cpu_envelope) - .await? - .ignore_rate_limits(); - if let Ok(gpu) = common::handle_managed_envelope(&state, gpu_envelope).await { - gpu.ignore_rate_limits(); + let mut envelope = envelope(items, meta, managed_err)?; + if gpu_crash_split { + let (cpu, gpu) = utils::gpu::split_crash(envelope); + if let Some(gpu) = gpu { + common::handle_managed_envelope(&state, gpu) + .await? + .ignore_rate_limits(); } - return Ok(TextResponse(Some(cpu_event_id))); + envelope = cpu; } - let envelope = envelope(items, meta, managed_err)?; - let id = envelope.event_id(); // Never respond with a 429 since clients often retry these diff --git a/relay-server/src/endpoints/unreal.rs b/relay-server/src/endpoints/unreal.rs index 99100cc2c5a..293f62ad64a 100644 --- a/relay-server/src/endpoints/unreal.rs +++ b/relay-server/src/endpoints/unreal.rs @@ -9,16 +9,16 @@ use serde::Deserialize; use crate::constants::UNREAL_USER_HEADER; use crate::endpoints::common::{self, BadStoreRequest, TextResponse}; -use crate::endpoints::minidump::is_gpu_crash_item; use crate::envelope::{ContentType, Envelope, Item, ItemType}; use crate::extractors::RequestMeta; -use crate::middlewares; +use crate::managed::Managed; use crate::service::ServiceState; use crate::services::outcome::{DiscardItemType, DiscardReason}; use crate::services::processor::ProcessingError; use crate::services::projects::project::ProjectState; use crate::statsd::RelayCounters; use crate::utils::extract_items; +use crate::{middlewares, utils}; #[derive(Debug, Deserialize)] struct UnrealQuery { @@ -107,35 +107,23 @@ async fn handle( state: ServiceState, params: UnrealParams, ) -> axum::response::Result { - let (mut envelope, gpu_crash_split) = params.extract_envelope(&state).await?; - let id = envelope.event_id(); + let (envelope, gpu_crash_split) = params.extract_envelope(&state).await?; + let mut envelope = Managed::from_envelope(envelope, state.outcome_aggregator().clone()); - // Gated on the org feature: only opted-in orgs split the GPU crash off the - // expanded report into a second (billed) event. if gpu_crash_split { - let scope_event = envelope - .get_item_by(|item| item.ty() == &ItemType::Event) - .cloned(); - let gpu_items = envelope.take_items_by(is_gpu_crash_item); - if !gpu_items.is_empty() { - let gpu_event_id = EventId::new(); - let mut gpu_envelope = - Envelope::from_request(Some(gpu_event_id), envelope.meta().clone()); - if let Some(scope_event) = scope_event { - gpu_envelope.add_item(scope_event); - } - for item in gpu_items { - gpu_envelope.add_item(item); - } - - if let Ok(handled) = common::handle_envelope(&state, gpu_envelope).await { - handled.ignore_rate_limits(); - } + let (cpu, gpu) = utils::gpu::split_crash(envelope); + if let Some(gpu) = gpu { + common::handle_managed_envelope(&state, gpu) + .await? + .ignore_rate_limits(); } + envelope = cpu; } + let id = envelope.event_id(); + // Never respond with a 429 since clients often retry these - common::handle_envelope(&state, envelope) + common::handle_managed_envelope(&state, envelope) .await? .ignore_rate_limits(); diff --git a/relay-server/src/utils/gpu.rs b/relay-server/src/utils/gpu.rs new file mode 100644 index 00000000000..cdf0a957fd3 --- /dev/null +++ b/relay-server/src/utils/gpu.rs @@ -0,0 +1,47 @@ +use relay_quotas::DataCategory; + +use crate::Envelope; +use crate::envelope::{AttachmentType, Item, ItemType}; +use crate::managed::Managed; + +/// Splits an envelope containing an error and a GPU crash into two separate envelopes. +/// +/// GPU crash attachments are moved into the GPU crash envelope, the error is cloned. +/// The error is cloned into the GPU envelope and GPU crash attachments are separated into the GPU crash. +pub fn split_crash( + envelope: Managed>, +) -> (Managed>, Option>>) { + if !envelope.items().any(is_gpu_crash_item) { + return (envelope, None); + } + + let Some(event) = envelope + .items() + .find(|item| item.ty() == &ItemType::Event) + .cloned() + else { + return (envelope, None); + }; + + let (cpu, gpu) = envelope.split_once(|mut envelope, records| { + let mut gpu = Envelope::from_request(None, envelope.meta().clone()); + gpu.add_item(event); + for item in envelope.take_items_by(is_gpu_crash_item) { + gpu.add_item(item); + } + + // We duplicate the error event into a second envelope. + records.modify_by(DataCategory::Error, 1); + + (envelope, gpu) + }); + + (cpu, Some(gpu)) +} + +fn is_gpu_crash_item(item: &Item) -> bool { + matches!( + item.attachment_type(), + Some(AttachmentType::NvGpuDump) | Some(AttachmentType::NvShaderDebug) + ) +} diff --git a/relay-server/src/utils/mod.rs b/relay-server/src/utils/mod.rs index c0de3d00b47..211bac61a77 100644 --- a/relay-server/src/utils/mod.rs +++ b/relay-server/src/utils/mod.rs @@ -2,6 +2,7 @@ mod api; mod debug; mod dynamic_sampling; mod error; +pub mod gpu; mod multipart; mod param_parser; mod pick; From 5207ccd9c35726f9b2cec1a2ac5a9d225cdd657c Mon Sep 17 00:00:00 2001 From: Amir Mujacic Date: Tue, 21 Jul 2026 10:26:39 +0200 Subject: [PATCH 3/3] Add GPU type processing --- .../src/processing/errors/errors/gpu.rs | 79 +++++++ .../src/processing/errors/errors/mod.rs | 3 + relay-server/src/utils/gpu.rs | 64 +++++- tests/integration/test_gpu.py | 206 ++++++++++++++++++ 4 files changed, 340 insertions(+), 12 deletions(-) create mode 100644 relay-server/src/processing/errors/errors/gpu.rs create mode 100644 tests/integration/test_gpu.py diff --git a/relay-server/src/processing/errors/errors/gpu.rs b/relay-server/src/processing/errors/errors/gpu.rs new file mode 100644 index 00000000000..a2a1d7a24fc --- /dev/null +++ b/relay-server/src/processing/errors/errors/gpu.rs @@ -0,0 +1,79 @@ +use relay_quotas::{DataCategory, RateLimits}; + +use crate::envelope::{AttachmentType, Item, ItemType}; +use crate::managed::{Counted, Quantities, RecordKeeper}; +use crate::processing::ForwardContext; +use crate::processing::errors::errors::{Context, Expansion, SentryError, utils}; +use crate::processing::errors::{Error, Result}; + +/// An NVIDIA Aftermath GPU crash dump (`.nv-gpudmp`). +/// +/// Relay splits the GPU crash onto its own event (see [`crate::utils::gpu`]), +/// which carries a copy of the CPU event's scope plus the dump and any shader +/// debug info (`.nvdbg`). Here we turn the copied scope into the event and keep +/// the dump and shader debug info as attachments; Sentry decodes the dump +/// out-of-band (via teapot), analogous to how a minidump is symbolicated. +#[derive(Debug)] +pub struct GpuCrash(pub Item); + +impl SentryError for GpuCrash { + fn event_category(&self) -> DataCategory { + DataCategory::Error + } + + fn try_expand(items: &mut Vec, ctx: Context<'_>) -> Result>> { + let Some(dump) = utils::take_item_by(items, |item| { + item.attachment_type() == Some(AttachmentType::NvGpuDump) + }) else { + return Ok(None); + }; + + let mut metrics = Default::default(); + let event = utils::take_event_from_crash_items(items, &mut metrics, ctx)?; + + Ok(Some(Expansion { + event: Box::new(event), + // The remaining attachments include the shader debug info (`.nvdbg`); + // the dump itself is kept via `serialize_into` below. + attachments: utils::take_items_of_type(items, ItemType::Attachment), + user_reports: utils::take_items_of_type(items, ItemType::UserReport), + error: Self(dump), + metrics, + fully_normalized: false, + })) + } + + fn apply_rate_limit( + &mut self, + _category: DataCategory, + limits: RateLimits, + records: &mut RecordKeeper<'_>, + ) -> Result<()> { + if !self.0.rate_limited() { + self.0.set_rate_limited(true); + records.reject_err(Error::RateLimited(limits), &self.0); + } + + Ok(()) + } + + fn serialize_into(self, items: &mut Vec, _ctx: ForwardContext<'_>) -> Result<()> { + items.push(self.0); + Ok(()) + } + + fn minidump_mut(&mut self) -> Option<&mut Item> { + None + } +} + +impl Counted for GpuCrash { + fn quantities(&self) -> Quantities { + // A rate limited dump no longer counts as an attachment, but it is still + // passed along so Sentry can decode it into the event later. + match self.0.rate_limited() { + true => Default::default(), + false => self.0.quantities(), + } + } +} diff --git a/relay-server/src/processing/errors/errors/mod.rs b/relay-server/src/processing/errors/errors/mod.rs index f40e5065a5d..29e8f8e1938 100644 --- a/relay-server/src/processing/errors/errors/mod.rs +++ b/relay-server/src/processing/errors/errors/mod.rs @@ -10,6 +10,7 @@ use crate::statsd::RelayCounters; mod apple_crash_report; mod generic; +mod gpu; mod minidump; mod nswitch; mod playstation; @@ -21,6 +22,7 @@ mod utils; pub use self::apple_crash_report::*; pub use self::generic::*; +pub use self::gpu::*; pub use self::minidump::*; pub use self::nswitch::*; pub use self::playstation::*; @@ -182,6 +184,7 @@ gen_error_kind![ Nswitch, Unreal, Minidump, + GpuCrash, AppleCrashReport, Playstation, Security, diff --git a/relay-server/src/utils/gpu.rs b/relay-server/src/utils/gpu.rs index cdf0a957fd3..c9d74639cab 100644 --- a/relay-server/src/utils/gpu.rs +++ b/relay-server/src/utils/gpu.rs @@ -1,13 +1,20 @@ +use relay_event_schema::protocol::EventId; use relay_quotas::DataCategory; use crate::Envelope; use crate::envelope::{AttachmentType, Item, ItemType}; use crate::managed::Managed; -/// Splits an envelope containing an error and a GPU crash into two separate envelopes. +/// Splits an envelope carrying an error event and a GPU crash into two envelopes. /// -/// GPU crash attachments are moved into the GPU crash envelope, the error is cloned. -/// The error is cloned into the GPU envelope and GPU crash attachments are separated into the GPU crash. +/// The GPU crash attachments (`.nv-gpudmp` / `.nvdbg`) are moved onto a second +/// envelope that also carries a copy of the scope items, so the GPU crash becomes +/// its own trace-connected, billed event while the CPU crash keeps the original. +/// The GPU crash processor (see [`crate::processing::errors`]) turns the copied +/// scope into the event. +/// +/// Returns the `(cpu, gpu)` envelopes. The GPU envelope is `None` when there is no +/// GPU crash item, or no scope to copy the GPU event from. pub fn split_crash( envelope: Managed>, ) -> (Managed>, Option>>) { @@ -15,23 +22,44 @@ pub fn split_crash( return (envelope, None); } - let Some(event) = envelope + // The GPU event is a copy of the CPU event's scope, so it inherits the trace, + // release and tags. Clone the scope items — an `Event`, or the crashpad + // `__sentry-event` / breadcrumb attachments the event is assembled from. With + // no scope there is nothing to copy, so leave the crash on the CPU event. + let scope: Vec = envelope .items() - .find(|item| item.ty() == &ItemType::Event) + .filter(|item| is_scope_item(item)) .cloned() - else { + .collect(); + if scope.is_empty() { return (envelope, None); - }; + } + + // The GPU event is billed as a duplicated error; cloning the scope also + // duplicates any attachment-shaped scope items, so account for those too. + let mut duplicated: Vec<(DataCategory, isize)> = vec![(DataCategory::Error, 1)]; + for item in &scope { + for (category, quantity) in item.quantities() { + if category != DataCategory::Error { + duplicated.push((category, quantity as isize)); + } + } + } - let (cpu, gpu) = envelope.split_once(|mut envelope, records| { - let mut gpu = Envelope::from_request(None, envelope.meta().clone()); - gpu.add_item(event); + let (cpu, gpu) = envelope.split_once(move |mut envelope, records| { + // A fresh id keeps the GPU event distinct from the CPU event it copies; + // the envelope header id is authoritative and overwrites the cloned one. + let mut gpu = Envelope::from_request(Some(EventId::new()), envelope.meta().clone()); + for item in scope { + gpu.add_item(item); + } for item in envelope.take_items_by(is_gpu_crash_item) { gpu.add_item(item); } - // We duplicate the error event into a second envelope. - records.modify_by(DataCategory::Error, 1); + for (category, quantity) in duplicated { + records.modify_by(category, quantity); + } (envelope, gpu) }); @@ -39,6 +67,18 @@ pub fn split_crash( (cpu, Some(gpu)) } +/// Scope items that carry the event: an [`ItemType::Event`], or the crashpad +/// `__sentry-event` ([`AttachmentType::EventPayload`]) and breadcrumb attachments +/// the event is assembled from. Deliberately narrower than [`Item::creates_event`] +/// (which also matches minidumps, which must stay on the CPU event). +fn is_scope_item(item: &Item) -> bool { + item.ty() == &ItemType::Event + || matches!( + item.attachment_type(), + Some(AttachmentType::EventPayload | AttachmentType::Breadcrumbs) + ) +} + fn is_gpu_crash_item(item: &Item) -> bool { matches!( item.attachment_type(), diff --git a/tests/integration/test_gpu.py b/tests/integration/test_gpu.py new file mode 100644 index 00000000000..810f9d4c96c --- /dev/null +++ b/tests/integration/test_gpu.py @@ -0,0 +1,206 @@ +"""Integration tests for the GPU crash split. + +Relay splits a minidump upload that carries an NVIDIA Aftermath GPU crash dump +(`.nv-gpudmp`) into two events: the CPU crash (minidump) keeps the original +event, and the GPU crash becomes a second, trace-connected event that carries a +copy of the scope plus the `.nv-gpudmp` / `.nvdbg` attachments. Gated on the +`organizations:gpu-crash-symbolication` feature. +""" + +import queue +from uuid import UUID + +import msgpack +import pytest + +MINIDUMP_ATTACHMENT_NAME = "upload_file_minidump" +EVENT_ATTACHMENT_NAME = "__sentry-event" +GPU_FEATURE = "organizations:gpu-crash-symbolication" + +# Relay infers attachment types from the file name: `.nv-gpudmp` -> the GPU dump, +# `.nvdbg` -> shader debug info. Their bytes are opaque to Relay (teapot decodes +# them), so dummy content is enough here. +MINIDUMP = (MINIDUMP_ATTACHMENT_NAME, "minidump.dmp", "MDMP content") +GPU_DUMP = ("gpudump", "crash.nv-gpudmp", b"NVGPU dummy dump") +SHADER_DBG = ("shaderdbg", "shader-abc.nvdbg", b"nvdbg dummy") + +SCOPE_EVENT_ID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +SCOPE_TRACE_ID = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + + +def scope_attachment(): + """The `__sentry-event` scope (msgpack) the SDK ships alongside the crash.""" + event = { + "event_id": SCOPE_EVENT_ID, + "release": "game@1.0.0", + "environment": "prod", + "tags": {"custom_tag": "custom_value"}, + "contexts": { + "trace": {"trace_id": SCOPE_TRACE_ID, "span_id": "cccccccccccccccc"} + }, + } + return (EVENT_ATTACHMENT_NAME, EVENT_ATTACHMENT_NAME, msgpack.packb(event)) + + +def attachment_types(envelope): + return { + item.headers.get("attachment_type") + for item in envelope.items + if item.headers.get("type") == "attachment" + } + + +def event_tags(event): + """Normalize the event's tags to a dict (Relay may emit dict or list-of-pairs).""" + tags = event.get("tags") or [] + if isinstance(tags, dict): + return tags + return {pair[0]: pair[1] for pair in tags if pair} + + +def project_with_gpu_feature(mini_sentry, project_id, *, enabled=True): + config = mini_sentry.add_full_project_config(project_id) + if enabled: + config["config"].setdefault("features", []).append(GPU_FEATURE) + return config + + +def assert_no_more_envelopes(mini_sentry): + with pytest.raises(queue.Empty): + mini_sentry.get_captured_envelope(timeout=2) + + +def test_gpu_crash_splits_into_two_events(mini_sentry, relay): + project_id = 42 + project_with_gpu_feature(mini_sentry, project_id) + relay = relay(mini_sentry) + + response = relay.send_minidump( + project_id=project_id, + files=[MINIDUMP, scope_attachment(), GPU_DUMP, SHADER_DBG], + ) + cpu_event_id = UUID(response.text.strip()) + + # Both envelopes are forwarded; classify them by their attachments (order is + # not guaranteed). + envelopes = [mini_sentry.get_captured_envelope() for _ in range(2)] + assert_no_more_envelopes(mini_sentry) + cpu, gpu = None, None + for envelope in envelopes: + if "event.nv_gpudmp" in attachment_types(envelope): + gpu = envelope + else: + cpu = envelope + assert cpu is not None and gpu is not None + + # The CPU event keeps the minidump; the GPU attachments moved off it. + cpu_types = attachment_types(cpu) + assert "event.minidump" in cpu_types + assert "event.nv_gpudmp" not in cpu_types + assert "event.nv_shader_debug" not in cpu_types + + # The GPU event carries the dump and shader debug, but not the minidump. + gpu_types = attachment_types(gpu) + assert {"event.nv_gpudmp", "event.nv_shader_debug"} <= gpu_types + assert "event.minidump" not in gpu_types + + # The GPU event is its own (billed) event with a distinct id. + assert UUID(cpu.headers["event_id"]) == cpu_event_id + assert cpu.headers["event_id"] == SCOPE_EVENT_ID + assert cpu.headers["event_id"] != gpu.headers["event_id"] + + # It is a copy of the scope: it inherits the trace, release and tags, so both + # events are trace-connected. + gpu_event = gpu.get_event() + assert gpu_event["release"] == "game@1.0.0" + assert gpu_event["environment"] == "prod" + assert gpu_event["contexts"]["trace"]["trace_id"] == SCOPE_TRACE_ID + assert event_tags(gpu_event).get("custom_tag") == "custom_value" + assert cpu.get_event()["contexts"]["trace"]["trace_id"] == SCOPE_TRACE_ID + + +def test_gpu_crash_not_split_without_feature(mini_sentry, relay): + project_id = 42 + project_with_gpu_feature(mini_sentry, project_id, enabled=False) + relay = relay(mini_sentry) + + relay.send_minidump( + project_id=project_id, + files=[MINIDUMP, scope_attachment(), GPU_DUMP, SHADER_DBG], + ) + + # No split: the GPU dump rides along on the single CPU event. + envelope = mini_sentry.get_captured_envelope() + assert {"event.minidump", "event.nv_gpudmp"} <= attachment_types(envelope) + assert_no_more_envelopes(mini_sentry) + + +def test_gpu_crash_not_split_without_scope(mini_sentry, relay): + # Without a scope (`__sentry-event`) there is nothing to copy the GPU event + # from, so we leave the crash on the CPU event rather than dropping it. + project_id = 42 + project_with_gpu_feature(mini_sentry, project_id) + relay = relay(mini_sentry) + + relay.send_minidump(project_id=project_id, files=[MINIDUMP, GPU_DUMP, SHADER_DBG]) + + envelope = mini_sentry.get_captured_envelope() + assert {"event.minidump", "event.nv_gpudmp"} <= attachment_types(envelope) + assert_no_more_envelopes(mini_sentry) + + +def test_gpu_crash_not_split_without_gpu_dump(mini_sentry, relay): + project_id = 42 + project_with_gpu_feature(mini_sentry, project_id) + relay = relay(mini_sentry) + + relay.send_minidump(project_id=project_id, files=[MINIDUMP, scope_attachment()]) + + envelope = mini_sentry.get_captured_envelope() + assert "event.nv_gpudmp" not in attachment_types(envelope) + assert_no_more_envelopes(mini_sentry) + + +def test_gpu_crash_split_with_processing( + mini_sentry, relay_with_processing, attachments_consumer +): + """The split survives full processing: both events reach the store. + + This exercises the processor's outcome accounting for the copied scope — a + naive attachment clone unbalances it and the GPU event gets dropped. + """ + project_id = 42 + config = project_with_gpu_feature(mini_sentry, project_id) + # The full project config scrubs module paths in the (dummy) minidump; drop it. + del config["config"]["piiConfig"] + + relay = relay_with_processing() + attachments_consumer = attachments_consumer() + + relay.send_minidump( + project_id=project_id, + files=[MINIDUMP, scope_attachment(), GPU_DUMP, SHADER_DBG], + ) + + # Both events reach processing; `get_event_only` skips the attachment chunks. + # Classify by their attachments (envelope order is not guaranteed). + events = {} + for _ in range(2): + message, event = attachments_consumer.get_event_only() + names = {att["name"] for att in message.get("attachments", [])} + kind = "gpu" if "crash.nv-gpudmp" in names else "cpu" + events[kind] = (event, message) + assert set(events) == {"cpu", "gpu"} + + cpu_event, _ = events["cpu"] + gpu_event, gpu_message = events["gpu"] + + # The CPU event is the minidump crash. + assert cpu_event["exception"]["values"][0]["mechanism"]["type"] == "minidump" + + # The GPU event is the copied scope, carrying the GPU crash attachments. + assert gpu_event["event_id"] != cpu_event["event_id"] + assert gpu_event["release"] == "game@1.0.0" + gpu_names = {att["name"] for att in gpu_message.get("attachments", [])} + assert {"crash.nv-gpudmp", "shader-abc.nvdbg"} <= gpu_names + assert "minidump.dmp" not in gpu_names