diff --git a/relay-dynamic-config/src/feature.rs b/relay-dynamic-config/src/feature.rs
index 338efc5b292..b400ad45dea 100644
--- a/relay-dynamic-config/src/feature.rs
+++ b/relay-dynamic-config/src/feature.rs
@@ -107,6 +107,10 @@ pub enum Feature {
/// See .
#[serde(rename = "projects:relay-upload-multipart")]
UploadMultipart,
+ /// Split an NVIDIA GPU crash dump (`.nv-gpudmp`) off a minidump/unreal upload
+ /// into its own event, symbolicated by teapot. Shares the Sentry-side flag.
+ #[serde(rename = "organizations:gpu-crash-symbolication")]
+ NvGpuCrashSplit,
/// Enables OTLP spans to use the Span V2 processing pipeline in Relay.
///
/// This is now the default behaviour of Relay.
diff --git a/relay-server/src/endpoints/minidump.rs b/relay-server/src/endpoints/minidump.rs
index dd3d59232a5..b649da031b8 100644
--- a/relay-server/src/endpoints/minidump.rs
+++ b/relay-server/src/endpoints/minidump.rs
@@ -252,6 +252,7 @@ struct UploadContext<'a> {
upload_attachments: UploadDecision,
upload_minidumps: UploadDecision,
inline_limit: usize,
+ gpu_crash_split: bool,
}
impl UploadContext<'_> {
@@ -337,6 +338,11 @@ impl<'a> AttachmentStrategy for MinidumpAttachmentStrategy<'a> {
}
fn infer_type(&self, field: &Field) -> AttachmentType {
+ match field.file_name() {
+ Some(name) if name.ends_with(".nv-gpudmp") => return AttachmentType::NvGpuDump,
+ Some(name) if name.ends_with(".nvdbg") => return AttachmentType::NvShaderDebug,
+ _ => {}
+ }
match field.name().unwrap_or("") {
MINIDUMP_FIELD_NAME => AttachmentType::Minidump,
ITEM_NAME_BREADCRUMBS1 => AttachmentType::Breadcrumbs,
@@ -526,6 +532,7 @@ async fn upload_context<'a>(
upload_attachments,
upload_minidumps,
inline_limit: global_config.options.attachment_inline_limit,
+ gpu_crash_split: project_config.has_feature(Feature::NvGpuCrashSplit),
}))
}
@@ -657,10 +664,26 @@ async fn handle(
));
return Ok(TextResponse(Some(EventId::new())));
}
+ // Gated on the org feature: only opted-in orgs split GPU crashes into a
+ // second (billed) event. Captured before `upload_context` is consumed below.
+ let gpu_crash_split = upload_context
+ .as_ref()
+ .is_some_and(|ctx| ctx.gpu_crash_split);
+
let items = items(upload_context, &state, &meta, content_type, request)
.await
.reject(&managed_err)?;
- let envelope = envelope(items, meta, managed_err)?;
+
+ let mut envelope = envelope(items, meta, managed_err)?;
+ if gpu_crash_split {
+ let (cpu, gpu) = utils::gpu::split_crash(envelope);
+ if let Some(gpu) = gpu {
+ common::handle_managed_envelope(&state, gpu)
+ .await?
+ .ignore_rate_limits();
+ }
+ envelope = cpu;
+ }
let id = envelope.event_id();
diff --git a/relay-server/src/endpoints/unreal.rs b/relay-server/src/endpoints/unreal.rs
index b0a409a9d01..293f62ad64a 100644
--- a/relay-server/src/endpoints/unreal.rs
+++ b/relay-server/src/endpoints/unreal.rs
@@ -11,13 +11,14 @@ use crate::constants::UNREAL_USER_HEADER;
use crate::endpoints::common::{self, BadStoreRequest, TextResponse};
use crate::envelope::{ContentType, Envelope, Item, ItemType};
use crate::extractors::RequestMeta;
-use crate::middlewares;
+use crate::managed::Managed;
use crate::service::ServiceState;
use crate::services::outcome::{DiscardItemType, DiscardReason};
use crate::services::processor::ProcessingError;
use crate::services::projects::project::ProjectState;
use crate::statsd::RelayCounters;
use crate::utils::extract_items;
+use crate::{middlewares, utils};
#[derive(Debug, Deserialize)]
struct UnrealQuery {
@@ -35,10 +36,11 @@ struct UnrealParams {
}
impl UnrealParams {
+ /// Returns the envelope and whether the org may split off a GPU crash event.
async fn extract_envelope(
self,
state: &ServiceState,
- ) -> Result, BadStoreRequest> {
+ ) -> Result<(Box, bool), BadStoreRequest> {
let Self { meta, query, data } = self;
if data.is_empty() {
@@ -88,7 +90,8 @@ impl UnrealParams {
item.set_unreal_expanded(true);
envelope.add_item(item);
}
- return Ok(envelope);
+ let gpu_crash_split = project_config.has_feature(Feature::NvGpuCrashSplit);
+ return Ok((envelope, gpu_crash_split));
}
}
@@ -96,7 +99,7 @@ impl UnrealParams {
item.set_payload(ContentType::OctetStream, data);
envelope.add_item(item);
- Ok(envelope)
+ Ok((envelope, false))
}
}
@@ -104,11 +107,23 @@ async fn handle(
state: ServiceState,
params: UnrealParams,
) -> axum::response::Result {
- let envelope = params.extract_envelope(&state).await?;
+ let (envelope, gpu_crash_split) = params.extract_envelope(&state).await?;
+ let mut envelope = Managed::from_envelope(envelope, state.outcome_aggregator().clone());
+
+ if gpu_crash_split {
+ let (cpu, gpu) = utils::gpu::split_crash(envelope);
+ if let Some(gpu) = gpu {
+ common::handle_managed_envelope(&state, gpu)
+ .await?
+ .ignore_rate_limits();
+ }
+ envelope = cpu;
+ }
+
let id = envelope.event_id();
// Never respond with a 429 since clients often retry these
- common::handle_envelope(&state, envelope)
+ common::handle_managed_envelope(&state, envelope)
.await?
.ignore_rate_limits();
diff --git a/relay-server/src/envelope/attachment.rs b/relay-server/src/envelope/attachment.rs
index eabaa502460..02920fe5f89 100644
--- a/relay-server/src/envelope/attachment.rs
+++ b/relay-server/src/envelope/attachment.rs
@@ -52,6 +52,14 @@ pub enum AttachmentType {
/// An application UI view hierarchy (json payload).
ViewHierarchy,
+
+ /// An NVIDIA Aftermath GPU crash dump (`.nv-gpudmp`), decoded by teapot.
+ ///
+ /// Carried on its own GPU crash event, split off a minidump/unreal upload.
+ NvGpuDump,
+
+ /// NVIDIA Aftermath shader debug info (`.nvdbg`) accompanying an `NvGpuDump`.
+ NvShaderDebug,
}
impl fmt::Display for AttachmentType {
@@ -67,6 +75,8 @@ impl fmt::Display for AttachmentType {
AttachmentType::UnrealContext => write!(f, "unreal.context"),
AttachmentType::UnrealLogs => write!(f, "unreal.logs"),
AttachmentType::ViewHierarchy => write!(f, "event.view_hierarchy"),
+ AttachmentType::NvGpuDump => write!(f, "event.nv_gpudmp"),
+ AttachmentType::NvShaderDebug => write!(f, "event.nv_shader_debug"),
}
}
}
@@ -99,6 +109,8 @@ impl std::str::FromStr for AttachmentType {
"event.view_hierarchy" => AttachmentType::ViewHierarchy,
"unreal.context" => AttachmentType::UnrealContext,
"unreal.logs" => AttachmentType::UnrealLogs,
+ "event.nv_gpudmp" => AttachmentType::NvGpuDump,
+ "event.nv_shader_debug" => AttachmentType::NvShaderDebug,
_ => return Err(UnknownAttachmentType),
})
}
diff --git a/relay-server/src/envelope/item.rs b/relay-server/src/envelope/item.rs
index 71b9d4ea2da..cdfb7bb96c7 100644
--- a/relay-server/src/envelope/item.rs
+++ b/relay-server/src/envelope/item.rs
@@ -631,13 +631,15 @@ impl Item {
| AttachmentType::EventPayload
| AttachmentType::Prosperodump
| AttachmentType::Breadcrumbs
- | AttachmentType::NintendoSwitchDyingMessage,
+ | AttachmentType::NintendoSwitchDyingMessage
+ | AttachmentType::NvGpuDump,
) => true,
Some(
AttachmentType::Attachment
| AttachmentType::UnrealContext
| AttachmentType::UnrealLogs
- | AttachmentType::ViewHierarchy,
+ | AttachmentType::ViewHierarchy
+ | AttachmentType::NvShaderDebug,
) => false,
// When an outdated Relay instance forwards an unknown attachment type for compatibility,
// we assume that the attachment does not create a new event. This will make it hard
diff --git a/relay-server/src/processing/errors/errors/gpu.rs b/relay-server/src/processing/errors/errors/gpu.rs
new file mode 100644
index 00000000000..a2a1d7a24fc
--- /dev/null
+++ b/relay-server/src/processing/errors/errors/gpu.rs
@@ -0,0 +1,79 @@
+use relay_quotas::{DataCategory, RateLimits};
+
+use crate::envelope::{AttachmentType, Item, ItemType};
+use crate::managed::{Counted, Quantities, RecordKeeper};
+use crate::processing::ForwardContext;
+use crate::processing::errors::errors::{Context, Expansion, SentryError, utils};
+use crate::processing::errors::{Error, Result};
+
+/// An NVIDIA Aftermath GPU crash dump (`.nv-gpudmp`).
+///
+/// Relay splits the GPU crash onto its own event (see [`crate::utils::gpu`]),
+/// which carries a copy of the CPU event's scope plus the dump and any shader
+/// debug info (`.nvdbg`). Here we turn the copied scope into the event and keep
+/// the dump and shader debug info as attachments; Sentry decodes the dump
+/// out-of-band (via teapot), analogous to how a minidump is symbolicated.
+#[derive(Debug)]
+pub struct GpuCrash(pub Item);
+
+impl SentryError for GpuCrash {
+ fn event_category(&self) -> DataCategory {
+ DataCategory::Error
+ }
+
+ fn try_expand(items: &mut Vec- , ctx: Context<'_>) -> Result