fix(replay): Release MediaMuxer when no frames are encoded

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+### Fixes
+
+- Release `MediaMuxer` when a replay segment has no encodable frames to avoid a resource leak ([#5583](https://github.com/getsentry/sentry-java/pull/5583))
+
 ## 8.44.1
 
 ### Fixes

sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt

@@ -199,6 +199,10 @@ public class ReplayCache(private val options: SentryOptions, private val replayI
 
     if (frameCount == 0) {
       options.logger.log(DEBUG, "Generated a video with no frames, not capturing a replay segment")
+      encoderLock.acquire().use {
+        encoder?.release()
+        encoder = null
+      }
       deleteFile(videoFile)
       return null
     }

sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt

@@ -66,8 +66,12 @@
     muxer.writeSampleData(videoTrackIndex, encodedData, bufferInfo)
   }
 
   override fun release() {
-    muxer.stop()
+    // stop() throws if the muxer was never started (e.g. no frame was ever muxed), so we guard it
+    // to ensure release() is always reached and the underlying resources are freed
+    if (started) {
+      muxer.stop()
+    }
     muxer.release()
   }