fix(replay): Release MediaMuxer when the encoder fails to start

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Fixes
 
 - Release `MediaMuxer` when a replay segment has no encodable frames to avoid a resource leak ([#5583](https://github.com/getsentry/sentry-java/pull/5583))
+- Release `MediaMuxer` when the replay video encoder fails to start to avoid a resource leak ([#5607](https://github.com/getsentry/sentry-java/pull/5607))
 
 ## 8.44.1
 

sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt

@@ -162,7 +162,16 @@ public class ReplayCache(private val options: SentryOptions, private val replayI
               bitRate = bitRate,
             ),
           )
-          .also { it.start() }
+          .apply {
+            // the constructor already opened the MediaMuxer, so release it if start() fails,
+            // otherwise the encoder is never assigned and its resources leak (CloseGuard warning)
+            try {
+              start()
+            } catch (t: Throwable) {
+              release()
+              throw t
+            }
+          }
       }
 
     val step = 1000 / frameRate.toLong()

sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt

@@ -289,10 +289,12 @@ internal class SimpleVideoEncoder(
       mediaCodec.stop()
       mediaCodec.release()
       surface?.release()
-
-      frameMuxer.release()
     } catch (e: Throwable) {
       options.logger.log(DEBUG, "Failed to properly release video encoder", e)
+    } finally {
+      // always release the muxer, even if draining/stopping the codec above threw (e.g. when the
+      // encoder failed to fully start), otherwise its MediaMuxer leaks (CloseGuard warning)
+      frameMuxer.release()
     }
   }
 }

sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt

@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicReference
 import kotlin.test.BeforeTest
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
 import kotlin.test.assertFalse
 import kotlin.test.assertNull
 import kotlin.test.assertTrue
@@ -35,6 +36,7 @@ import org.junit.rules.TemporaryFolder
 import org.junit.runner.RunWith
 import org.robolectric.annotation.Config
 import org.robolectric.shadows.ShadowBitmapFactory
+import org.robolectric.shadows.ShadowCloseGuard
 
 @RunWith(AndroidJUnit4::class)
 @Config(sdk = [26], shadows = [ReplayShadowMediaCodec::class])
@@ -55,6 +57,7 @@ class ReplayCacheTest {
   @BeforeTest
   fun `set up`() {
     ReplayShadowMediaCodec.framesToEncode = 5
+    ReplayShadowMediaCodec.throwOnStart = false
     ShadowBitmapFactory.setAllowInvalidImageData(true)
   }
 
@@ -92,6 +95,26 @@ class ReplayCacheTest {
     assertNull(video)
   }
 
+  @Test
+  fun `releases the muxer when the encoder fails to start`() {
+    ReplayShadowMediaCodec.throwOnStart = true
+    val replayCache = fixture.getSut(tmpDir)
+
+    val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888)
+    replayCache.addFrame(bitmap, 1)
+
+    ShadowCloseGuard.reset()
+    assertFailsWith<IllegalStateException> {
+      replayCache.createVideoOf(5000L, 0, 0, 100, 200, 1, 20_000)
+    }
+
+    val muxerLeaks =
+      ShadowCloseGuard.getErrors().filter { error ->
+        error.stackTrace.any { it.className.contains("MediaMuxer") }
+      }
+    assertTrue(muxerLeaks.isEmpty(), "MediaMuxer was not released: $muxerLeaks")
+  }
+
   @Test
   fun `deletes frames after creating a video`() {
     ReplayShadowMediaCodec.framesToEncode = 3

sentry-android-replay/src/test/java/io/sentry/android/replay/util/ReplayShadowMediaCodec.kt

@@ -15,12 +15,16 @@ class ReplayShadowMediaCodec : ShadowMediaCodec() {
   companion object {
     var frameRate = 1
     var framesToEncode = 5
+    var throwOnStart = false
   }
 
   private val encoded = AtomicBoolean(false)
 
   @Implementation
   fun start() {
+    if (throwOnStart) {
+      throw IllegalStateException("Simulated codec start failure")
+    }
     super.native_start()
   }