feat: synchronized audio + haptics (iOS, Android, RN, Flutter, KMP)

Android/Pulsar/src/main/java/com/swmansion/pulsar/audio/AudioHapticPlayer.kt

@@ -0,0 +1,111 @@
+package com.swmansion.pulsar.audio
+
+import android.content.Context
+import android.media.AudioAttributes
+import android.media.SoundPool
+import android.os.Build
+import android.os.Handler
+import android.os.Looper
+import android.util.Log
+import com.swmansion.pulsar.types.SoundData
+
+class AudioHapticPlayer(
+    private val context: Context,
+    private val sound: SoundData,
+    private val hapticChannelsMuted: Boolean,
+) {
+    companion object {
+        private const val TAG = "Pulsar"
+        private const val LOAD_SUCCESS = 0
+    }
+
+    private var soundPool: SoundPool? = null
+    private var soundId: Int = 0
+    private var streamId: Int = 0
+    private var isLoaded = false
+    private var playPending = false
+    private val mainHandler = Handler(Looper.getMainLooper())
+
+    fun load() {
+        val attributesBuilder = AudioAttributes.Builder()
+            .setUsage(AudioAttributes.USAGE_MEDIA)
+            .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION)
+
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+            attributesBuilder.setHapticChannelsMuted(hapticChannelsMuted)
+        }
+
+        val pool = SoundPool.Builder()
+            .setMaxStreams(4)
+            .setAudioAttributes(attributesBuilder.build())
+            .build()
+
+        pool.setOnLoadCompleteListener { _, sampleId, status ->
+            if (status == LOAD_SUCCESS && sampleId == soundId) {
+                isLoaded = true
+                if (playPending) {
+                    playPending = false
+                    scheduleStream()
+                }
+            } else {
+                Log.w(TAG, "Failed to load sound (status=$status): ${sound.uri}")
+            }
+        }
+
+        soundId = loadInto(pool)
+        soundPool = pool
+    }
+
+    fun play() {
+        if (isLoaded) {
+            scheduleStream()
+        } else {
+            playPending = true
+        }
+    }
+
+    fun stop() {
+        playPending = false
+        mainHandler.removeCallbacksAndMessages(null)
+        if (streamId != 0) {
+            soundPool?.stop(streamId)
+            streamId = 0
+        }
+    }
+
+    fun release() {
+        stop()
+        soundPool?.release()
+        soundPool = null
+        isLoaded = false
+    }
+
+    private fun scheduleStream() {
+        if (sound.offset > 0) {
+            mainHandler.postDelayed({ startStream() }, sound.offset)
+        } else {
+            startStream()
+        }
+    }
+
+    private fun startStream() {
+        val pool = soundPool ?: return
+        streamId = pool.play(soundId, sound.volume, sound.volume, 1, 0, 1f)
+    }
+
+    private fun loadInto(pool: SoundPool): Int {
+        val uri = sound.uri
+        val isPath = uri.startsWith("/") || uri.startsWith("file://")
+
+        if (!isPath) {
+            val name = uri.substringAfterLast('/').substringBeforeLast('.')
+            val resId = context.resources.getIdentifier(name, "raw", context.packageName)
+            if (resId != 0) {
+                return pool.load(context, resId, 1)
+            }
+        }
+
+        val path = if (uri.startsWith("file://")) uri.removePrefix("file://") else uri
+        return pool.load(path, 1)
+    }
+}

Android/Pulsar/src/main/java/com/swmansion/pulsar/composers/PatternComposer.kt

@@ -3,21 +3,31 @@ package com.swmansion.pulsar.composers
 import android.os.Build
 import android.os.VibrationEffect
 import android.util.Log
+import com.swmansion.pulsar.audio.AudioHapticPlayer
 import com.swmansion.pulsar.audio.AudioSimulator
 import com.swmansion.pulsar.haptics.HapticEngineWrapper
 import com.swmansion.pulsar.types.PatternData
+import com.swmansion.pulsar.types.SoundData
 
 class PatternComposer(
     private val engine: HapticEngineWrapper,
     private val audioSimulator: AudioSimulator
 ) {
     companion object {
         private const val TAG = "Pulsar"
+
+        internal fun isOggUri(uri: String): Boolean {
+            val extension = uri.substringAfterLast('/').substringAfterLast('.', "").lowercase()
+            return extension == "ogg"
+        }
     }
 
     private var vibrationEffect: VibrationEffect? = null
     private var audioBuffer: ByteArray? = null
 
+    private var soundPlayer: AudioHapticPlayer? = null
+    private var useCoupledHaptics = false
+
     fun parsePattern(hapticsData: PatternData) {
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
             vibrationEffect = try {
@@ -36,20 +46,53 @@ class PatternComposer(
         audioBuffer = audioSimulator.parsePattern(hapticsData)
     }
 
+    fun parsePatternWithSound(hapticsData: PatternData, sound: SoundData) {
+        parsePattern(hapticsData)
+
+        soundPlayer?.release()
+
+        useCoupledHaptics = sound.hapticChannels && isOggUri(sound.uri) && engine.supportsAudioCoupledHaptics()
+
+        soundPlayer = AudioHapticPlayer(
+            context = engine.getContext(),
+            sound = sound,
+            hapticChannelsMuted = !useCoupledHaptics,
+        ).also { it.load() }
+    }
+
     fun play() {
-        audioSimulator.play(audioBuffer)
-        vibrationEffect?.let { engine.vibrate(it) }
+        val player = soundPlayer
+        if (player != null) {
+            player.play()
+            if (!useCoupledHaptics) {
+                vibrationEffect?.let { engine.vibrate(it) }
+            }
+        } else {
+            audioSimulator.play(audioBuffer)
+            vibrationEffect?.let { engine.vibrate(it) }
+        }
     }
 
     fun playAudioOnly() {
-        audioSimulator.play(audioBuffer)
+        val player = soundPlayer
+        if (player != null) {
+            player.play()
+        } else {
+            audioSimulator.play(audioBuffer)
+        }
     }
 
     fun stop() {
+        soundPlayer?.stop()
         audioSimulator.stop()
         engine.stop()
     }
 
+    fun release() {
+        soundPlayer?.release()
+        soundPlayer = null
+    }
+
     private fun summarizePattern(hapticsData: PatternData): String {
         val discreteCount = hapticsData.discretePattern.size
         val amplitudeCount = hapticsData.continuousPattern.amplitude.size

Android/Pulsar/src/main/java/com/swmansion/pulsar/haptics/HapticEngineWrapper.kt

@@ -79,6 +79,12 @@ class HapticEngineWrapper(private val context: Context) {
         return hapticBuilder
     }
 
+    fun getContext(): Context = context
+
+    fun supportsAudioCoupledHaptics(): Boolean {
+        return Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && (vibrator?.hasVibrator() == true)
+    }
+
     fun isAmplitudeSupported(): Boolean {
         return Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && vibrationService.hasAmplitudeControl()
     }

Android/Pulsar/src/main/java/com/swmansion/pulsar/types/Points.kt

@@ -37,6 +37,25 @@ data class ContinuousPattern(
   val frequency: List<ValuePoint>,
 )
 
+/**
+ * @param uri absolute file path, `file://` uri, or the name of a resource in `res/raw`.
+ *   The format comes from the extension; when none is given the default is `.wav`,
+ *   so a bare name like `"beep"` loads `res/raw/beep.wav`.
+ * @param volume playback volume from [0-1].
+ * @param offset delay applied to audio start relative to haptics, in ms (fallback path only).
+ * @param hapticChannels whether the file carries baked haptic channels. When true
+ *   (default) the coupled path is used on supported devices, but only for an
+ *   **explicit** `.ogg` uri; when false the file is treated as plain audio and
+ *   Pulsar's own haptics are always played alongside it. Any non-`.ogg` uri —
+ *   including a bare name (which defaults to `.wav`) — uses the fallback regardless.
+ */
+data class SoundData(
+  val uri: String,
+  val volume: Float = 1f,
+  val offset: Long = 0L,
+  val hapticChannels: Boolean = true,
+)
+
 data class PatternData(
   val continuousPattern: ContinuousPattern,
   val discretePattern: List<ConfigPoint>,

Android/Pulsar/src/test/java/com/swmansion/pulsar/composers/AudioSyncUnitTest.kt

@@ -0,0 +1,67 @@
+package com.swmansion.pulsar.composers
+
+import com.swmansion.pulsar.types.SoundData
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+/**
+ * Pure-JVM coverage of the synchronized-audio decision logic that does not need
+ * the Android framework: the `.ogg` vs. non-`.ogg` detection that drives whether
+ * the coupled (audio-coupled haptics) path or the audio + generated-haptics
+ * fallback is used, plus the [SoundData] defaults the RN/Flutter/KMP wrappers rely on.
+ */
+class AudioSyncUnitTest {
+
+    @Test
+    fun explicitOggUrisAreCoupledEligible() {
+        // .ogg is the only format that can carry baked haptic channels.
+        assertTrue(PatternComposer.isOggUri("boom.ogg"))
+        assertTrue(PatternComposer.isOggUri("sounds/boom.ogg"))
+        assertTrue(PatternComposer.isOggUri("/data/local/boom.ogg"))
+        assertTrue(PatternComposer.isOggUri("file:///assets/boom.ogg"))
+    }
+
+    @Test
+    fun oggDetectionIsCaseInsensitive() {
+        assertTrue(PatternComposer.isOggUri("boom.OGG"))
+        assertTrue(PatternComposer.isOggUri("boom.Ogg"))
+    }
+
+    @Test
+    fun nonOggFilesAreNotCoupledEligible() {
+        assertFalse(PatternComposer.isOggUri("boom.wav"))
+        assertFalse(PatternComposer.isOggUri("boom.mp3"))
+        assertFalse(PatternComposer.isOggUri("boom.WAV"))
+        assertFalse(PatternComposer.isOggUri("boom.m4a"))
+        assertFalse(PatternComposer.isOggUri("/sdcard/boom.mp3"))
+        assertFalse(PatternComposer.isOggUri("file:///assets/boom.wav"))
+    }
+
+    @Test
+    fun bareResourceNameDefaultsToWavAndIsNotCoupled() {
+        // No extension => default .wav => plain audio + fallback haptics, so a
+        // bare name like "beep" can't silently take the coupled path.
+        assertFalse(PatternComposer.isOggUri("beep"))
+        assertFalse(PatternComposer.isOggUri("sounds/beep"))
+        assertFalse(PatternComposer.isOggUri(""))
+    }
+
+    @Test
+    fun soundDataAppliesTheDocumentedDefaults() {
+        val sound = SoundData(uri = "boom.ogg")
+        assertEquals(1f, sound.volume)
+        assertEquals(0L, sound.offset)
+        assertTrue(sound.hapticChannels)
+    }
+
+    @Test
+    fun soundDataKeepsExplicitValues() {
+        val sound = SoundData(uri = "boom.wav", volume = 0.5f, offset = 30L, hapticChannels = false)
+        assertEquals("boom.wav", sound.uri)
+        assertEquals(0.5f, sound.volume)
+        assertEquals(30L, sound.offset)
+        assertFalse(sound.hapticChannels)
+    }
+}

Android/Pulsar/src/test/java/com/swmansion/pulsar/haptics/RobolectricAudioCoupledHapticsTest.kt

@@ -0,0 +1,52 @@
+package com.swmansion.pulsar.haptics
+
+import android.os.Vibrator
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.robolectric.RobolectricTestRunner
+import org.robolectric.RuntimeEnvironment
+import org.robolectric.Shadows.shadowOf
+import org.robolectric.annotation.Config
+
+/**
+ * Robolectric coverage of `HapticEngineWrapper.supportsAudioCoupledHaptics()` — the
+ * capability gate that decides whether an `.ogg` sound file's baked haptic channels
+ * can drive the vibrator (perfect sync) or the audio + generated-haptics fallback is
+ * used. It requires API 29+ (where `setHapticChannelsMuted` exists) and a vibrator.
+ */
+@RunWith(RobolectricTestRunner::class)
+class RobolectricAudioCoupledHapticsTest {
+
+    private fun engineWithVibrator(hasVibrator: Boolean): HapticEngineWrapper {
+        val app = RuntimeEnvironment.getApplication()
+        val vibrator = app.getSystemService(Vibrator::class.java)
+        shadowOf(vibrator).setHasVibrator(hasVibrator)
+        return HapticEngineWrapper(app)
+    }
+
+    @Test
+    @Config(sdk = [34])
+    fun supportedWhenApi29PlusAndVibratorPresent() {
+        assertTrue(engineWithVibrator(hasVibrator = true).supportsAudioCoupledHaptics())
+    }
+
+    @Test
+    @Config(sdk = [34])
+    fun unsupportedWithoutAVibrator() {
+        assertFalse(engineWithVibrator(hasVibrator = false).supportsAudioCoupledHaptics())
+    }
+
+    @Test
+    @Config(sdk = [29])
+    fun supportedExactlyAtApi29() {
+        assertTrue(engineWithVibrator(hasVibrator = true).supportsAudioCoupledHaptics())
+    }
+
+    @Test
+    @Config(sdk = [28])
+    fun unsupportedBelowApi29EvenWithAVibrator() {
+        assertFalse(engineWithVibrator(hasVibrator = true).supportsAudioCoupledHaptics())
+    }
+}