diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 90c3ad221..02fe74e7d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,13 +8,13 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' - name: Cache Gradle packages - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} @@ -24,7 +24,7 @@ jobs: - name: Assemble run: ./gradlew assemble - name: Upload APKs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: apks path: | @@ -34,13 +34,13 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' - name: Cache Gradle packages - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} @@ -52,7 +52,7 @@ jobs: working-directory: . run: ./gradlew test - name: Upload test reports - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: test-reports path: '**/build/reports/tests' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index bd12a2d5d..727f70eb7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,16 +56,16 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v5 with: distribution: 'zulu' java-version: '21' # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -87,6 +87,6 @@ jobs: run: ./gradlew --scan --full-stacktrace -Dorg.gradle.dependency.verification=off compileDebugAndroidTestSources - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c0cd69b11..8cf963265 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -11,8 +11,8 @@ jobs: update-doc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d80b958e0..848b7e9db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,9 +11,9 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index 9d5dc2a5d..c3a63f37e 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -9,9 +9,9 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38f99355a..2b1c47f32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,8 +19,8 @@ jobs: sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm ls /dev/kvm - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 with: java-version: '21' distribution: 'adopt' @@ -43,7 +43,7 @@ jobs: env: INTEGRATION_TESTS_API_KEY: ${{ secrets.INTEGRATION_TESTS_API_KEY }} - name: Upload test reports - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: test-reports-${{ matrix.api-level }} path: | diff --git a/README.md b/README.md index fe37c3e69..ab4a765c2 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,54 @@ -# StreamPack: RTMP and SRT live streaming SDK for Android +[![Maven Central](https://img.shields.io/maven-central/v/io.github.thibaultbee.streampack/streampack-core)](https://central.sonatype.com/artifact/io.github.thibaultbee.streampack/streampack-core) +[![License](https://img.shields.io/github/license/ThibaultBee/StreamPack)](LICENSE.md) +[![API](https://img.shields.io/badge/API-21%2B-brightgreen.svg)](https://android-arsenal.com/api?level=21) +[![Stars](https://img.shields.io/github/stars/ThibaultBee/StreamPack?style=social)](https://github.com/ThibaultBee/StreamPack/stargazers) -StreamPack is a flexible live streaming library for Android made for both demanding video -broadcasters and new video enthusiasts. +# StreamPack — Android Live Streaming SDK (RTMP, SRT, Camera2, Kotlin) -## Hop On Board! 🚀 +StreamPack is a modular, high-performance Android library designed for low-latency live streaming. +It empowers developers to build professional broadcasting applications with support for SRT, RTMP, +and RTMPS protocols. Leveraging the latest Android Camera2 and MediaCodec APIs, it is fully +extensible, allowing for the integration of custom protocols and custom audio/video sources. -⭐ If you like this project, don’t forget to star it! +## 🏗️ 5-minute Quick Start / Boilerplate -💖 Want to support its development? Consider becoming a sponsor. +If you want to create a new application, you should use the +template [StreamPack boilerplate](https://github.com/ThibaultBee/StreamPack-boilerplate). In 5 +minutes, you will be able to stream live video to your server. -🛠️ Contributions are welcome—feel free to open issues or submit pull requests! +## Table of Contents -## Setup +- [Setup](#-setup) +- [Features](#-features) +- [Quick Start](#-quick-start) +- [Permissions](#-permissions) +- [Documentation](#-documentations) +- [Demos](#-demos) +- [Contributing](#-contributing--support) +- [Tips](#-tips) +- [License](#-license) + +## 📦 Setup Get StreamPack core latest artifacts on Maven Central: ```groovy dependencies { - implementation 'io.github.thibaultbee.streampack:streampack-core:3.1.2' - // For UI (incl. PreviewView) - implementation 'io.github.thibaultbee.streampack:streampack-ui:3.1.2' + implementation 'io.github.thibaultbee.streampack:streampack-core:3.2.0' + // For xml UI (incl. PreviewView) + implementation 'io.github.thibaultbee.streampack:streampack-ui:3.2.0' + // Or compose UI (incl. SourcePreview) + implementation 'io.github.thibaultbee.streampack:streampack-compose:3.2.0' // For services (incl. screen capture/media projection service) - implementation 'io.github.thibaultbee.streampack:streampack-services:3.1.2' + implementation 'io.github.thibaultbee.streampack:streampack-services:3.2.0' // For RTMP - implementation 'io.github.thibaultbee.streampack:streampack-rtmp:3.1.2' + implementation 'io.github.thibaultbee.streampack:streampack-rtmp:3.2.0' // For SRT - implementation 'io.github.thibaultbee.streampack:streampack-srt:3.1.2' + implementation 'io.github.thibaultbee.streampack:streampack-srt:3.2.0' } ``` -## Features +## ✨ Features * Video: * Source: Cameras, Screen recorder @@ -59,20 +78,14 @@ dependencies { * Ultra low-latency based on [SRT](https://github.com/Haivision/srt) * Network adaptive bitrate mechanism for [SRT](https://github.com/Haivision/srt) -## Quick start +## 🚀 Quick Start -If you want to create a new application, you should use the -template [StreamPack boilerplate](https://github.com/ThibaultBee/StreamPack-boilerplate). In 5 -minutes, you will be able to stream live video to your server. - -## Getting started - -### Getting started for a camera stream +### 📷 Camera Stream 1. Request the required permissions in your Activity/Fragment. See the [Permissions](#permissions) section for more information. -2. Creates a `View` to display the preview in your layout +2. Create a `View` to display the preview in your layout As a camera preview, you can also use a `SurfaceView`, a `TextureView` or any `View` where that can provide a `Surface`. @@ -90,7 +103,7 @@ minutes, you will be able to stream live video to your server. `app:enableZoomOnPinch` is a boolean to enable zoom on pinch gesture. -3. Instantiates the streamer (main live streaming class) +3. Instantiate the streamer (main live streaming class) A `Streamer` is a class that represents a whole streaming pipeline from capture to endpoint ( incl. encoding, muxing, sending). @@ -117,7 +130,7 @@ minutes, you will be able to stream live video to your server. */ val streamer = cameraSingleStreamer(context = requireContext()) - + /** * To have multiple independent outputs (like for live and record), use a `cameraDualStreamer` or even the `StreamerPipeline`. * @@ -127,12 +140,11 @@ minutes, you will be able to stream live video to your server. * streamer.setVideoSource(CameraSourceFactory()) // Same as streamer.setCameraId(context.defaultCameraId) * streamer.setAudioSource(MicrophoneSourceFactory()) */ - ``` For more information, check the [Streamers](docs/Streamers.md) documentation. -4. Configures audio and video settings +4. Configure audio and video settings ```kotlin val streamer = cameraSingleStreamer(context = requireContext()) // Already instantiated streamer @@ -157,7 +169,9 @@ minutes, you will be able to stream live video to your server. } ``` -5. Inflates the preview with the streamer +5. Inflate the preview with the streamer + + Either `xml` UI ```kotlin val streamer = cameraSingleStreamer(context = requireContext()) // Already instantiated streamer @@ -175,7 +189,17 @@ minutes, you will be able to stream live video to your server. streamer.startPreview(preview) ``` -6. Sets the device orientation + Or Compose UI + + ```kotlin + val streamer = cameraSingleStreamer(context = requireContext()) // Already instantiated streamer + + Box { + SourcePreview(streamer, modifier = Modifier.fillMaxSize()) + } + ``` + +6. Set the device orientation ```kotlin // Already instantiated streamer @@ -223,7 +247,7 @@ minutes, you will be able to stream live video to your server. You can also create your own `targetRotation` provider. -7. Starts the live streaming +7. Start the live streaming ```kotlin // Already instantiated streamer @@ -242,7 +266,7 @@ minutes, you will be able to stream live video to your server. // streamer.startStream("rtmp://serverip:1935/s/streamKey") // For RTMP/RTMPS ``` -8. Stops and releases the streamer +8. Stop and release the streamer ```kotlin // Already instantiated streamer @@ -258,7 +282,7 @@ the [documentation](#documentations). For a complete example, check out the [demos/camera](demos/camera) directory. -### Getting started for a screen recorder stream +### 🖥️ Screen recorder stream 1. Add the `streampack-services` dependency in your `build.gradle` file: @@ -268,17 +292,17 @@ For a complete example, check out the [demos/camera](demos/camera) directory. } ``` -2. Requests the required permissions in your Activity/Fragment. See the +2. Request the required permissions in your Activity/Fragment. See the [Permissions](#permissions) section for more information. -3. Creates a `MyService` that extends `MediaProjectionService` (so you can customize +3. Create a `MyService` that extends `MediaProjectionService` (so you can customize notifications among other things). -4. Creates a screen record `Intent` and requests the activity result +4. Create a screen record `Intent` and requests the activity result ```kotlin MediaProjectionUtils.createScreenCaptureIntent(context = requireContext()) ``` -5. Starts the service +5. Start the service ```kotlin MediaProjectionService.bindService( @@ -299,7 +323,7 @@ For a complete example, check out the [demos/camera](demos/camera) directory. For a complete example, check out the [demos/screenrecorder](demos/screenrecorder) directory . -## Permissions +## 🔑 Permissions You need to add the following permissions in your `AndroidManifest.xml`: @@ -316,7 +340,7 @@ You need to add the following permissions in your `AndroidManifest.xml`: To record locally, you also need to request the following dangerous permission: `android.permission.WRITE_EXTERNAL_STORAGE`. -### Permissions for a camera stream +### Camera stream To use the camera, you need to request the following permission: @@ -341,7 +365,7 @@ For the PlayStore, your application might declare this in its `AndroidManifest.x ``` -### Permissions for a screen recorder stream +### Screen recorder stream To use the screen recorder, you need to request the following permission: @@ -367,7 +391,7 @@ You will also have to declare the `Service`, ``` -## Documentations +## 📖 Documentations - [StreamPack API guide](https://thibaultbee.github.io/StreamPack) - Additional documentations are available in the `docs` directory: @@ -375,7 +399,7 @@ You will also have to declare the `Service`, - [Streamers](docs/Streamers.md) - [Streamer elements](docs/AdvancedStreamer.md) -## Demos +## 🎬 Demos ### Camera and audio demo @@ -401,7 +425,7 @@ Tells FFplay to listen on IP `0.0.0.0` and port `1935`. ffplay -listen 1 -i 'rtmp://0.0.0.0:1935/s/streamKey' ``` -On StreamPack sample app settings, set `Endpoint` -> `Type` to `Stream to a remove RTMP device`, +On StreamPack sample app settings, set `Endpoint` -> `Type` to `Stream to a remote RTMP device`, then set the server `URL` to `rtmp://serverip:1935/s/streamKey`. At this point, StreamPack sample app should successfully sends audio and video frames. On FFplay side, you should be able to watch this live stream. @@ -418,7 +442,15 @@ On StreamPack sample app settings, set the server `IP` to your server IP and ser At this point, StreamPack sample app should successfully sends audio and video frames. On FFplay side, you should be able to watch this live stream. -## Tips +## 🤝 Contributing & Support + +⭐ If you like this project, don’t forget to star it! + +💖 Want to support its development? Consider becoming a sponsor. + +🛠️ Contributions are welcome—feel free to open issues or submit pull requests! + +## 💡 Tips ### RTMP or SRT @@ -489,7 +521,7 @@ Even if StreamPack sdk supports a `minSdkVersion` 21. I strongly recommend to se `minSdkVersion` of your application to a higher version (the highest is the best!) for better performance. -## Licence +## 📄 License Copyright 2021 Thibault B. diff --git a/build.gradle.kts b/build.gradle.kts index f5a6323b9..da27d7574 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,8 +5,8 @@ plugins { } allprojects { - val versionCode by extra { 3_001_002 } - val versionName by extra { "3.1.2" } + val versionCode by extra { 3_002_000 } + val versionName by extra { "3.2.0" } group = "io.github.thibaultbee.streampack" version = versionName diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt index f929a92b6..4405578e4 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubAudioSource.kt @@ -1,12 +1,11 @@ package io.github.thibaultbee.streampack.core.elements.sources import android.content.Context -import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer class StubAudioSource : IAudioSourceInternal { private val _isStreamingFlow = MutableStateFlow(false) @@ -15,13 +14,11 @@ class StubAudioSource : IAudioSourceInternal { private val _configurationFlow = MutableStateFlow(null) val configurationFlow = _configurationFlow.asStateFlow() - override fun fillAudioFrame(frame: RawFrame): RawFrame { - return frame + override val minBufferSize = 0 + override fun fillAudioFrame(buffer: ByteBuffer): Long { + return 0L } - override fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory) = - frameFactory.create(8192, 0) - override suspend fun startStream() { _isStreamingFlow.value = true } diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt index 22eca54d4..883c6ada1 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/sources/StubVideoSource.kt @@ -8,12 +8,12 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.ISurfaceSour import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoFrameSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.VideoSourceConfig -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory import io.github.thibaultbee.streampack.core.elements.utils.time.Timebase import io.github.thibaultbee.streampack.core.pipelines.IVideoDispatcherProvider import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer class StubVideoSurfaceSource(override val timebase: Timebase = Timebase.REALTIME) : StubVideoSource(), @@ -44,8 +44,9 @@ class StubVideoSurfaceSource(override val timebase: Timebase = Timebase.REALTIME } class StubVideoFrameSource : StubVideoSource(), IVideoFrameSourceInternal { - override fun getVideoFrame(frameFactory: IReadOnlyRawFrameFactory) = - frameFactory.create(8192, 0L) + override fun getVideoFrame(buffer: ByteBuffer): Long { + return 0L + } class Factory : IVideoSourceInternal.Factory { override suspend fun create( diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt index 8de63b080..230440dbe 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipelineTest.kt @@ -57,13 +57,13 @@ class StreamerPipelineTest { context: Context, audioSource: IAudioSourceInternal.Factory?, videoSource: IVideoSourceInternal.Factory?, - audioOutputMode: StreamerPipeline.AudioOutputMode = StreamerPipeline.AudioOutputMode.PUSH + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.PUSH ): StreamerPipeline { val pipeline = StreamerPipeline( context, withAudio = audioSource != null, withVideo = videoSource != null, - audioOutputMode = audioOutputMode, + audioInputMode = audioInputMode, ) audioSource?.let { pipeline.setAudioSource(it) } videoSource?.let { pipeline.setVideoSource(it) } @@ -83,8 +83,8 @@ class StreamerPipelineTest { ) streamerPipeline.addOutput(output) - val audioSource = streamerPipeline.audioInput?.sourceFlow?.value as IAudioSourceInternal - val videoSource = streamerPipeline.videoInput?.sourceFlow?.value as IVideoSourceInternal + val audioSource = streamerPipeline.audioInput.sourceFlow.value as IAudioSourceInternal + val videoSource = streamerPipeline.videoInput.sourceFlow.value as IVideoSourceInternal streamerPipeline.startStream() assertTrue(streamerPipeline.isStreamingFlow.first { it }) diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/StubPipelineOutput.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/StubPipelineOutput.kt index f133bf2a5..6ae4db862 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/StubPipelineOutput.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/StubPipelineOutput.kt @@ -223,13 +223,9 @@ internal class StubAudioSyncConfigurableEncodingPipelineOutputInternal : override val endpoint: IEndpoint get() = TODO("Not yet implemented") - override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) { - TODO("Not yet implemented") - } - - override fun removeBitrateRegulatorController() { - TODO("Not yet implemented") - } + override var bitrateRegulatorControllerFactory: IBitrateRegulatorController.Factory? + get() = TODO("Not yet implemented") + set(value) {} override suspend fun open(descriptor: MediaDescriptor) { TODO("Not yet implemented") @@ -272,13 +268,9 @@ internal class StubVideoSurfaceConfigurableEncodingPipelineOutputInternal : override val endpoint: IEndpoint get() = TODO("Not yet implemented") - override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) { - TODO("Not yet implemented") - } - - override fun removeBitrateRegulatorController() { - TODO("Not yet implemented") - } + override var bitrateRegulatorControllerFactory: IBitrateRegulatorController.Factory? + get() = TODO("Not yet implemented") + set(value) {} override suspend fun open(descriptor: MediaDescriptor) { TODO("Not yet implemented") diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt index 1531e0257..ac31423dd 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutputTest.kt @@ -20,7 +20,6 @@ import android.util.Log import androidx.core.net.toFile import androidx.test.platform.app.InstrumentationRegistry import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.DummyEndpoint @@ -37,6 +36,7 @@ import io.github.thibaultbee.streampack.core.pipelines.outputs.IConfigurableVide import io.github.thibaultbee.streampack.core.pipelines.outputs.IPipelineEventOutputInternal import io.github.thibaultbee.streampack.core.pipelines.outputs.isStreaming import io.github.thibaultbee.streampack.core.pipelines.outputs.releaseBlocking +import io.github.thibaultbee.streampack.core.utils.FakeRawFrame import junit.framework.TestCase.assertTrue import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first @@ -317,7 +317,7 @@ class EncodingPipelineOutputTest { try { output.queueAudioFrame( - RawFrame( + FakeRawFrame( ByteBuffer.allocateDirect(16384), Random.nextLong() ) @@ -330,14 +330,11 @@ class EncodingPipelineOutputTest { output.startStream(descriptor) output.queueAudioFrame( - RawFrame( + FakeRawFrame( ByteBuffer.allocateDirect(16384), Random.nextLong() ) ) - - // Wait for frame to be received - assertNotNull(dummyEndpoint.frameFlow.filterNotNull().first()) } companion object { diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt index f8ce74d43..4013ea95e 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/dual/file/CameraDualStreamerFileTest.kt @@ -28,6 +28,7 @@ import io.github.thibaultbee.streampack.core.streamer.dual.utils.DualStreamerCon import io.github.thibaultbee.streampack.core.streamer.utils.StreamerUtils import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.dual.cameraDualStreamer +import io.github.thibaultbee.streampack.core.streamers.dual.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking @@ -72,8 +73,7 @@ class CameraDualStreamerFileTest( private val videoConfig by lazy { DualStreamerConfigUtils.videoConfig( firstDescriptor, secondDescriptor, Size( - VIDEO_WIDTH, - VIDEO_HEIGHT + VIDEO_WIDTH, VIDEO_HEIGHT ) ) } @@ -132,8 +132,8 @@ class CameraDualStreamerFileTest( private const val STREAM_DURATION_MS = 30_000L private const val STREAM_POLLING_MS = 1_000L - private const val VIDEO_WIDTH = 1280 - private const val VIDEO_HEIGHT = 720 + private const val VIDEO_WIDTH = 640 + private const val VIDEO_HEIGHT = 360 @JvmStatic @Parameterized.Parameters( @@ -157,6 +157,10 @@ class CameraDualStreamerFileTest( false, null ), + /* + * Current test on webM failed because of the VP9 encoder. Disable for now. + */ + /* arrayOf( UriMediaDescriptor(FileUtils.createCacheFile("video.webm").toUri()), true, @@ -164,7 +168,8 @@ class CameraDualStreamerFileTest( UriMediaDescriptor(FileUtils.createCacheFile("video.mp4").toUri()), true, null - ), + ) + */ ) } } diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt index 548b4e053..cda7a9718 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerFileTest.kt @@ -28,6 +28,7 @@ import io.github.thibaultbee.streampack.core.streamer.single.utils.SingleStreame import io.github.thibaultbee.streampack.core.streamer.utils.StreamerUtils import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking @@ -108,8 +109,8 @@ class CameraSingleStreamerFileTest( private const val STREAM_DURATION_MS = 30_000L private const val STREAM_POLLING_MS = 1_000L - private const val VIDEO_WIDTH = 1280 - private const val VIDEO_HEIGHT = 720 + private const val VIDEO_WIDTH = 640 + private const val VIDEO_HEIGHT = 360 @JvmStatic @Parameterized.Parameters( @@ -132,11 +133,16 @@ class CameraSingleStreamerFileTest( false, null ), + /* + * Current test on webM failed because of the VP9 encoder. Disable for now. + */ + /* arrayOf( UriMediaDescriptor(FileUtils.createCacheFile("video.webm").toUri()), true, null ) + */ ) } } diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt index 9a1f8a278..34da1422f 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/file/CameraSingleStreamerMultiEndpointTest.kt @@ -27,6 +27,7 @@ import io.github.thibaultbee.streampack.core.streamer.utils.VideoUtils import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt index 0f5fcebd7..ee9232fe6 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/CameraSingleStreamerStateTest.kt @@ -30,6 +30,7 @@ import io.github.thibaultbee.streampack.core.streamer.surface.SurfaceViewTestAct import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.cameraSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.FileUtils import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt index bfe9a2e05..81387d36b 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/state/SingleStreamerStateTest.kt @@ -22,6 +22,7 @@ import io.github.thibaultbee.streampack.core.interfaces.startStream import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import io.github.thibaultbee.streampack.core.streamers.single.setConfig import io.github.thibaultbee.streampack.core.utils.DeviceTest import kotlinx.coroutines.test.runTest import org.junit.After diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/utils/SingleStreamerConfigUtils.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/utils/SingleStreamerConfigUtils.kt index 72cfbd776..72fae4c40 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/utils/SingleStreamerConfigUtils.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/single/utils/SingleStreamerConfigUtils.kt @@ -50,18 +50,23 @@ object SingleStreamerConfigUtils { * * @return a [VideoConfig] for test */ - fun defaultVideoConfig(resolution: Size = Size(640, 360)) = VideoConfig( + fun defaultVideoConfig(resolution: Size = defaultSize) = VideoConfig( resolution = resolution ) /** * Creates an video configuration from a [MediaDescriptor] for test */ - fun videoConfig(descriptor: MediaDescriptor, resolution: Size = Size(640, 360)): VideoConfig { + fun videoConfig(descriptor: MediaDescriptor, resolution: Size = defaultSize): VideoConfig { return if (descriptor.type.containerType == MediaContainerType.WEBM) { - VideoCodecConfig(mimeType = MediaFormat.MIMETYPE_VIDEO_VP9, resolution = resolution) + VideoCodecConfig( + mimeType = MediaFormat.MIMETYPE_VIDEO_VP9, + resolution = resolution + ) } else { defaultVideoConfig() } } + + private val defaultSize = Size(640, 360) } \ No newline at end of file diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt index 6571955c4..d4835c14e 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/streamer/utils/StreamerUtils.kt @@ -4,7 +4,7 @@ import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.Media import io.github.thibaultbee.streampack.core.interfaces.ICloseableStreamer import io.github.thibaultbee.streampack.core.interfaces.startStream import io.github.thibaultbee.streampack.core.streamers.dual.DualStreamer -import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.ISingleStreamer import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.withContext @@ -14,7 +14,7 @@ import kotlin.time.Duration.Companion.seconds object StreamerUtils { suspend fun runSingleStream( - streamer: SingleStreamer, + streamer: ISingleStreamer, descriptor: MediaDescriptor, duration: Duration, pollDuration: Duration = 1.seconds diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/utils/FakeRawFramey.kt similarity index 59% rename from core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt rename to core/src/androidTest/java/io/github/thibaultbee/streampack/core/utils/FakeRawFramey.kt index dc9375392..2edb8a4fe 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IFrameProcessor.kt +++ b/core/src/androidTest/java/io/github/thibaultbee/streampack/core/utils/FakeRawFramey.kt @@ -1,11 +1,11 @@ /* - * Copyright 2025 Thibault B. + * Copyright (C) 2026 Thibault B. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.github.thibaultbee.streampack.core.elements.processing +package io.github.thibaultbee.streampack.core.utils -import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import java.nio.ByteBuffer -/** - * Interface to process a frame. - * - * @param T type of frame to process (probably [RawFrame] or [Frame]) - */ -interface IFrameProcessor { - fun processFrame(frame: T): T -} \ No newline at end of file +data class FakeRawFrame( + override val rawBuffer: ByteBuffer, + override val timestampInUs: Long +) : RawFrame { + override fun close() { + // Nothing to do + } +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/configuration/mediadescriptor/UriMediaDescriptor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/configuration/mediadescriptor/UriMediaDescriptor.kt index 353da3ac0..f7ccfdfd1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/configuration/mediadescriptor/UriMediaDescriptor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/configuration/mediadescriptor/UriMediaDescriptor.kt @@ -1,10 +1,55 @@ package io.github.thibaultbee.streampack.core.configuration.mediadescriptor +import android.content.ContentValues import android.content.Context import android.net.Uri +import android.os.Build +import android.provider.MediaStore +import androidx.core.net.toUri import io.github.thibaultbee.streampack.core.elements.endpoints.MediaContainerType import io.github.thibaultbee.streampack.core.elements.endpoints.MediaSinkType -import androidx.core.net.toUri + +/** + * Creates a media descriptor in the video [android.content.ContentResolver]. + * + * @param context the context to infer container type from content uri + * @param contentValues the content values to create the media descriptor from + * @param customData custom data to attach to the media descriptor + */ +fun videoUriMediaDescriptor( + context: Context, + contentValues: ContentValues, + customData: List = emptyList() +): UriMediaDescriptor { + val collection = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + MediaStore.Video.Media.getContentUri( + MediaStore.VOLUME_EXTERNAL_PRIMARY + ) + } else { + MediaStore.Video.Media.EXTERNAL_CONTENT_URI + } + return UriMediaDescriptor(context, contentValues, collection, customData) +} + +/** + * Creates a media descriptor in the collection [android.content.ContentResolver]. + * + * @param context the context to infer container type from content uri + * @param contentValues the content values to create the media descriptor from + * @param collection the collection to create the media descriptor from + * @param customData custom data to attach to the media descriptor + */ +private fun UriMediaDescriptor( + context: Context, + contentValues: ContentValues, + collection: Uri, + customData: List = emptyList() +): UriMediaDescriptor { + val contentResolver = context.contentResolver + val uri = contentResolver.insert(collection, contentValues) + ?: throw RuntimeException("Unable to create file: $contentValues in $collection") + return UriMediaDescriptor(context, uri, customData) +} /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt index 018f4dbc1..da62dceec 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/data/Frame.kt @@ -16,163 +16,167 @@ package io.github.thibaultbee.streampack.core.elements.data import android.media.MediaFormat -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removePrefixes -import java.io.Closeable +import io.github.thibaultbee.streampack.core.elements.utils.extensions.deepCopy +import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import java.nio.ByteBuffer /** - * A raw frame internal representation. + * Encoded frame representation */ -data class RawFrame( +interface RawFrame : AutoCloseable { /** * Contains an audio or video frame data. */ - val rawBuffer: ByteBuffer, + val rawBuffer: ByteBuffer /** * Presentation timestamp in µs */ - var timestampInUs: Long, + val timestampInUs: Long +} - /** - * A callback to call when frame is closed. - */ - val onClosed: (RawFrame) -> Unit = {} -) : Closeable { - override fun close() { - try { - onClosed(this) - } catch (_: Throwable) { - // Nothing to do +/** + * Deep copy the [RawFrame.rawBuffer] into a new [RawFrame]. + * + * For better memory allocation, you should close the returned frame after usage. + */ +fun RawFrame.deepCopy( + bufferPool: IBufferPool, + timestampInUs: Long = this.timestampInUs, + onClosed: (RawFrame) -> Unit = {} +): RawFrame { + val copy = this.rawBuffer.deepCopy(bufferPool) + return copy( + rawBuffer = copy, timestampInUs = timestampInUs, onClosed = { + onClosed(it) + bufferPool.put(copy) } - } + ) } +/** + * Copy a [RawFrame] to a new [RawFrame]. + * + * For better memory allocation, you should close the returned frame after usage. + */ +fun RawFrame.copy( + rawBuffer: ByteBuffer = this.rawBuffer, + timestampInUs: Long = this.timestampInUs, + onClosed: (RawFrame) -> Unit = {} +): RawFrame { + val pool = RawFramePool.default + return pool.get( + rawBuffer, timestampInUs, + { frame -> + onClosed(frame) + }) +} -data class Frame( +/** + * Encoded frame representation + */ +interface Frame : AutoCloseable { /** * Contains an audio or video frame data. */ - val rawBuffer: ByteBuffer, + val rawBuffer: ByteBuffer /** * Presentation timestamp in µs */ - val ptsInUs: Long, + val ptsInUs: Long /** * Decoded timestamp in µs (not used). */ - val dtsInUs: Long? = null, + val dtsInUs: Long? /** * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) */ - val isKeyFrame: Boolean, + val isKeyFrame: Boolean /** - * Contains csd buffers for key frames and audio frames only. + * Gets the csd buffers for key frames and audio frames. + * * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. * ESDS for AAC frames,... */ - val extra: List?, + val extra: Extra? /** * Contains frame format.. * TODO: to remove */ val format: MediaFormat -) { - init { - removePrefixes() - } +} + +interface WithClosable { + val onClosed: (T) -> Unit } /** - * Removes the [Frame.extra] prefixes from the [Frame.rawBuffer]. - * - * With MediaCodec, the encoded frames may contain prefixes like SPS, PPS for H264/H265 key frames. - * It also modifies the position of the [Frame.rawBuffer] to skip the prefixes.s + * Copy a [Frame] to a new [Frame]. * - * @return A [ByteBuffer] without prefixes. + * For better memory allocation, you should close the returned frame after usage. */ -fun Frame.removePrefixes(): ByteBuffer { - return if (extra != null) { - rawBuffer.removePrefixes(extra) - } else { - rawBuffer - } +fun Frame.copy( + rawBuffer: ByteBuffer = this.rawBuffer, + ptsInUs: Long = this.ptsInUs, + dtsInUs: Long? = this.dtsInUs, + isKeyFrame: Boolean = this.isKeyFrame, + extra: Extra? = this.extra, + format: MediaFormat = this.format, + onClosed: (Frame) -> Unit = {} +): Frame { + val pool = FramePool.default + return pool.get( + rawBuffer, ptsInUs, dtsInUs, isKeyFrame, extra, format, + { frame -> + onClosed(frame) + }) } +/** + * Ensures that extra are not used at the same time. + * + * After accessing the extra, they are automatically rewind for a new usage. + */ +class Extra(private val extraBuffers: List) { + private val lock = Any() -fun FrameWithCloseable( - /** - * Contains an audio or video frame data. - */ - rawBuffer: ByteBuffer, - - /** - * Presentation timestamp in µs - */ - ptsInUs: Long, - - /** - * Decoded timestamp in µs (not used). - */ - dtsInUs: Long?, - - /** - * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) - */ - isKeyFrame: Boolean, - - /** - * Contains csd buffers for key frames and audio frames only. - * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. - * ESDS for AAC frames,... - */ - extra: List?, - - /** - * Contains frame format.. - */ - format: MediaFormat, + private val _length by lazy { extraBuffers.sumOf { it.remaining() } } - /** - * A callback to call when frame is closed. - */ - onClosed: (FrameWithCloseable) -> Unit, -) = FrameWithCloseable( - Frame( - rawBuffer, - ptsInUs, - dtsInUs, - isKeyFrame, - extra, - format - ), - onClosed -) + fun getLength(): Int { + return synchronized(lock) { + _length + } + } -/** - * Frame internal representation. - */ -data class FrameWithCloseable( - val frame: Frame, - val onClosed: (FrameWithCloseable) -> Unit -) : Closeable { - override fun close() { - try { - onClosed(this) - } catch (_: Throwable) { - // Nothing to do + fun get(extra: List.() -> T): T { + return synchronized(lock) { + val result = extraBuffers.extra() + extraBuffers.forEach { it.rewind() } + result } } } /** - * Uses the resource and unwraps the [Frame] to pass it to the given block. + * Gets the duplicated extra. + * + * Prefers to use [Extra.get] as it does not create new resources. */ -inline fun FrameWithCloseable.useAndUnwrap(block: (Frame) -> T) = use { - block(frame) -} \ No newline at end of file +val Extra.extra: List + get() = get { this.map { it.duplicate() } } + +/** + * Gets the duplicated extra at [index] + * + * Prefers to use [Extra.get] as it does not create new resources. + */ +fun Extra.get(index: Int): ByteBuffer { + return get { this[index].duplicate() } +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt index 2bedd128a..b5d480a15 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/IEncoder.kt @@ -16,7 +16,7 @@ package io.github.thibaultbee.streampack.core.elements.encoders import android.view.Surface -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendReleasable import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendStreamable @@ -77,7 +77,7 @@ interface IEncoderInternal : SuspendStreamable, SuspendReleasable, /** * A channel where the encoder will send encoded frames. */ - val outputChannel: SendChannel + val outputChannel: SendChannel } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt index e62806ca5..56de657c1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/encoders/mediacodec/MediaCodecEncoder.kt @@ -22,7 +22,8 @@ import android.media.MediaFormat import android.os.Bundle import android.util.Log import android.view.Surface -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Extra +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.EncoderMode import io.github.thibaultbee.streampack.core.elements.encoders.IEncoderInternal @@ -31,6 +32,8 @@ import io.github.thibaultbee.streampack.core.elements.encoders.mediacodec.extens import io.github.thibaultbee.streampack.core.elements.encoders.mediacodec.extensions.isValid import io.github.thibaultbee.streampack.core.elements.utils.extensions.extra import io.github.thibaultbee.streampack.core.elements.utils.extensions.put +import io.github.thibaultbee.streampack.core.elements.utils.extensions.removePrefixes +import io.github.thibaultbee.streampack.core.elements.utils.pool.FramePool import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -40,6 +43,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext +import java.nio.ByteBuffer import kotlin.math.min /** @@ -59,8 +63,8 @@ internal constructor( private val mediaCodec: MediaCodec private val format: MediaFormat - private var outputFormat: MediaFormat? = null - private val frameFactory by lazy { FrameFactory(mediaCodec, isVideo) } + + private val frameFactory by lazy { FrameFactory(mediaCodec, isVideo, mediaCodec.outputFormat) } private val isVideo = encoderConfig.isVideo private val tag = if (isVideo) VIDEO_ENCODER_TAG else AUDIO_ENCODER_TAG + "(${this.hashCode()})" @@ -350,7 +354,7 @@ internal constructor( info.isValid -> { try { val frame = frameFactory.frame( - index, outputFormat!!, info, tag + index, info, tag ) try { listener.outputChannel.send(frame) @@ -434,7 +438,6 @@ internal constructor( } override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { - outputFormat = format Logger.i(tag, "Format changed : $format") } @@ -521,7 +524,7 @@ internal constructor( } val inputBuffer = requireNotNull(mediaCodec.getInputBuffer(inputBufferId)) val size = min(frame.rawBuffer.remaining(), inputBuffer.remaining()) - inputBuffer.put(frame.rawBuffer, frame.rawBuffer.position(), size) + inputBuffer.put(frame.rawBuffer, 0, size) mediaCodec.queueInputBuffer( inputBufferId, 0, size, frame.timestampInUs, 0 ) @@ -556,7 +559,6 @@ internal constructor( if (outputBufferId >= 0) { processOutputFrameUnsafe(mediaCodec, outputBufferId, bufferInfo) } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - outputFormat = mediaCodec.outputFormat Logger.i(tag, "Format changed: ${mediaCodec.outputFormat}") } } @@ -577,6 +579,18 @@ internal constructor( } } + fun FrameFactory( + codec: MediaCodec, + isVideo: Boolean, + outputFormat: MediaFormat + ): FrameFactory { + val extraBuffers: List? by lazy { outputFormat.extra } + val extra: Extra? by lazy { + extraBuffers?.map { it.duplicate() }?.let { Extra(it) } + } + return FrameFactory(codec, isVideo, outputFormat, extra, extraBuffers) + } + /** * A workaround to address the fact that some AAC encoders do not provide frame with `presentationTimeUs` in order. * If a frame is received with a timestamp lower or equal to the previous one, it is corrected by adding 1 to the previous timestamp. @@ -586,25 +600,32 @@ internal constructor( */ class FrameFactory( private val codec: MediaCodec, - private val isVideo: Boolean - ) { + private val isVideo: Boolean, + private val outputFormat: MediaFormat, + private val extra: Extra?, + private val extraBuffers: List? + ) : AutoCloseable { private var previousPresentationTimeUs = 0L + private val pool = FramePool() + /** * Create a [Frame] from a [RawFrame] * * @return the created frame */ fun frame( - index: Int, outputFormat: MediaFormat, info: BufferInfo, tag: String - ): FrameWithCloseable { + index: Int, + info: BufferInfo, + tag: String + ): Frame { var pts = info.presentationTimeUs if (pts <= previousPresentationTimeUs) { pts = previousPresentationTimeUs + 1 Logger.w(tag, "Correcting timestamp: $pts <= $previousPresentationTimeUs") } previousPresentationTimeUs = pts - return createFrame(codec, index, outputFormat, pts, info.isKeyFrame, tag) + return createFrame(codec, index, pts, info.isKeyFrame, tag) } /** @@ -617,26 +638,28 @@ internal constructor( private fun createFrame( codec: MediaCodec, index: Int, - outputFormat: MediaFormat, ptsInUs: Long, isKeyFrame: Boolean, tag: String - ): FrameWithCloseable { + ): Frame { val buffer = requireNotNull(codec.getOutputBuffer(index)) - return FrameWithCloseable( - buffer, - ptsInUs, // pts - null, // dts + val extra = if (isKeyFrame || !isVideo) { + extra!! + } else { + null + } + val rawBuffer = if (extraBuffers != null) { + buffer.removePrefixes(extraBuffers) + } else { + buffer + } + + return pool.get( + rawBuffer, + ptsInUs, + null, isKeyFrame, - try { - if (isKeyFrame || !isVideo) { - outputFormat.extra - } else { - null - } - } catch (_: Throwable) { - null - }, + extra, outputFormat, onClosed = { try { @@ -647,6 +670,9 @@ internal constructor( }) } + override fun close() { + pool.close() + } } companion object { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt index 7d7d8f721..190e95b7e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/CombineEndpoint.kt @@ -17,7 +17,8 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.utils.extensions.intersect import io.github.thibaultbee.streampack.core.logger.Logger @@ -36,7 +37,6 @@ import kotlinx.coroutines.flow.merge import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch - /** * Combines multiple endpoints into one. * @@ -115,14 +115,6 @@ open class CombineEndpoint( .reduce { acc, iEndpointInfo -> acc intersect iEndpointInfo } } - /** - * Throws [UnsupportedOperationException] because [CombineEndpoint] does not have metrics. - * - * Call [IEndpoint.metrics] on each endpoint to get their metrics. - */ - override val metrics: Any - get() = throw UnsupportedOperationException("CombineEndpoint does not have metrics.") - private fun createNewStreamId(): Int { var i = 0 while (endpointsToStreamIdsMap.keys.any { it.second == i }) { @@ -218,8 +210,7 @@ open class CombineEndpoint( * * If all endpoints write fails, it throws the exception of the first endpoint that failed. */ - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { - val frame = closeableFrame.frame + override suspend fun write(frame: Frame, streamPid: Int) { val throwables = mutableListOf() /** @@ -231,9 +222,10 @@ open class CombineEndpoint( endpointInternals.filter { it.isOpenFlow.value }.forEach { endpoint -> try { val deferred = CompletableDeferred() - val duplicatedFrame = FrameWithCloseable( - frame.copy(rawBuffer = frame.rawBuffer.duplicate()), - { deferred.complete(Unit) }) + val duplicatedFrame = frame.copy( + rawBuffer = frame.rawBuffer.duplicate(), + onClosed = { deferred.complete(Unit) } + ) val endpointStreamId = endpointsToStreamIdsMap[Pair(endpoint, streamPid)]!! deferreds += deferred @@ -246,7 +238,7 @@ open class CombineEndpoint( coroutineScope.launch { deferreds.forEach { it.await() } - closeableFrame.close() + frame.close() } if (throwables.isNotEmpty()) { diff --git a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt similarity index 78% rename from core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt index b4a06c464..9d5b06d0a 100644 --- a/core/src/androidTest/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DummyEndpoint.kt @@ -16,29 +16,24 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context -import android.util.Log import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +/** + * A dummy endpoint for testing. + */ class DummyEndpoint : IEndpointInternal { private val _isOpenFlow = MutableStateFlow(false) override val isOpenFlow = _isOpenFlow.asStateFlow() - private val _frameFlow = MutableStateFlow(null) - val frameFlow = _frameFlow.asStateFlow() - private val _isStreamingFlow = MutableStateFlow(false) val isStreamingFlow = _isStreamingFlow.asStateFlow() - private val _configFlow = MutableStateFlow(null) - val configFlow = _configFlow.asStateFlow() - override val info: IEndpoint.IEndpointInfo get() = TODO("Not yet implemented") @@ -46,8 +41,6 @@ class DummyEndpoint : IEndpointInternal { TODO("Not yet implemented") } - override val metrics: Any - get() = TODO("Not yet implemented") override val throwableFlow: StateFlow = MutableStateFlow(null).asStateFlow() override suspend fun open(descriptor: MediaDescriptor) { @@ -58,20 +51,15 @@ class DummyEndpoint : IEndpointInternal { _isOpenFlow.emit(false) } - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { - val frame = closeableFrame.frame - Log.i(TAG, "write: $frame") - _frameFlow.emit(frame) - closeableFrame.close() + override suspend fun write(frame: Frame, streamPid: Int) { + frame.close() } override suspend fun addStreams(streamConfigs: List): Map { - streamConfigs.forEach { _configFlow.emit(it) } return streamConfigs.associateWith { it.hashCode() } } override suspend fun addStream(streamConfig: CodecConfig): Int { - _configFlow.emit(streamConfig) return streamConfig.hashCode() } @@ -82,10 +70,6 @@ class DummyEndpoint : IEndpointInternal { override suspend fun stopStream() { _isStreamingFlow.emit(false) } - - companion object { - private const val TAG = "DummyEndpoint" - } } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt index c242000ef..a9fc04263 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpoint.kt @@ -18,13 +18,16 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.createDefaultTsServiceInfo -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.CompositeEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.TsMuxer import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.data.TSServiceInfo import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.ContentSink import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.FileSink +import io.github.thibaultbee.streampack.core.elements.metrics.EmptyEndpointMetrics +import io.github.thibaultbee.streampack.core.elements.metrics.EndpointMetrics +import io.github.thibaultbee.streampack.core.elements.metrics.WithEndpointMetrics import io.github.thibaultbee.streampack.core.elements.utils.ConflatedJob import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider @@ -48,7 +51,7 @@ open class DynamicEndpoint( private val context: Context, private val defaultDispatcher: CoroutineDispatcher, private val ioDispatcher: CoroutineDispatcher -) : IEndpointInternal { +) : IEndpointInternal, WithEndpointMetrics { private val coroutineScope = CoroutineScope(defaultDispatcher) private val mutex = Mutex() @@ -84,8 +87,8 @@ open class DynamicEndpoint( override fun getInfo(type: MediaDescriptor.Type) = getEndpoint(type).getInfo(type) - override val metrics: Any - get() = endpoint?.metrics ?: throw IllegalStateException("Endpoint is not opened") + override val metrics: EndpointMetrics + get() = (endpoint as? WithEndpointMetrics)?.metrics ?: EmptyEndpointMetrics init { coroutineScope.launch { @@ -146,8 +149,8 @@ open class DynamicEndpoint( endpoint.addStream(streamConfig) } - override suspend fun write(closeableFrame: FrameWithCloseable, streamPid: Int) = - safeEndpoint { endpoint -> endpoint.write(closeableFrame, streamPid) } + override suspend fun write(frame: Frame, streamPid: Int) = + safeEndpoint { endpoint -> endpoint.write(frame, streamPid) } override suspend fun startStream() = safeEndpoint { endpoint -> endpoint.startStream() } @@ -176,13 +179,14 @@ open class DynamicEndpoint( private fun prepareEndpoint(mediaDescriptor: MediaDescriptor): IEndpointInternal { val endpoint = getEndpoint(mediaDescriptor.type) - if (endpoint is CompositeEndpoint) { - if (endpoint.muxer is TsMuxer) { + if (endpoint is io.github.thibaultbee.streampack.core.elements.endpoints.composites.ICompositeEndpoint) { + val muxer = endpoint.muxer + if (muxer is TsMuxer) { // Clean up services - endpoint.muxer.removeServices() + muxer.removeServices() val serviceInfo = mediaDescriptor.getCustomData(TSServiceInfo::class.java) ?: createDefaultTsServiceInfo() - endpoint.muxer.addService(serviceInfo) + muxer.addService(serviceInfo) } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/Endpoints.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/Endpoints.kt index e31a2710d..f9353a32e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/Endpoints.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/Endpoints.kt @@ -1,8 +1,9 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context -import io.github.thibaultbee.streampack.core.elements.endpoints.composites.CompositeEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.composites.CompositeEndpointWithMetrics import io.github.thibaultbee.streampack.core.elements.endpoints.composites.CompositeEndpoints +import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.ISinkWithMetricsInternal import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.TsMuxer import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.data.TSServiceInfo import kotlinx.coroutines.CoroutineDispatcher @@ -98,6 +99,6 @@ object Endpoints { if (serviceInfo != null) { muxer.addService(serviceInfo) } - return CompositeEndpoint(muxer, sink) + return CompositeEndpointWithMetrics(muxer, sink as ISinkWithMetricsInternal<*>) } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt index b87e8627f..8f4fa80be 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/IEndpoint.kt @@ -18,7 +18,6 @@ package io.github.thibaultbee.streampack.core.elements.endpoints import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.FileSink import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendCloseable @@ -46,14 +45,14 @@ interface IEndpointInternal : IEndpoint, SuspendStreamable, /** * Writes a [Frame] to the [IEndpointInternal]. * - * The [FrameWithCloseable.close] must be called when the frame has been processed and the [Frame.rawBuffer] is not used anymore. - * The [IEndpointInternal] must called [FrameWithCloseable.close] even if the frame is dropped or it somehow crashes. + * The [Frame.close] must be called when the frame has been processed and the [Frame.rawBuffer] is not used anymore. + * The [IEndpointInternal] must called [Frame.close] even if the frame is dropped or it somehow crashes. * - * @param closeableFrame the [Frame] to write + * @param frame the [Frame] to write * @param streamPid the stream id the [Frame] belongs to */ suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, streamPid: Int, ) @@ -171,9 +170,4 @@ interface IEndpoint { val supportedEncoders: List } } - - /** - * Metrics of the endpoint. - */ - val metrics: Any } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaContainerType.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaContainerType.kt index b9821d2ab..9284f961c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaContainerType.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaContainerType.kt @@ -15,6 +15,9 @@ enum class MediaContainerType(val values: Set) { companion object { private fun inferFromExtension(extension: String): MediaContainerType { + if (extension.isEmpty()) { + throw IllegalArgumentException("Extension cannot be empty") + } val type = entries.firstOrNull { it.values.contains(extension) } return type ?: throw IllegalArgumentException("Unsupported extension: $extension") } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt index 1d30fba65..0be8a776e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaMuxerEndpoint.kt @@ -24,7 +24,7 @@ import android.media.MediaMuxer.OutputFormat import android.os.Build import android.os.ParcelFileDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider @@ -64,9 +64,6 @@ class MediaMuxerEndpoint( override fun getInfo(type: MediaDescriptor.Type) = Companion.getInfo(type) - override val metrics: Any - get() = TODO("Not yet implemented") - private val _isOpenFlow = MutableStateFlow(false) override val isOpenFlow = _isOpenFlow.asStateFlow() @@ -142,9 +139,8 @@ class MediaMuxerEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) = withContext(ioDispatcher) { - val frame = closeableFrame.frame mutex.withLock { try { if (state != State.STARTED && state != State.PENDING_START) { @@ -183,7 +179,7 @@ class MediaMuxerEndpoint( Logger.e(TAG, "Error while writing frame: ${t.message}") throw t } finally { - closeableFrame.close() + frame.close() } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt index f54b85d94..939a6af6d 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/MediaSinkType.kt @@ -22,11 +22,11 @@ enum class MediaSinkType(val schemes: Set) { if (isLocalFileScheme(scheme)) { return FILE } - try { - return entries.first { it.schemes.contains(scheme) } - } catch (t: Throwable) { - throw IllegalArgumentException("Unknown scheme: $scheme", t) + if (scheme.isNullOrEmpty()) { + throw IllegalArgumentException("Scheme cannot be empty") } + return entries.firstOrNull { it.schemes.contains(scheme) } + ?: throw IllegalArgumentException("Unknown scheme: $scheme") } private fun isLocalFileScheme(scheme: String?): Boolean { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt index 92f58fcee..51493a843 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/CompositeEndpoint.kt @@ -17,7 +17,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites import android.content.Context import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal @@ -25,7 +25,9 @@ import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data. import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.IMuxer import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.IMuxerInternal import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.ISinkInternal +import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.ISinkWithMetricsInternal import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.SinkConfiguration +import io.github.thibaultbee.streampack.core.elements.metrics.WithEndpointMetrics import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -37,11 +39,10 @@ import kotlinx.coroutines.sync.withLock /** * An [IEndpointInternal] implementation that combines a [IMuxerInternal] and a [ISinkInternal]. */ -class CompositeEndpoint( +open class CompositeEndpoint( override val muxer: IMuxerInternal, override val sink: ISinkInternal -) : - ICompositeEndpointInternal { +) : ICompositeEndpointInternal { /** * The video and audio configurations. * It is used to configure the sink. @@ -51,9 +52,6 @@ class CompositeEndpoint( override val info by lazy { EndpointInfo(muxer.info) } override fun getInfo(type: MediaDescriptor.Type) = info - override val metrics: Any - get() = sink.metrics - init { muxer.listener = object : IMuxerInternal.IMuxerListener { @@ -80,9 +78,9 @@ class CompositeEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, streamPid: Int - ) = muxer.write(closeableFrame, streamPid) + ) = muxer.write(frame, streamPid) override suspend fun addStreams(streamConfigs: List): Map { return mutex.withLock { @@ -131,6 +129,14 @@ class CompositeEndpoint( } } +/** + * An [IEndpointInternal] implementation of [CompositeEndpoint] with [WithEndpointMetrics]. + */ +class CompositeEndpointWithMetrics( + muxer: IMuxerInternal, + sink: ISinkWithMetricsInternal +) : CompositeEndpoint(muxer, sink), WithEndpointMetrics by sink + /** * A factory to build a [CompositeEndpoint]. */ @@ -145,3 +151,18 @@ class CompositeEndpointFactory( return CompositeEndpoint(muxer, sink) } } + +/** + * A factory to build a [CompositeEndpointWithMetrics]. + */ +class CompositeEndpointWithMetricsFactory( + val muxer: IMuxerInternal, + val sink: ISinkWithMetricsInternal +) : IEndpointInternal.Factory { + override fun create( + context: Context, + dispatcherProvider: IDispatcherProvider + ): IEndpointInternal { + return CompositeEndpointWithMetrics(muxer, sink) + } +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt index 1b609738b..108736b7c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/IMuxerInternal.kt @@ -15,7 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data.Packet import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.interfaces.Releasable @@ -32,7 +32,7 @@ interface IMuxerInternal : fun onOutputFrame(packet: Packet) } - fun write(closeableFrame: FrameWithCloseable, streamPid: Int) + fun write(frame: Frame, streamPid: Int) fun addStreams(streamsConfig: List): Map diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt index 4c4f630e5..b96ca44b2 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/Mp4Muxer.kt @@ -15,9 +15,8 @@ */ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4 -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data.Packet -import io.github.thibaultbee.streampack.core.elements.data.useAndUnwrap import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.IMuxerInternal import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.boxes.FileTypeBox @@ -58,9 +57,9 @@ class Mp4Muxer( override val streamConfigs: List get() = tracks.map { it.config } - override fun write(closeableFrame: FrameWithCloseable, streamPid: Int) { + override fun write(frame: Frame, streamPid: Int) { synchronized(this) { - closeableFrame.useAndUnwrap { frame -> + frame.use { frame -> if (segmenter!!.mustWriteSegment(frame)) { writeSegment() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt index 2ff9387c4..373b6eb42 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/Chunk.kt @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.extra import io.github.thibaultbee.streampack.core.elements.utils.extensions.unzip import java.nio.ByteBuffer @@ -50,8 +51,9 @@ class Chunk(val id: Int) { private val sampleSizes: List get() = samples.map { it.frame.rawBuffer.remaining() } + // TODO: pass `Extra` instead of `List` val extra: List> - get() = samples.mapNotNull { it.frame.extra }.unzip() + get() = samples.mapNotNull { it.frame.extra?.extra }.unzip() val format: List get() = samples.map { it.frame.format } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt index 076d1c132..28dde5d47 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/mp4/models/TrackChunks.kt @@ -18,6 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaFormat import android.util.Size import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig @@ -55,7 +56,6 @@ import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxer import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.boxes.VPCodecConfigurationBox import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.utils.createHandlerBox import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.mp4.utils.createTypeMediaHeaderBox -import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.elements.utils.av.audio.opus.OpusCsdParser import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.AudioSpecificConfigDescriptor import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.ESDescriptor @@ -63,12 +63,13 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.descriptors.SLCon import io.github.thibaultbee.streampack.core.elements.utils.av.video.avc.AVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.hevc.HEVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.vpx.VPCodecConfigurationRecord -import io.github.thibaultbee.streampack.core.elements.utils.extensions.clone +import io.github.thibaultbee.streampack.core.elements.utils.extensions.deepCopy import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAnnexB import io.github.thibaultbee.streampack.core.elements.utils.extensions.isAvcc -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.resolution import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize +import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import java.nio.ByteBuffer /** @@ -175,7 +176,7 @@ class TrackChunks( } val frameCopy = - frame.copy(rawBuffer = frame.rawBuffer.clone()) // Do not keep mediacodec buffer + frame.copy(rawBuffer = frame.rawBuffer.deepCopy()) // Do not keep mediacodec buffer chunks.last().add(frameId, frameCopy) frameId++ } @@ -183,34 +184,38 @@ class TrackChunks( fun write() { chunks.forEach { chunk -> chunk.writeTo { frame -> - when (track.config.mimeType) { - MediaFormat.MIMETYPE_VIDEO_HEVC, - MediaFormat.MIMETYPE_VIDEO_AVC -> { - if (frame.rawBuffer.isAnnexB) { - // Replace start code with size (from Annex B to AVCC) - val noStartCodeBuffer = frame.rawBuffer.removeStartCode() - val sizeBuffer = ByteBuffer.allocate(4) - sizeBuffer.putInt(0, noStartCodeBuffer.remaining()) - onNewSample(sizeBuffer) - onNewSample(noStartCodeBuffer) - } else if (frame.rawBuffer.isAvcc) { - onNewSample(frame.rawBuffer) - } else { - throw IllegalArgumentException( - "Unsupported buffer format: buffer start with 0x${ - frame.rawBuffer.get( - 0 - ).toString(16) - }, 0x${frame.rawBuffer.get(1).toString(16)}, 0x${ - frame.rawBuffer.get(2).toString(16) - }, 0x${frame.rawBuffer.get(3).toString(16)}" - ) + try { + when (track.config.mimeType) { + MediaFormat.MIMETYPE_VIDEO_HEVC, + MediaFormat.MIMETYPE_VIDEO_AVC -> { + if (frame.rawBuffer.isAnnexB) { + // Replace start code with size (from Annex B to AVCC) + val noStartCodeBuffer = frame.rawBuffer.skipStartCode() + val sizeBuffer = ByteBuffer.allocate(4) + sizeBuffer.putInt(0, noStartCodeBuffer.remaining()) + onNewSample(sizeBuffer) + onNewSample(noStartCodeBuffer) + } else if (frame.rawBuffer.isAvcc) { + onNewSample(frame.rawBuffer) + } else { + throw IllegalArgumentException( + "Unsupported buffer format: buffer start with 0x${ + frame.rawBuffer.get( + 0 + ).toString(16) + }, 0x${frame.rawBuffer.get(1).toString(16)}, 0x${ + frame.rawBuffer.get(2).toString(16) + }, 0x${frame.rawBuffer.get(3).toString(16)}" + ) + } } - } - else -> { - onNewSample(frame.rawBuffer) - } // Nothing + else -> { + onNewSample(frame.rawBuffer) + } // Nothing + } + } finally { + frame.close() } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt index d0ab92de7..1376bc07e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxer.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxe import android.media.MediaCodecInfo import android.media.MediaFormat import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.IMuxerInternal @@ -40,7 +40,7 @@ import java.util.MissingFormatArgumentException import kotlin.random.Random class TsMuxer : IMuxerInternal { - private val byteBufferPool = ByteBufferPool(true) + private val tsBufferPool = ByteBufferPool(true) override val info by lazy { TSMuxerInfo } private val tsServices = mutableListOf() @@ -60,10 +60,10 @@ class TsMuxer : IMuxerInternal { private val tsId = Random.nextInt(Byte.MIN_VALUE.toInt(), Byte.MAX_VALUE.toInt()).toShort() private var pat = Pat( - byteBufferPool, listener, tsServices, tsId, packetCount = 0 + tsBufferPool, listener, tsServices, tsId, packetCount = 0 ) private var sdt = Sdt( - byteBufferPool, listener, tsServices, tsId, packetCount = 0 + tsBufferPool, listener, tsServices, tsId, packetCount = 0 ) override val streamConfigs: List @@ -72,30 +72,30 @@ class TsMuxer : IMuxerInternal { /** * Encodes a frame to MPEG-TS format. * Each audio frames and each video key frames must come with an extra buffer containing sps, pps,... - * @param closeableFrame frame to mux + * @param frame frame to mux * @param streamPid Pid of frame stream. Throw a NoSuchElementException if streamPid refers to an unknown stream */ override fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame - try { + frame.use { frame -> val pes = getPes(streamPid.toShort()) val mimeType = pes.stream.config.mimeType val newFrame = when { mimeType == MediaFormat.MIMETYPE_VIDEO_AVC -> { // Copy sps & pps before buffer if (frame.isKeyFrame) { - if (frame.extra == null) { - throw MissingFormatArgumentException("Missing extra for AVC") - } + val extra = frame.extra + ?: throw MissingFormatArgumentException("Missing extra for AVC") val buffer = - ByteBuffer.allocate(6 + frame.extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(6 + extra.getLength() + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the AVC access unit buffer.putInt(0x00000001) buffer.put(0x09.toByte()) buffer.put(0xf0.toByte()) - frame.extra.forEach { buffer.put(it) } + extra.get { + forEach { buffer.put(it) } + } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) @@ -107,17 +107,18 @@ class TsMuxer : IMuxerInternal { mimeType == MediaFormat.MIMETYPE_VIDEO_HEVC -> { // Copy sps & pps & vps before buffer if (frame.isKeyFrame) { - if (frame.extra == null) { - throw MissingFormatArgumentException("Missing extra for HEVC") - } + val extra = frame.extra + ?: throw MissingFormatArgumentException("Missing extra for HEVC") val buffer = - ByteBuffer.allocate(7 + frame.extra.sumOf { it.limit() } + frame.rawBuffer.limit()) + ByteBuffer.allocate(7 + extra.getLength() + frame.rawBuffer.limit()) // Add access unit delimiter (AUD) before the HEVC access unit buffer.putInt(0x00000001) buffer.put(0x46.toByte()) buffer.put(0x01.toByte()) buffer.put(0x50.toByte()) - frame.extra.forEach { buffer.put(it) } + extra.get { + forEach { buffer.put(it) } + } buffer.put(frame.rawBuffer) buffer.rewind() frame.copy(rawBuffer = buffer) @@ -133,11 +134,13 @@ class TsMuxer : IMuxerInternal { frame.rawBuffer, pes.stream.config as AudioCodecConfig ).toByteBuffer() } else { - LATMFrameWriter.fromDecoderSpecificInfo( - frame.rawBuffer, - frame.extra!!.first() - ) - .toByteBuffer() + frame.extra!!.get { + LATMFrameWriter.fromDecoderSpecificInfo( + frame.rawBuffer, + this.first() + ) + .toByteBuffer() + } } ) } @@ -157,11 +160,15 @@ class TsMuxer : IMuxerInternal { else -> throw IllegalArgumentException("Unsupported mimeType $mimeType") } - synchronized(this) { - generateStreams(newFrame, pes) + try { + synchronized(this) { + generateStreams(newFrame, pes) + } + } finally { + if (frame != newFrame) { + newFrame.close() + } } - } finally { - closeableFrame.close() } } @@ -352,12 +359,12 @@ class TsMuxer : IMuxerInternal { service.pmt = service.pmt?.apply { versionNumber = (versionNumber + 1).toByte() streams = service.streams - } ?: Pmt(byteBufferPool, listener, service, service.streams, getNewPid()) + } ?: Pmt(tsBufferPool, listener, service, service.streams, getNewPid()) // Init PES newStreams.forEach { Pes( - byteBufferPool, + tsBufferPool, listener, it, service.pcrPid == it.pid, @@ -433,12 +440,12 @@ class TsMuxer : IMuxerInternal { tsServices.forEach { removeStreams(it) } - byteBufferPool.clear() + tsBufferPool.clear() } override fun release() { tsServices.clear() - byteBufferPool.close() + tsBufferPool.close() } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/sinks/AbstractSink.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/sinks/AbstractSink.kt index 91ac63a37..de4c45efa 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/sinks/AbstractSink.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/sinks/AbstractSink.kt @@ -13,9 +13,6 @@ import io.github.thibaultbee.streampack.core.logger.Logger abstract class AbstractSink : ISinkInternal { abstract val supportedSinkTypes: List - override val metrics: Any - get() = TODO("Not yet implemented") - override suspend fun open(mediaDescriptor: MediaDescriptor) { if (isOpenFlow.value) { Logger.w(TAG, "Sink is already opened") diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/sinks/ISink.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/sinks/ISink.kt index 1e36c9cdd..052817cfe 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/sinks/ISink.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/sinks/ISink.kt @@ -22,6 +22,8 @@ import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendCloseabl import io.github.thibaultbee.streampack.core.elements.interfaces.SuspendStreamable import kotlinx.coroutines.flow.StateFlow +import io.github.thibaultbee.streampack.core.elements.metrics.WithEndpointMetrics + interface ISinkInternal : ISink, Configurable, SuspendStreamable, SuspendCloseable { /** @@ -39,15 +41,12 @@ interface ISinkInternal : ISink, Configurable, SuspendStreama suspend fun write(packet: Packet): Int } +interface ISinkWithMetricsInternal : ISinkInternal, WithEndpointMetrics + interface ISink { /** * Whether if the endpoint is opened. * For example, if the file is opened for [FileSink]. */ val isOpenFlow: StateFlow - - /** - * Metrics of the sink. - */ - val metrics: Any } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/interfaces/Snapshotable.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/interfaces/Snapshotable.kt new file mode 100644 index 000000000..168efe8f5 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/interfaces/Snapshotable.kt @@ -0,0 +1,79 @@ +package io.github.thibaultbee.streampack.core.elements.interfaces + +import android.graphics.Bitmap +import androidx.annotation.IntRange +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.io.FileOutputStream +import java.io.OutputStream + +/** + * An interface to take a snapshot of the current video frame. + */ +interface ISnapshotable { + /** + * Takes a snapshot of the current video frame. + * + * The snapshot is returned as a [Bitmap]. + * + * @param rotationDegrees The rotation to apply to the snapshot, in degrees. 0 means no rotation. + * @return The snapshot as a [Bitmap]. + */ + suspend fun takeSnapshot(@IntRange(from = 0, to = 359) rotationDegrees: Int = 0): Bitmap +} + +/** + * Takes a JPEG snapshot of the current video frame. + * + * The snapshot is saved to the specified file. + * + * @param filePathString The path of the file to save the snapshot to. + * @param quality The quality of the JPEG, from 0 to 100. + * @param rotationDegrees The rotation to apply to the snapshot, in degrees. + */ +suspend fun ISnapshotable.takeJpegSnapshot( + filePathString: String, + @IntRange(from = 0, to = 100) quality: Int = 100, + @IntRange(from = 0, to = 359) rotationDegrees: Int = 0, + dispatcher: CoroutineDispatcher = Dispatchers.IO +) = takeJpegSnapshot(withContext(dispatcher) { + FileOutputStream(filePathString) +}, quality, rotationDegrees) + + +/** + * Takes a JPEG snapshot of the current video frame. + * + * The snapshot is saved to the specified file. + * + * @param file The file to save the snapshot to. + * @param quality The quality of the JPEG, from 0 to 100. + * @param rotationDegrees The rotation to apply to the snapshot, in degrees. + */ +suspend fun ISnapshotable.takeJpegSnapshot( + file: File, + @IntRange(from = 0, to = 100) quality: Int = 100, + @IntRange(from = 0, to = 359) rotationDegrees: Int = 0, + dispatcher: CoroutineDispatcher = Dispatchers.IO +) = takeJpegSnapshot(withContext(dispatcher) { + FileOutputStream(file) +}, quality, rotationDegrees) + +/** + * Takes a snapshot of the current video frame. + * + * The snapshot is saved as a JPEG to the specified output stream. + * @param outputStream The output stream to save the snapshot to. + * @param quality The quality of the JPEG, from 0 to 100. + * @param rotationDegrees The rotation to apply to the snapshot, in degrees. + */ +suspend fun ISnapshotable.takeJpegSnapshot( + outputStream: OutputStream, + @IntRange(from = 0, to = 100) quality: Int = 100, + @IntRange(from = 0, to = 359) rotationDegrees: Int = 0 +) { + val bitmap = takeSnapshot(rotationDegrees) + bitmap.compress(Bitmap.CompressFormat.JPEG, quality, outputStream) +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/metrics/EndpointMetrics.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/metrics/EndpointMetrics.kt new file mode 100644 index 000000000..1aded53d6 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/metrics/EndpointMetrics.kt @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.metrics + +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.isActive +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds + +interface BasicEndpointMetrics { + /** + * The duration of the interval + */ + val uptime: Duration + + /** + * The number of packets written. + */ + val packetsWritten: Long + + /** + * The number of packets dropped before writing (e.g. due to congestion or timeout). + */ + val packetsWriteDropped: Long + + /** + * The number of packets lost during the transmission. + */ + val packetsWriteLost: Long + + /** + * The number of bytes successfully written. + */ + val bytesWritten: Long + + /** + * The number of bytes dropped before writing (e.g. due to congestion or timeout). + */ + val bytesWriteDropped: Long + /** + * Subtracts two [BasicEndpointMetrics]s. + */ + operator fun minus(other: BasicEndpointMetrics): BasicEndpointMetrics { + return object : BasicEndpointMetrics { + override val uptime = this@BasicEndpointMetrics.uptime - other.uptime + override val packetsWritten = this@BasicEndpointMetrics.packetsWritten - other.packetsWritten + override val packetsWriteDropped = this@BasicEndpointMetrics.packetsWriteDropped - other.packetsWriteDropped + override val packetsWriteLost = this@BasicEndpointMetrics.packetsWriteLost - other.packetsWriteLost + override val bytesWritten = this@BasicEndpointMetrics.bytesWritten - other.bytesWritten + override val bytesWriteDropped = this@BasicEndpointMetrics.bytesWriteDropped - other.bytesWriteDropped + } + } +} + +/** + * The total written bitrate in bits per second (bps). + */ +val BasicEndpointMetrics.writtenBitrateInBps: Long + get() = uptime.inWholeMilliseconds.let { if (it == 0L) 0L else (bytesWritten * 8000) / it } + +/** + * Endpoint metrics interface + */ +interface EndpointMetrics : BasicEndpointMetrics { + /** + * The implementation-specific metrics wrapper. + */ + val rawMetrics: T +} + +/** + * An empty implementation of [EndpointMetrics] that returns zeros for all metrics. + */ +object EmptyEndpointMetrics : EndpointMetrics { + override val uptime: Duration = Duration.ZERO + override val packetsWritten: Long = 0L + override val packetsWriteDropped: Long = 0L + override val packetsWriteLost: Long = 0L + override val bytesWritten: Long = 0L + override val bytesWriteDropped: Long = 0L + override val rawMetrics: Any = Unit +} + +/** + * A specific [WithMetrics] for [EndpointMetrics]. + * + * The members from [BasicEndpointMetrics] represent cumulative metrics. + */ +interface WithEndpointMetrics : WithMetrics> + +/** + * Represents a pair of instant and cumulative metrics. + */ +data class TrackedMetrics( + val instant: BasicEndpointMetrics, + val cumulative: BasicEndpointMetrics +) + +/** + * Returns a Flow that emits the [BasicEndpointMetrics] difference since the last emission. + * Every collector gets its own isolated [EndpointMetricsTracker] to prevent state collisions. + * + * @param interval The delay between emissions. + */ +fun WithEndpointMetrics<*>.metricsFlow(interval: Duration = 1000.milliseconds): Flow = flow { + val tracker = EndpointMetricsTracker(this@metricsFlow) + while (currentCoroutineContext().isActive) { + delay(interval) + emit(TrackedMetrics(tracker.instant, tracker.cumulative)) + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/metrics/EndpointMetricsTracker.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/metrics/EndpointMetricsTracker.kt new file mode 100644 index 000000000..52e8cdc29 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/metrics/EndpointMetricsTracker.kt @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.metrics + +/** + * Tracks [BasicEndpointMetrics] over time, providing both cumulative and instant (diff since last read) metrics. + */ +class EndpointMetricsTracker internal constructor(private val metricsProvider: WithEndpointMetrics<*>) { + private var lastMetrics: BasicEndpointMetrics? = null + + /** + * The cumulative metrics since the start. + */ + val cumulative: BasicEndpointMetrics + get() = metricsProvider.metrics + + /** + * The instant metrics (difference between the current and the previous read). + * If no previous read exists, it equals the current metrics. + * Avoid calling this from 2 different parts. + */ + @get:Synchronized + val instant: BasicEndpointMetrics + get() { + val current = metricsProvider.metrics + val last = lastMetrics + lastMetrics = current + + return if (last != null) { + current - last + } else { + current + } + } + + /** + * The implementation-specific metrics wrapper. + */ + val rawMetrics + get() = metricsProvider.metrics.rawMetrics +} + +/** + * Creates an [EndpointMetricsTracker] for this [WithEndpointMetrics]. + */ +fun WithEndpointMetrics<*>.createMetricsTracker() = EndpointMetricsTracker(this) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/metrics/WithMetrics.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/metrics/WithMetrics.kt new file mode 100644 index 000000000..e0bff0cbb --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/metrics/WithMetrics.kt @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.metrics + +/** + * Minimal metrics interface + */ +interface WithMetrics { + /** + * Metrics of the element. + */ + val metrics: T +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt new file mode 100644 index 000000000..0bad99623 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/IEffectProcessor.kt @@ -0,0 +1,62 @@ +/* + * Copyright 2025 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.processing + +/** + * Interface to process a data and returns a result. + * + * @param T type of data to proces) + */ +interface IProcessor { + /** + * Process a data and returns a result. + * + * @param data data to process + * @return processed data + */ + fun process(data: T): T +} + +/** + * Interface to process a data and returns a result. + * + * @param T type of data to proces) + */ +interface IEffectProcessor { + /** + * Process a data and returns a result. + * + * @param isMuted whether the data contains only 0 + * @param data data to process + * @return processed data + */ + fun process(isMuted: Boolean, data: T): T +} + +/** + * Interface to process a data. + * + * @param T type of data to process + */ +interface IEffectConsumer { + /** + * Process a data. + * + * @param isMuted whether the data contains only 0 + * @param data data to process + */ + fun consume(isMuted: Boolean, data: T) +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt index 3dd308c16..dc37e2d66 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/RawFramePullPush.kt @@ -16,8 +16,9 @@ package io.github.thibaultbee.streampack.core.elements.processing import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFrameFactory +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioFrameSourceInternal +import io.github.thibaultbee.streampack.core.elements.utils.pool.ByteBufferPool +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -30,80 +31,85 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.atomic.AtomicBoolean -fun RawFramePullPush( - frameProcessor: IFrameProcessor, - onFrame: suspend (RawFrame) -> Unit, - processDispatcher: CoroutineDispatcher, - isDirect: Boolean = true -) = RawFramePullPush(frameProcessor, onFrame, RawFrameFactory(isDirect), processDispatcher) - /** * A component that pull a frame from an input and push it to [onFrame] output. * * @param frameProcessor the frame processor * @param onFrame the output frame callback - * @param frameFactory the frame factory to create frames + * @param bufferPool the [ByteBuffer] pool * @param processDispatcher the dispatcher to process frames on */ class RawFramePullPush( - private val frameProcessor: IFrameProcessor, + private val frameProcessor: IProcessor, val onFrame: suspend (RawFrame) -> Unit, - private val frameFactory: IRawFrameFactory, + private val bufferPool: ByteBufferPool, private val processDispatcher: CoroutineDispatcher, ) { private val coroutineScope = CoroutineScope(SupervisorJob() + processDispatcher) private val mutex = Mutex() - private var getFrame: (suspend (frameFactory: IRawFrameFactory) -> RawFrame)? = null + private val pool = RawFramePool() + + private var source: IAudioFrameSourceInternal? = null private val isReleaseRequested = AtomicBoolean(false) private var job: Job? = null - suspend fun setInput(getFrame: suspend (frameFactory: IRawFrameFactory) -> RawFrame) { + suspend fun setInput(source: IAudioFrameSourceInternal) { mutex.withLock { - this.getFrame = getFrame + this.source = source } } suspend fun removeInput() { mutex.withLock { - this.getFrame = null + this.source = null } } + private suspend fun CoroutineScope.runProcessing() { + while (isActive) { + val rawFrame = mutex.withLock { + val unwrapSource = source ?: return@withLock null + try { + val buffer = bufferPool.get(unwrapSource.minBufferSize) + val timestampInUs = unwrapSource.fillAudioFrame(buffer) + pool.get(buffer, timestampInUs) + } catch (t: Throwable) { + Logger.e(TAG, "Failed to get frame: ${t.message}") + null + } + } + if (rawFrame == null) { + continue + } + + // Process buffer with effects + val processedFrame = try { + frameProcessor.process(rawFrame) + } catch (t: Throwable) { + Logger.e(TAG, "Failed to pre-process frame: ${t.message}") + continue + } + + // Store for outputs + onFrame(processedFrame) + } + Logger.d(TAG, "Processing loop ended") + } + fun startStream() { if (isReleaseRequested.get()) { Logger.w(TAG, "Already released") return } + if (job != null) { + Logger.w(TAG, "Already running") + return + } job = coroutineScope.launch { - while (isActive) { - val rawFrame = mutex.withLock { - val listener = getFrame ?: return@withLock null - try { - listener(frameFactory) - } catch (t: Throwable) { - Logger.e(TAG, "Failed to get frame: ${t.message}") - null - } - } - if (rawFrame == null) { - continue - } - - // Process buffer with effects - val processedFrame = try { - frameProcessor.processFrame(rawFrame) - } catch (t: Throwable) { - Logger.e(TAG, "Failed to pre-process frame: ${t.message}") - continue - } - - // Store for outputs - onFrame(processedFrame) - } - Logger.e(TAG, "Processing loop ended") + runProcessing() } } @@ -115,7 +121,8 @@ class RawFramePullPush( job?.cancel() job = null - frameFactory.clear() + pool.clear() + bufferPool.clear() } fun release() { @@ -127,7 +134,8 @@ class RawFramePullPush( } coroutineScope.cancel() - frameFactory.close() + pool.close() + bufferPool.close() } companion object { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt index 83af9a5a7..bde293f80 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioEffects.kt @@ -16,30 +16,47 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.processing.IFrameProcessor +import io.github.thibaultbee.streampack.core.elements.processing.IEffectConsumer +import io.github.thibaultbee.streampack.core.elements.processing.IEffectProcessor import java.io.Closeable /** * The base audio effect. */ -interface IAudioEffect : IFrameProcessor, Closeable +sealed interface IAudioEffect : Closeable /** - * Mute audio effect. + * An audio effect that can be dispatched to another thread. The result is not use by the audio pipeline. + * Example: a VU meter. */ -class MuteEffect : IAudioEffect { +interface IConsumerAudioEffect : IAudioEffect, IEffectConsumer + +/** + * An audio effect that can't be dispatched to another thread. The result is used by the audio pipeline. + * + * The [RawFrame.rawBuffer] can't be modified. + */ +interface IProcessorAudioEffect : IAudioEffect, IEffectProcessor + +/** + * An audio effect that mute the audio. + */ +class MuteEffect : IProcessorAudioEffect { private var mutedByteArray: ByteArray? = null - override fun processFrame(frame: RawFrame): RawFrame { - val remaining = frame.rawBuffer.remaining() - val position = frame.rawBuffer.position() + override fun process(isMuted: Boolean, data: RawFrame): RawFrame { + if (!isMuted) { + return data + } + val remaining = data.rawBuffer.remaining() + val position = data.rawBuffer.position() if (remaining != mutedByteArray?.size) { mutedByteArray = ByteArray(remaining) } - frame.rawBuffer.put(mutedByteArray!!) - frame.rawBuffer.position(position) + data.rawBuffer.put(mutedByteArray!!) + data.rawBuffer.position(position) - return frame + return data } override fun close() { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt index 4cd6e7b7a..efa6f9494 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/AudioFrameProcessor.kt @@ -16,22 +16,76 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.processing.IFrameProcessor +import io.github.thibaultbee.streampack.core.elements.data.deepCopy +import io.github.thibaultbee.streampack.core.elements.processing.IProcessor +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import java.io.Closeable +import java.nio.ByteBuffer +import java.util.concurrent.CopyOnWriteArrayList +import java.util.function.IntFunction /** * Audio frame processor. - * - * Only supports mute effect for now. */ -class AudioFrameProcessor : IFrameProcessor, - IAudioFrameProcessor { +class AudioFrameProcessor( + private val bufferPool: IBufferPool, + dispatcher: CoroutineDispatcher, + private val effects: CopyOnWriteArrayList = CopyOnWriteArrayList() +) : IProcessor, IAudioFrameProcessor, Closeable, MutableList by effects { + private val coroutineScope = CoroutineScope(dispatcher + SupervisorJob()) + + /** + * Whether the audio is muted. + * + * When the audio is muted, the audio effect are not processed. Only consumer effects are processed. + */ override var isMuted = false private val muteEffect = MuteEffect() - override fun processFrame(frame: RawFrame): RawFrame { - if (isMuted) { - return muteEffect.processFrame(frame) + private fun launchConsumerEffect( + effect: IConsumerAudioEffect, + isMuted: Boolean, + data: RawFrame + ) { + val consumeFrame = data.deepCopy(bufferPool) + coroutineScope.launch { + effect.consume(isMuted, consumeFrame) + } + } + + override fun process(data: RawFrame): RawFrame { + val isMuted = isMuted + + var processedFrame = muteEffect.process(isMuted, data) + + effects.forEach { + if (it is IProcessorAudioEffect) { + processedFrame = it.process(isMuted, processedFrame) + } else if (it is IConsumerAudioEffect) { + launchConsumerEffect(it, isMuted, processedFrame) + } } - return frame + + return processedFrame + } + + override fun close() { + effects.forEach { it.close() } + effects.clear() + + muteEffect.close() + + coroutineScope.cancel() + } + + @Deprecated("'fun toArray(generator: IntFunction!>!): Array<(out) T!>!' is deprecated. This declaration is redundant in Kotlin and might be removed soon.") + @Suppress("DEPRECATION") + override fun toArray(generator: IntFunction?>): Array { + return super.toArray(generator) } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt index 24c7606e1..c21155385 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/audio/IAudioFrameProcessor.kt @@ -18,9 +18,9 @@ package io.github.thibaultbee.streampack.core.elements.processing.audio /** * Public interface for audio frame processor. */ -interface IAudioFrameProcessor { +interface IAudioFrameProcessor : MutableList { /** - * Mute audio. + * Whether the processor is muted. */ var isMuted: Boolean } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt index 5766ad546..70c736443 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt @@ -23,6 +23,7 @@ import android.view.Surface import androidx.annotation.IntRange import androidx.concurrent.futures.CallbackToFutureAdapter import com.google.common.util.concurrent.ListenableFuture +import io.github.thibaultbee.streampack.core.elements.interfaces.ISnapshotable import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils @@ -38,12 +39,14 @@ import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Compan import io.github.thibaultbee.streampack.core.pipelines.IVideoDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.utils.HandlerThreadExecutor import java.util.concurrent.atomic.AtomicBoolean +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine private class DefaultSurfaceProcessor( private val dynamicRangeProfile: DynamicRangeProfile, private val glThread: HandlerThreadExecutor, -) : ISurfaceProcessorInternal, SurfaceTexture.OnFrameAvailableListener { +) : ISurfaceProcessorInternal, SurfaceTexture.OnFrameAvailableListener, ISnapshotable { private val renderer = OpenGlRenderer() private val glHandler = glThread.handler @@ -82,6 +85,10 @@ private class DefaultSurfaceProcessor( } val future = submitSafely { + if (isReleaseRequested.get()) { + throw IllegalStateException("SurfaceProcessor is released") + } + val surfaceTexture = SurfaceTexture(renderer.textureName) surfaceTexture.setDefaultBufferSize(surfaceSize.width, surfaceSize.height) surfaceTexture.setOnFrameAvailableListener(this, glHandler) @@ -102,6 +109,10 @@ private class DefaultSurfaceProcessor( } override fun removeInputSurface(surface: Surface) { + if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") + return + } executeSafely { val surfaceInput = surfaceInputs.find { it.surface == surface } if (surfaceInput != null) { @@ -137,10 +148,13 @@ private class DefaultSurfaceProcessor( override fun addOutputSurface(surfaceOutput: ISurfaceOutput) { if (isReleaseRequested.get()) { - return + throw IllegalStateException("SurfaceProcessor is released") } executeSafely { + if (isReleaseRequested.get()) { + throw IllegalStateException("SurfaceProcessor is released") + } if (!surfaceOutputs.map { it.targetSurface }.contains(surfaceOutput.targetSurface)) { renderer.registerOutputSurface(surfaceOutput.targetSurface) surfaceOutputs.add(surfaceOutput) @@ -161,20 +175,30 @@ private class DefaultSurfaceProcessor( override fun removeOutputSurface(surfaceOutput: ISurfaceOutput) { if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") return } executeSafely { + if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") + return@executeSafely + } removeOutputSurfaceInternal(surfaceOutput) } } override fun removeOutputSurface(surface: Surface) { if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") return } executeSafely { + if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") + return@executeSafely + } val surfaceOutput = surfaceOutputs.firstOrNull { it.targetSurface == surface } if (surfaceOutput != null) { @@ -194,10 +218,15 @@ private class DefaultSurfaceProcessor( override fun removeAllOutputSurfaces() { if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") return } executeSafely { + if (isReleaseRequested.get()) { + Logger.w(TAG, "SurfaceProcessor is released") + return@executeSafely + } removeAllOutputSurfacesInternal() } } @@ -225,7 +254,27 @@ private class DefaultSurfaceProcessor( } } - override fun snapshot( + /** + * Takes a snapshot of the current video frame. + * + * The snapshot is returned as a [Bitmap]. + * + * @param rotationDegrees The rotation to apply to the snapshot, in degrees. 0 means no rotation. + * @return The snapshot as a [Bitmap]. + */ + override suspend fun takeSnapshot(rotationDegrees: Int): Bitmap { + return suspendCoroutine { continuation -> + val listener = snapshot(rotationDegrees) + try { + val bitmap = listener.get() + continuation.resume(bitmap) + } catch (e: Exception) { + continuation.resumeWith(Result.failure(e)) + } + } + } + + private fun snapshot( @IntRange(from = 0, to = 359) rotationDegrees: Int ): ListenableFuture { if (isReleaseRequested.get()) { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/ISurfaceProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/ISurfaceProcessor.kt index f017e1fa6..35d1c8fec 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/ISurfaceProcessor.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/ISurfaceProcessor.kt @@ -51,8 +51,6 @@ interface ISurfaceProcessorInternal : ISurfaceProcessor, Releasable { fun removeAllOutputSurfaces() - fun snapshot(@IntRange(from = 0, to = 359) rotationDegrees: Int): ListenableFuture - /** * Factory interface for creating instances of [ISurfaceProcessorInternal]. */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt index 18c89bb6c..209ae9ea8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/IAudioFrameSourceInternal.kt @@ -15,26 +15,20 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.audio -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import java.nio.ByteBuffer interface IAudioFrameSourceInternal { /** - * Gets an audio frame from the source. - * - * @param frame the [RawFrame] to fill with audio data. - * @return a [RawFrame] containing audio data. + * Gets the size of the buffer to allocate. + * When using encoder is callback mode, it's unused. */ - fun fillAudioFrame(frame: RawFrame): RawFrame + val minBufferSize: Int /** * Gets an audio frame from the source. * - * The [RawFrame] to fill with audio data is created by the [frameFactory]. - * - * @param frameFactory a [IRawFrameFactory] to create [RawFrame]. - * @return a [RawFrame] containing audio data. + * @param buffer the [ByteBuffer] to fill with audio data. + * @return the timestamp in microseconds. */ - fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame + fun fillAudioFrame(buffer: ByteBuffer): Long } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt index 324438aaf..cc1746614 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/audio/audiorecord/AudioRecordSource.kt @@ -22,18 +22,18 @@ import android.media.AudioTimestamp import android.media.audiofx.AudioEffect import android.os.Build import androidx.annotation.RequiresPermission -import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordEffect.Companion.isValidUUID import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordEffect.Factory.Companion.getFactoryForEffectType +import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordSource.Companion.availableEffect import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.AudioRecordSource.Companion.isEffectAvailable -import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.elements.utils.extensions.type -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import io.github.thibaultbee.streampack.core.elements.utils.time.TimeUtils import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow +import java.nio.ByteBuffer import java.util.UUID /** @@ -42,7 +42,9 @@ import java.util.UUID */ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSource { private var audioRecord: AudioRecord? = null - private var bufferSize: Int? = null + private var _minBufferSize: Int? = null + override val minBufferSize: Int + get() = _minBufferSize ?: throw IllegalStateException("Audio source is not initialized") private var processor: EffectProcessor? = null private var pendingAudioEffects = mutableListOf() @@ -74,9 +76,9 @@ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSour } } - bufferSize = getMinBufferSize(config) + _minBufferSize = getMinBufferSize(config) - audioRecord = buildAudioRecord(config, bufferSize!!).also { + audioRecord = buildAudioRecord(config, _minBufferSize!!).also { val previousEffects = processor?.getAll() ?: emptyList() processor?.clear() @@ -157,30 +159,19 @@ internal sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSour } - override fun fillAudioFrame(frame: RawFrame): RawFrame { + override fun fillAudioFrame( + buffer: ByteBuffer + ): Long { val audioRecord = requireNotNull(audioRecord) { "Audio source is not initialized" } if (audioRecord.recordingState != AudioRecord.RECORDSTATE_RECORDING) { throw IllegalStateException("Audio source is not recording") } - val buffer = frame.rawBuffer val length = audioRecord.read(buffer, buffer.remaining()) - if (length > 0) { - frame.timestampInUs = getTimestampInUs(audioRecord) - return frame - } else { - frame.close() + if (length <= 0) { throw IllegalArgumentException(audioRecordErrorToString(length)) } - } - - override fun getAudioFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame { - val bufferSize = requireNotNull(bufferSize) { "Buffer size is not initialized" } - - /** - * Dummy timestamp: it is overwritten later. - */ - return fillAudioFrame(frameFactory.create(bufferSize, 0)) + return getTimestampInUs(audioRecord) } /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt index 3c206900d..a90d61103 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/IVideoFrameSource.kt @@ -15,17 +15,15 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.video -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory +import java.nio.ByteBuffer interface IVideoFrameSourceInternal { /** * Gets a video frame from a source. * - * @param frameFactory a [IRawFrameFactory] to create [RawFrame]. - * @return a [RawFrame] containing video data. + * @param buffer The buffer to fill with the video frame. + * @return The timestamp in microseconds of the video frame. */ - fun getVideoFrame(frameFactory: IReadOnlyRawFrameFactory): RawFrame + fun getVideoFrame(buffer: ByteBuffer): Long } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt index bc87cc182..c6c85a4c1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSettings.kt @@ -15,13 +15,19 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.video.camera +import android.Manifest import android.content.Context import android.graphics.PointF import android.graphics.Rect import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraMetadata +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_ACTIVE_SCAN +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_FOCUSED_LOCKED +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_INACTIVE +import android.hardware.camera2.CameraMetadata.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED import android.hardware.camera2.CaptureRequest import android.hardware.camera2.CaptureResult +import android.hardware.camera2.TotalCaptureResult import android.hardware.camera2.params.ColorSpaceTransform import android.hardware.camera2.params.MeteringRectangle import android.hardware.camera2.params.RggbChannelVector @@ -30,6 +36,7 @@ import android.util.Range import android.util.Rational import androidx.annotation.IntRange import androidx.annotation.RequiresApi +import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.is90or270 import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraController import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.autoExposureModes @@ -48,17 +55,30 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.scalerMaxZoom import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.sensitivityRange import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.zoomRatioRange -import io.github.thibaultbee.streampack.core.elements.utils.extensions.clamp +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureResultListener +import io.github.thibaultbee.streampack.core.elements.utils.extensions.coerceIn import io.github.thibaultbee.streampack.core.elements.utils.extensions.isApplicationPortrait import io.github.thibaultbee.streampack.core.elements.utils.extensions.isNormalized import io.github.thibaultbee.streampack.core.elements.utils.extensions.launchIn import io.github.thibaultbee.streampack.core.elements.utils.extensions.normalize import io.github.thibaultbee.streampack.core.elements.utils.extensions.rotate import io.github.thibaultbee.streampack.core.logger.Logger +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.callbackFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.shareIn +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import java.util.concurrent.CancellationException +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicLong /** @@ -68,7 +88,7 @@ import kotlinx.coroutines.sync.withLock * @param characteristics Camera characteristics of the current camera. */ class CameraSettings internal constructor( - val coroutineScope: CoroutineScope, + private val coroutineScope: CoroutineScope, val characteristics: CameraCharacteristics, private val cameraController: CameraController ) { @@ -77,6 +97,41 @@ class CameraSettings internal constructor( */ val cameraId = cameraController.cameraId + /** + * Current physical camera id. + */ + @delegate:RequiresApi(Build.VERSION_CODES.Q) + val physicalCameraIdFlow: Flow by lazy { + captureResultSharedFlow.map { it[CaptureResult.LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_ID]!! } + .distinctUntilChanged() + } + + private val captureResultSharedFlow by lazy { + callbackFlow { + val captureCallback = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + trySend(result) + return false + } + } + cameraController.addCaptureCallbackListener(captureCallback) + awaitClose { + coroutineScope.launch { + try { + cameraController.removeCaptureCallbackListener(captureCallback) + } catch (e: Exception) { + Logger.w(TAG, "Failed to remove capture callback listener on close", e) + } + } + } + }.shareIn(coroutineScope, SharingStarted.WhileSubscribed(), 1) + } + + /** + * The total capture result flow. + */ + val captureResultFlow: Flow by lazy { captureResultSharedFlow } + /** * Whether the camera is available. * To be used before calling any camera settings. @@ -135,6 +190,7 @@ class CameraSettings internal constructor( * @param key the key to get * @return the value associated with the key */ + @RequiresPermission(Manifest.permission.CAMERA) fun get(key: CaptureRequest.Key) = cameraController.getSetting(key) /** @@ -145,22 +201,107 @@ class CameraSettings internal constructor( * @param key the key to set * @param value the value to set */ - fun set(key: CaptureRequest.Key, value: T) = + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun set(key: CaptureRequest.Key, value: T) = cameraController.setSetting(key, value) /** * Applies settings to the camera repeatedly in a synchronized way. * * This method returns when the capture callback is received with the passed request. + * + * @return the total capture result */ - suspend fun applyRepeatingSessionSync() = cameraController.setRepeatingSessionSync() + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun applyRepeatingSessionSync(): TotalCaptureResult { + val deferred = CompletableDeferred() + + val captureResult = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + deferred.complete(result) + return false + } + } + applyRepeatingSession(captureResult) + return deferred.await() + } + + /** + * Applies settings to the camera repeatedly. + * + * @param onCaptureResult the capture result callback. Return `true` to stop the callback. + */ + @RequiresPermission(Manifest.permission.CAMERA) + private suspend fun applyRepeatingSession(onCaptureResult: CaptureResultListener) { + val tag = TagBundle.Factory.default.create() + val captureCallback = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + val resultTag = result.request.tag as? TagBundle + val keyId = resultTag?.keyId ?: return false + if (keyId >= tag.keyId) { + return onCaptureResult.onCaptureResult(result) + } + return false + } + } + + cameraController.addCaptureCallbackListener(captureCallback) + cameraController.applyRepeatingSession(tag) + } /** * Applies settings to the camera repeatedly. */ - suspend fun applyRepeatingSession() = cameraController.setRepeatingSession() + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun applyRepeatingSession() = cameraController.applyRepeatingSession() - class Flash( + /** + * Resets all settings to default. + */ + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun resetAll() { + cameraController.resetSettings() + applyRepeatingSession() + } + + private class TagBundle(val keyId: Long) { + private val tagMap = mutableMapOf().apply { + put(TAG_KEY_ID, keyId) + } + + val keys: Set + get() = tagMap.keys + + fun setTag(key: String, value: Any?) { + tagMap[key] = value + } + + companion object { + private const val TAG_KEY_ID = "TAG_KEY_ID" + } + + /** + * Factory for [TagBundle]. + * + * The purpose is to make sure the tag always contains an increasing id. + */ + class Factory private constructor() { + /** + * Next session id. + */ + private val nextSessionUpdateId = AtomicLong(0) + + fun create(): TagBundle { + return TagBundle(nextSessionUpdateId.getAndIncrement()) + } + + companion object { + val default = Factory() + } + } + } + + class Flash internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -169,8 +310,7 @@ class CameraSettings internal constructor( * * @return `true` if camera has a flash device, [Boolean.Companion.toString] otherwise. */ - val isAvailable: Boolean - get() = characteristics.isFlashAvailable + val isAvailable: Boolean by lazy { characteristics.isFlashAvailable } /** * Enables or disables flash. @@ -181,8 +321,10 @@ class CameraSettings internal constructor( /** * @return `true` if flash is already on, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) get() = getFlash() == CaptureResult.FLASH_MODE_TORCH + @RequiresPermission(Manifest.permission.CAMERA) private fun getFlash(): Int = cameraSettings.get(CaptureRequest.FLASH_MODE) ?: CaptureResult.FLASH_MODE_OFF @@ -191,6 +333,7 @@ class CameraSettings internal constructor( * * @param isEnable `true` to enable flash, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setIsEnable(isEnable: Boolean) { if (isEnable) { setFlash(CaptureRequest.FLASH_MODE_TORCH) @@ -206,13 +349,69 @@ class CameraSettings internal constructor( * * @param mode flash mode */ + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun setFlash(mode: Int) { cameraSettings.set(CaptureRequest.FLASH_MODE, mode) cameraSettings.applyRepeatingSession() } + + /** + * Gets the range of supported flash strength. + * Range is from [1-x]. + * + * Use the range to call [setStrengthLevel] + */ + @delegate:RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + val availableStrengthLevelRange: Range by lazy { + Range( + 1, + characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_MAX_LEVEL] + ?: DEFAULT_STRENGTH_LEVEL + ) + } + + /** + * Gets the default flash strength. + * Range is from [availableStrengthLevelRange]. + * + * Use the range to call [setStrengthLevel] + */ + @delegate:RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + val defaultStrengthLevel: Int by lazy { + characteristics[CameraCharacteristics.FLASH_TORCH_STRENGTH_DEFAULT_LEVEL] + ?: DEFAULT_STRENGTH_LEVEL + } + + val strengthLevel: Int + /** + * Gets the flash strength. + * + * @return the flash strength + */ + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + @RequiresPermission(Manifest.permission.CAMERA) + get() = cameraSettings.get(CaptureRequest.FLASH_STRENGTH_LEVEL) + ?: DEFAULT_STRENGTH_LEVEL + + /** + * Sets the flash strength. + * + * @param level flash strength. Range is from [1-x]. + * @see [availableStrengthLevelRange] + */ + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun setStrengthLevel(level: Int) { + cameraSettings.set(CaptureRequest.FLASH_STRENGTH_LEVEL, level) + cameraSettings.applyRepeatingSession() + } + + companion object { + private const val DEFAULT_STRENGTH_LEVEL = 1 + } } - class WhiteBalance( + class WhiteBalance internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -221,8 +420,7 @@ class CameraSettings internal constructor( * * @return list of supported white balance modes. */ - val availableAutoModes: List - get() = characteristics.autoWhiteBalanceModes + val availableAutoModes: List by lazy { characteristics.autoWhiteBalanceModes } /** * Gets the auto white balance mode. @@ -236,6 +434,7 @@ class CameraSettings internal constructor( * * @return current camera auto white balance mode */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AWB_MODE) ?: CaptureResult.CONTROL_AWB_MODE_OFF @@ -245,6 +444,7 @@ class CameraSettings internal constructor( * @param autoMode auto white balance mode * @see [availableAutoModes] */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setAutoMode(autoMode: Int) { cameraSettings.set(CaptureRequest.CONTROL_AWB_MODE, autoMode) cameraSettings.applyRepeatingSession() @@ -253,19 +453,20 @@ class CameraSettings internal constructor( /** * Get maximum number of available white balance metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfWhiteBalanceMeteringRegions + val maxNumOfMeteringRegions: Int by lazy { characteristics.maxNumberOfWhiteBalanceMeteringRegions } /** * Gets the white balance metering regions. */ val meteringRegions: List + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AWB_REGIONS)?.toList() ?: emptyList() /** * Sets the white balance metering regions. */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setMeteringRegions(value: List) { cameraSettings.set( CaptureRequest.CONTROL_AWB_REGIONS, value.toTypedArray() @@ -277,6 +478,7 @@ class CameraSettings internal constructor( * Gets the auto white balance lock state. */ val isLocked: Boolean + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AWB_LOCK) ?: false /** @@ -284,13 +486,14 @@ class CameraSettings internal constructor( * * @param isLocked the lock state. `true` to lock auto white balance, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setIsLocked(isLocked: Boolean) { cameraSettings.set(CaptureRequest.CONTROL_AWB_LOCK, isLocked) cameraSettings.applyRepeatingSession() } } - class Iso( + class Iso internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -301,8 +504,9 @@ class CameraSettings internal constructor( * * @see [sensorSensitivity] */ - val availableSensorSensitivityRange: Range - get() = characteristics.sensitivityRange ?: DEFAULT_SENSITIVITY_RANGE + val availableSensorSensitivityRange: Range by lazy { + characteristics.sensitivityRange ?: DEFAULT_SENSITIVITY_RANGE + } /** * Gets lens focus distance. @@ -315,6 +519,7 @@ class CameraSettings internal constructor( * * @return the sensitivity */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.SENSOR_SENSITIVITY) ?: DEFAULT_SENSITIVITY @@ -324,10 +529,11 @@ class CameraSettings internal constructor( * * @param sensorSensitivity the sensitivity */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setSensorSensitivity(sensorSensitivity: Int) { cameraSettings.set( CaptureRequest.SENSOR_SENSITIVITY, - sensorSensitivity.clamp(availableSensorSensitivityRange) + sensorSensitivity.coerceIn(availableSensorSensitivityRange) ) cameraSettings.applyRepeatingSession() } @@ -338,7 +544,7 @@ class CameraSettings internal constructor( } } - class ColorCorrection( + class ColorCorrection internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -347,11 +553,10 @@ class CameraSettings internal constructor( * * @return `true` if camera has a flash device, `false` otherwise. */ - val isAvailable: Boolean - get() { - return characteristics[CameraCharacteristics.COLOR_CORRECTION_AVAILABLE_ABERRATION_MODES] - ?.contains(CaptureRequest.COLOR_CORRECTION_MODE_TRANSFORM_MATRIX) == true - } + val isAvailable: Boolean by lazy { + characteristics[CameraCharacteristics.COLOR_CORRECTION_AVAILABLE_ABERRATION_MODES] + ?.contains(CaptureRequest.COLOR_CORRECTION_MODE_TRANSFORM_MATRIX) == true + } /** * Gets color correction gain. @@ -361,6 +566,7 @@ class CameraSettings internal constructor( * @see [WhiteBalance.autoMode] */ val rggbChannelVector: RggbChannelVector? + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.COLOR_CORRECTION_GAINS) /** @@ -370,6 +576,7 @@ class CameraSettings internal constructor( * * @param rggbChannelVector the color correction gain */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setRggbChannelVector(rggbChannelVector: RggbChannelVector) { cameraSettings.set( CaptureRequest.CONTROL_AWB_MODE, @@ -393,7 +600,7 @@ class CameraSettings internal constructor( } } - class Exposure( + class Exposure internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -404,9 +611,7 @@ class CameraSettings internal constructor( * * @see [autoMode] */ - val availableAutoModes: List - get() = characteristics.autoExposureModes - + val availableAutoModes: List by lazy { characteristics.autoExposureModes } /** * Gets auto exposure mode. @@ -419,6 +624,7 @@ class CameraSettings internal constructor( * * @return the auto exposure mode */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AE_MODE) ?: CaptureResult.CONTROL_AE_MODE_OFF @@ -427,6 +633,7 @@ class CameraSettings internal constructor( * * @param autoMode the exposure auto mode */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setAutoMode(autoMode: Int) { cameraSettings.set(CaptureRequest.CONTROL_AE_MODE, autoMode) cameraSettings.applyRepeatingSession() @@ -440,9 +647,10 @@ class CameraSettings internal constructor( * @see [availableCompensationStep] * @see [compensation] */ - val availableCompensationRange: Range - get() = characteristics.exposureRange + val availableCompensationRange: Range by lazy { + characteristics.exposureRange ?: DEFAULT_COMPENSATION_RANGE + } /** * Gets current camera exposure compensation step. @@ -456,9 +664,9 @@ class CameraSettings internal constructor( * @see [availableCompensationRange] * @see [compensation] */ - val availableCompensationStep: Rational - get() = characteristics.exposureStep - ?: DEFAULT_COMPENSATION_STEP_RATIONAL + val availableCompensationStep: Rational by lazy { + characteristics.exposureStep ?: DEFAULT_COMPENSATION_STEP_RATIONAL + } /** * Gets exposure compensation. @@ -472,6 +680,7 @@ class CameraSettings internal constructor( * * @return the exposure compensation */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AE_EXPOSURE_COMPENSATION) ?: DEFAULT_COMPENSATION @@ -480,10 +689,11 @@ class CameraSettings internal constructor( * * @param compensation the exposure compensation */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setCompensation(compensation: Int) { cameraSettings.set( CaptureRequest.CONTROL_AE_EXPOSURE_COMPENSATION, - compensation.clamp(availableCompensationRange) + compensation.coerceIn(availableCompensationRange) ) cameraSettings.applyRepeatingSession() } @@ -491,13 +701,13 @@ class CameraSettings internal constructor( /** * Get maximum number of available exposure metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfExposureMeteringRegions + val maxNumOfMeteringRegions by lazy { characteristics.maxNumberOfExposureMeteringRegions } /** * Gets the exposure metering regions. */ val meteringRegions: List + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AE_REGIONS)?.toList() ?: emptyList() @@ -506,6 +716,7 @@ class CameraSettings internal constructor( * * @param meteringRegions the metering regions */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setMeteringRegions(meteringRegions: List) { cameraSettings.set( CaptureRequest.CONTROL_AE_REGIONS, meteringRegions.toTypedArray() @@ -513,12 +724,17 @@ class CameraSettings internal constructor( cameraSettings.applyRepeatingSession() } + + @delegate:RequiresApi(Build.VERSION_CODES.M) + val isLockAvailable: Boolean by lazy { characteristics[CameraCharacteristics.CONTROL_AE_LOCK_AVAILABLE] == true } + /** * Gets auto exposure lock. * * @return the auto exposure lock state */ val isLocked: Boolean + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AE_LOCK) ?: false /** @@ -526,6 +742,7 @@ class CameraSettings internal constructor( * * @param isLocked the lock state. `true` to lock auto exposure, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setIsLocked(isLocked: Boolean) { cameraSettings.set( CaptureRequest.CONTROL_AE_LOCK, isLocked @@ -544,6 +761,8 @@ class CameraSettings internal constructor( protected val characteristics: CameraCharacteristics, protected val cameraSettings: CameraSettings ) { + private val listeners: CopyOnWriteArrayList = CopyOnWriteArrayList() + abstract val availableRatioRange: Range internal abstract suspend fun getCropSensorRegion(): Rect @@ -558,7 +777,7 @@ class CameraSettings internal constructor( suspend fun onPinch(scale: Float) { val scaledRatio: Float = getZoomRatio() * speedUpZoomByX(scale, 2) // Clamp the ratio with the zoom range. - setZoomRatio(scaledRatio.clamp(availableRatioRange.lower, availableRatioRange.upper)) + setZoomRatio(scaledRatio.coerceIn(availableRatioRange.lower, availableRatioRange.upper)) } private fun speedUpZoomByX(scaleFactor: Float, ratio: Int): Float { @@ -569,7 +788,21 @@ class CameraSettings internal constructor( } } - class CropScalerRegionZoom( + fun addListener(listener: OnZoomChangedListener) { + listeners.add(listener) + } + + fun removeListener(listener: OnZoomChangedListener) { + listeners.remove(listener) + } + + protected fun notifyZoomListeners(zoomRatio: Float) { + listeners.forEach { + it.onZoomChanged(zoomRatio) + } + } + + class CropScalerRegionZoom internal constructor( characteristics: CameraCharacteristics, cameraSettings: CameraSettings ) : Zoom(characteristics, cameraSettings) { @@ -579,18 +812,25 @@ class CameraSettings internal constructor( private var persistentZoomRatio = 1f private var currentCropRect: Rect? = null - override val availableRatioRange: Range - get() = Range( + override val availableRatioRange: Range by lazy { + Range( DEFAULT_ZOOM_RATIO, characteristics.scalerMaxZoom ) + } override suspend fun getZoomRatio(): Float = mutex.withLock { persistentZoomRatio } + @RequiresPermission(Manifest.permission.CAMERA) override suspend fun setZoomRatio(zoomRatio: Float) { mutex.withLock { - val clampedValue = zoomRatio.clamp(availableRatioRange) + val clampedValue = zoomRatio.coerceIn(availableRatioRange) + if (clampedValue == persistentZoomRatio) { + return@withLock + } + persistentZoomRatio = clampedValue + currentCropRect = getCropRegion( characteristics, clampedValue @@ -598,8 +838,8 @@ class CameraSettings internal constructor( cameraSettings.set( CaptureRequest.SCALER_CROP_REGION, currentCropRect ) - cameraSettings.applyRepeatingSession() - persistentZoomRatio = clampedValue + cameraSettings.applyRepeatingSessionSync() + notifyZoomListeners(clampedValue) } } @@ -644,22 +884,31 @@ class CameraSettings internal constructor( } @RequiresApi(Build.VERSION_CODES.R) - class RZoom(characteristics: CameraCharacteristics, cameraSettings: CameraSettings) : + class RZoom internal constructor( + characteristics: CameraCharacteristics, + cameraSettings: CameraSettings + ) : Zoom(characteristics, cameraSettings) { - override val availableRatioRange: Range - get() = characteristics.zoomRatioRange - ?: DEFAULT_ZOOM_RATIO_RANGE + override val availableRatioRange: Range by lazy { + characteristics.zoomRatioRange ?: DEFAULT_ZOOM_RATIO_RANGE + } + @RequiresPermission(Manifest.permission.CAMERA) override suspend fun getZoomRatio(): Float { return cameraSettings.get(CaptureRequest.CONTROL_ZOOM_RATIO) ?: DEFAULT_ZOOM_RATIO } + @RequiresPermission(Manifest.permission.CAMERA) override suspend fun setZoomRatio(zoomRatio: Float) { + if (zoomRatio == getZoomRatio()) { + return + } cameraSettings.set( - CaptureRequest.CONTROL_ZOOM_RATIO, zoomRatio.clamp(availableRatioRange) + CaptureRequest.CONTROL_ZOOM_RATIO, zoomRatio.coerceIn(availableRatioRange) ) cameraSettings.applyRepeatingSession() + notifyZoomListeners(zoomRatio) } override suspend fun getCropSensorRegion(): Rect { @@ -671,7 +920,7 @@ class CameraSettings internal constructor( const val DEFAULT_ZOOM_RATIO = 1f val DEFAULT_ZOOM_RATIO_RANGE = Range(DEFAULT_ZOOM_RATIO, DEFAULT_ZOOM_RATIO) - fun build( + internal fun build( characteristics: CameraCharacteristics, cameraSettings: CameraSettings ): Zoom { @@ -682,10 +931,22 @@ class CameraSettings internal constructor( } } } + + /** + * Listener for zoom change. + */ + interface OnZoomChangedListener { + /** + * Called when the zoom ratio changes. + * + * @param zoomRatio the zoom ratio + */ + fun onZoomChanged(zoomRatio: Float) + } } - class Focus( + class Focus internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -696,8 +957,7 @@ class CameraSettings internal constructor( * * @see [autoMode] */ - val availableAutoModes: List - get() = characteristics.autoFocusModes + val availableAutoModes: List by lazy { characteristics.autoFocusModes } /** * Gets the auto focus mode. @@ -710,6 +970,7 @@ class CameraSettings internal constructor( * * @return the auto focus mode */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AF_MODE) ?: CaptureResult.CONTROL_AF_MODE_OFF @@ -718,6 +979,7 @@ class CameraSettings internal constructor( * * @param autoMode the auto focus mode */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setAutoMode(autoMode: Int) { cameraSettings.set(CaptureRequest.CONTROL_AF_MODE, autoMode) cameraSettings.applyRepeatingSession() @@ -730,8 +992,9 @@ class CameraSettings internal constructor( * * @see [lensDistance] */ - val availableLensDistanceRange: Range - get() = characteristics.lensDistanceRange + val availableLensDistanceRange: Range by lazy { + characteristics.lensDistanceRange + } /** * Gets the lens focus distance. @@ -744,6 +1007,7 @@ class CameraSettings internal constructor( * * @return the lens focus distance */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.LENS_FOCUS_DISTANCE) ?: DEFAULT_LENS_DISTANCE @@ -754,9 +1018,10 @@ class CameraSettings internal constructor( * * @param lensDistance the lens focus distance */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setLensDistance(lensDistance: Float) { cameraSettings.set( - CaptureRequest.LENS_FOCUS_DISTANCE, lensDistance.clamp(availableLensDistanceRange) + CaptureRequest.LENS_FOCUS_DISTANCE, lensDistance.coerceIn(availableLensDistanceRange) ) cameraSettings.applyRepeatingSession() } @@ -764,14 +1029,15 @@ class CameraSettings internal constructor( /** * Get maximum number of available focus metering regions. */ - val maxNumOfMeteringRegions: Int - get() = characteristics.maxNumberOfFocusMeteringRegions - ?: DEFAULT_MAX_NUM_OF_METERING_REGION + val maxNumOfMeteringRegions: Int by lazy { + characteristics.maxNumberOfFocusMeteringRegions ?: DEFAULT_MAX_NUM_OF_METERING_REGION + } /** * Gets the focus metering regions. */ val meteringRegions: List + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_AF_REGIONS)?.toList() ?: emptyList() @@ -780,6 +1046,7 @@ class CameraSettings internal constructor( * * @param meteringRegions the metering regions */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setMeteringRegions(meteringRegions: List) { cameraSettings.set( CaptureRequest.CONTROL_AF_REGIONS, meteringRegions.toTypedArray() @@ -795,7 +1062,7 @@ class CameraSettings internal constructor( } } - class Stabilization( + class Stabilization internal constructor( private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings ) { @@ -810,6 +1077,7 @@ class CameraSettings internal constructor( * * @return `true` if video stabilization is enabled, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE) == CaptureResult.CONTROL_VIDEO_STABILIZATION_MODE_ON /** @@ -817,6 +1085,7 @@ class CameraSettings internal constructor( * * @param isEnableVideo `true` to enable video stabilization, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setIsEnableVideo(isEnableVideo: Boolean) { if (isEnableVideo) { cameraSettings.set( @@ -839,8 +1108,9 @@ class CameraSettings internal constructor( * * @see [isEnableOptical] */ - val isOpticalAvailable: Boolean - get() = characteristics.isOpticalStabilizationAvailable + val isOpticalAvailable: Boolean by lazy { + characteristics.isOpticalStabilizationAvailable + } /** @@ -856,6 +1126,7 @@ class CameraSettings internal constructor( * * @return `true` if optical video stabilization is enabled, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) get() = cameraSettings.get(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE) == CaptureResult.LENS_OPTICAL_STABILIZATION_MODE_ON /** @@ -863,6 +1134,7 @@ class CameraSettings internal constructor( * * @param isEnableOptical `true` to enable optical video stabilization, otherwise `false` */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setIsEnableOptical(isEnableOptical: Boolean) { if (isEnableOptical) { cameraSettings.set( @@ -879,7 +1151,7 @@ class CameraSettings internal constructor( } } - class FocusMetering( + class FocusMetering internal constructor( private val coroutineScope: CoroutineScope, private val characteristics: CameraCharacteristics, private val cameraSettings: CameraSettings, @@ -891,6 +1163,7 @@ class CameraSettings internal constructor( private var autoCancelHandle: Job? = null @Suppress("UNCHECKED_CAST") + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun cancelAfAeTrigger() { // Cancel previous AF trigger cameraSettings.set( @@ -908,7 +1181,8 @@ class CameraSettings internal constructor( } @Suppress("UNCHECKED_CAST") - private fun addFocusMetering( + @RequiresPermission(Manifest.permission.CAMERA) + private suspend fun addFocusMetering( afRects: List, aeRects: List, awbRects: List @@ -940,6 +1214,7 @@ class CameraSettings internal constructor( } @Suppress("UNCHECKED_CAST") + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun triggerAf(overrideAeMode: Boolean) { cameraSettings.set( CaptureRequest.CONTROL_AF_TRIGGER, @@ -951,9 +1226,38 @@ class CameraSettings internal constructor( ) cameraSettings.set(CaptureRequest.CONTROL_AE_MODE, aeMode) } - cameraSettings.applyRepeatingSession() + + val deferred = CompletableDeferred() + val captureResult = object : CaptureResultListener { + override fun onCaptureResult(result: TotalCaptureResult): Boolean { + return when (val afState = result[CaptureResult.CONTROL_AF_STATE]) { + CONTROL_AF_STATE_FOCUSED_LOCKED -> { + deferred.complete(Unit) + true + } + + CONTROL_AF_STATE_NOT_FOCUSED_LOCKED -> { + deferred.completeExceptionally(Exception("AF not focused")) + true + } + + CONTROL_AF_STATE_INACTIVE -> { + deferred.completeExceptionally(CancellationException("AF has been cancelled")) + true + } + + CONTROL_AF_STATE_ACTIVE_SCAN -> false + else -> { + deferred.completeExceptionally(IllegalStateException("AF is not in an expected state $afState")) + } + } + } + } + cameraSettings.applyRepeatingSession(captureResult) + deferred.await() } + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun executeMetering( afRectangles: List, aeRectangles: List, @@ -968,7 +1272,7 @@ class CameraSettings internal constructor( */ cameraSettings.set( CaptureRequest.CONTROL_AF_TRIGGER, - CameraMetadata.CONTROL_AF_TRIGGER_IDLE + CameraMetadata.CONTROL_AF_TRIGGER_CANCEL ) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { cameraSettings.set( @@ -979,9 +1283,6 @@ class CameraSettings internal constructor( cameraSettings.applyRepeatingSessionSync() addFocusMetering(afRectangles, aeRectangles, awbRectangles) - if (afRectangles.isNotEmpty()) { - triggerAf(true) - } // Auto cancel AF trigger after timeoutDurationMs if (timeoutDurationMs > 0) { @@ -994,8 +1295,13 @@ class CameraSettings internal constructor( } } } + + if (afRectangles.isNotEmpty()) { + triggerAf(true) + } } + @RequiresPermission(Manifest.permission.CAMERA) private suspend fun startFocusAndMetering( afPoints: List, aePoints: List, @@ -1121,6 +1427,7 @@ class CameraSettings internal constructor( * @param fovRotationDegree the orientation of the field of view * @param timeoutDurationMs duration in milliseconds after which the focus and metering will be cancelled automatically */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun onTap( context: Context, point: PointF, @@ -1153,6 +1460,7 @@ class CameraSettings internal constructor( * @param fovRotationDegree the orientation of the field of view * @param timeoutDurationMs duration in milliseconds after which the focus and metering will be cancelled automatically */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun onTap( context: Context, afPoints: List, @@ -1186,6 +1494,7 @@ class CameraSettings internal constructor( /** * Cancel the focus and metering. */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun cancelFocusAndMetering() { disableAutoCancel() @@ -1309,10 +1618,10 @@ class CameraSettings internal constructor( (centerY + height / 2).toInt() ) - focusRect.left = focusRect.left.clamp(cropRegion.right, cropRegion.left) - focusRect.right = focusRect.right.clamp(cropRegion.right, cropRegion.left) - focusRect.top = focusRect.top.clamp(cropRegion.bottom, cropRegion.top) - focusRect.bottom = focusRect.bottom.clamp(cropRegion.bottom, cropRegion.top) + focusRect.left = focusRect.left.coerceIn(cropRegion.right, cropRegion.left) + focusRect.right = focusRect.right.coerceIn(cropRegion.right, cropRegion.left) + focusRect.top = focusRect.top.coerceIn(cropRegion.bottom, cropRegion.top) + focusRect.bottom = focusRect.bottom.coerceIn(cropRegion.bottom, cropRegion.top) return MeteringRectangle(focusRect, DEFAULT_METERING_WEIGHT_MAX) } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt index 38c607749..953dcd700 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/CameraSource.kt @@ -66,7 +66,7 @@ internal class CameraSource( characteristics, dispatcherProvider, cameraId, - captureRequestBuilder = { + onCaptureRequestBuilder = { defaultCaptureRequest(this) } ) @@ -119,8 +119,7 @@ internal class CameraSource( private fun defaultCaptureRequest( captureRequest: CaptureRequestWithTargetsBuilder ) { - if (settings.focus.availableAutoModes.contains(CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO) - ) { + if (settings.focus.availableAutoModes.contains(CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO)) { captureRequest.set( CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO ) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt index 13c33e653..052e62d5d 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraController.kt @@ -23,9 +23,11 @@ import android.util.Range import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.CameraCaptureSessionCompatBuilder import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraDispatcherProvider +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureResultListener import io.github.thibaultbee.streampack.core.elements.utils.av.video.DynamicRangeProfile import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.CoroutineScope @@ -46,7 +48,7 @@ internal class CameraController( private val characteristics: CameraCharacteristics, dispatcherProvider: CameraDispatcherProvider, val cameraId: String, - val captureRequestBuilder: CaptureRequestWithTargetsBuilder.() -> Unit = {} + val onCaptureRequestBuilder: CaptureRequestWithTargetsBuilder.() -> Unit = {} ) { private val sessionCompat = CameraCaptureSessionCompatBuilder.build(dispatcherProvider) @@ -57,6 +59,9 @@ internal class CameraController( private var deviceController: CameraDeviceController? = null private var sessionController: CameraSessionController? = null + private var captureRequestBuilder: CaptureRequestWithTargetsBuilder? = null + private val sessionCallback = CameraSessionCallback(coroutineScope) + private val controllerMutex = Mutex() private val outputs = mutableMapOf() @@ -151,21 +156,35 @@ internal class CameraController( } } + @RequiresPermission(Manifest.permission.CAMERA) + private suspend fun getCaptureRequestBuilder(): CaptureRequestWithTargetsBuilder { + return if (captureRequestBuilder != null) { + captureRequestBuilder!! + } else { + getDeviceController().createCaptureRequestBuilder().apply { + captureRequestBuilder = this + + val minFrameDuration = 1_000_000_000 / fpsRange.upper.toLong() + set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange) + set(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) + onCaptureRequestBuilder() + } + } + } + /** * To be used under [controllerMutex] */ @RequiresPermission(Manifest.permission.CAMERA) private suspend fun getSessionController(): CameraSessionController { return if (sessionController == null) { - val deviceController = getDeviceController() CameraSessionController.create( - coroutineScope, + getDeviceController(), + sessionCallback, sessionCompat, - deviceController, outputs.values.toList(), dynamicRange = dynamicRangeProfile.dynamicRange, - fpsRange = fpsRange, - captureRequestBuilder + defaultDispatcher ).apply { applySessionController(this) Logger.d(TAG, "Session controller created") @@ -175,12 +194,10 @@ internal class CameraController( sessionController!! } else { try { - val deviceController = getDeviceController() sessionController!!.recreate( - deviceController, + getDeviceController(), outputs.values.toList(), - dynamicRange = dynamicRangeProfile.dynamicRange, - fpsRange = fpsRange + dynamicRange = dynamicRangeProfile.dynamicRange ).apply { applySessionController(this) Logger.d(TAG, "Session controller recreated") @@ -193,7 +210,10 @@ internal class CameraController( } } - private fun applySessionController(sessionController: CameraSessionController) { + @RequiresPermission(Manifest.permission.CAMERA) + private suspend fun applySessionController( + sessionController: CameraSessionController + ) { this.sessionController = sessionController isActiveJob = coroutineScope.launch { @@ -201,6 +221,19 @@ internal class CameraController( _isActiveFlow.emit(!it) } } + + // Re-add targets that are in the new outputs (identified by their name) + captureRequestBuilder?.let { + val targets = outputs.values.filter { output -> + it.hasTarget(output) + } + it.clearTargets() + if (targets.isNotEmpty()) { + it.addTargets(targets) + } + } + + applyRepeatingSessionUnsafe() } /** @@ -222,8 +255,7 @@ internal class CameraController( sessionController.recreate( getDeviceController(), outputs.values.toList(), - dynamicRange = dynamicRangeProfile.dynamicRange, - fpsRange = fpsRange + dynamicRange = dynamicRangeProfile.dynamicRange ).apply { applySessionController(this) Logger.d(TAG, "Session controller restarted") @@ -239,53 +271,18 @@ internal class CameraController( @RequiresPermission(Manifest.permission.CAMERA) suspend fun addTarget(name: String) = withContext(defaultDispatcher) { controllerMutex.withLock { - val sessionController = getSessionController() - sessionController.addTarget(name) - } - } + val target = outputs[name] ?: throw IllegalArgumentException( + "Output $name not found in outputs ${outputs.keys.joinToString(", ")}" + ) - /** - * Adds a target to the current capture session. - * - * @param target The target to add - * @return true if the target has been added, false otherwise - */ - @RequiresPermission(Manifest.permission.CAMERA) - suspend fun addTarget(target: CameraSurface) = withContext(defaultDispatcher) { - controllerMutex.withLock { - val sessionController = getSessionController() - sessionController.addTarget(target) - } - } - - /** - * Adds targets to the current capture session. - * - * @param targets The targets to add - * @return true if the targets have been added, false otherwise - */ - @RequiresPermission(Manifest.permission.CAMERA) - suspend fun addTargets(targets: List) = withContext(defaultDispatcher) { - controllerMutex.withLock { - val sessionController = getSessionController() - sessionController.addTargets(targets) - } - } - - /** - * Removes a target from the current capture session. - * - * @param target The target to remove - */ - @RequiresPermission(Manifest.permission.CAMERA) - suspend fun removeTarget(target: CameraSurface) { - withContext(defaultDispatcher) { - controllerMutex.withLock { - val sessionController = getSessionController() - sessionController.removeTarget(target) - if (sessionController.isEmpty()) { - closeControllers() + val wasAdded = getCaptureRequestBuilder().addTarget(target) + if (wasAdded) { + try { + applyRepeatingSessionUnsafe() + } catch (t: Throwable) { + Logger.e(TAG, "Error to add target $name", t) } + Logger.d(TAG, "Target $name added") } } } @@ -299,11 +296,18 @@ internal class CameraController( suspend fun removeTarget(name: String) { withContext(defaultDispatcher) { controllerMutex.withLock { - val sessionController = getSessionController() - sessionController.removeTarget(name) - if (sessionController.isEmpty()) { - closeControllers() + val target = outputs[name] ?: return@withContext + + val wasRemoved = getCaptureRequestBuilder().removeTarget(target) + if (!wasRemoved) { + return@withLock + } + try { + applyRepeatingSessionUnsafe() + } catch (t: Throwable) { + Logger.e(TAG, "Error to remove target $name", t) } + Logger.d(TAG, "Target $name removed") } } } @@ -311,22 +315,44 @@ internal class CameraController( /** * Gets a setting from the current capture request. */ + @RequiresPermission(Manifest.permission.CAMERA) fun getSetting(key: CaptureRequest.Key): T? { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - return sessionController.getSetting(key) + val captureRequestBuilder = + requireNotNull(captureRequestBuilder) { "CaptureRequestBuilder is null" } + return captureRequestBuilder.get(key) + } + + /** + * Resets the current capture request to its default settings. + * + * Don't forget to call [applyRepeatingSession] to apply the setting. + */ + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun resetSettings() { + withContext(defaultDispatcher) { + controllerMutex.withLock { + captureRequestBuilder ?: return@withLock + val currentCaptureRequestBuilder = getCaptureRequestBuilder() + // Get targets of previous builder + val targets = currentCaptureRequestBuilder.getTargets() + captureRequestBuilder = null + getCaptureRequestBuilder().addTargets(targets) + } + } } /** * Sets a setting to the current capture request. * - * Don't forget to call [setRepeatingSession] to apply the setting. + * Don't forget to call [applyRepeatingSession] to apply the setting. * * @param key The setting key * @param value The setting value */ - fun setSetting(key: CaptureRequest.Key, value: T) { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setSetting(key, value) + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun setSetting(key: CaptureRequest.Key, value: T) { + val captureRequestBuilder = getCaptureRequestBuilder() + captureRequestBuilder.set(key, value) } /** @@ -334,6 +360,7 @@ internal class CameraController( * * @param fpsRange The fps range */ + @RequiresPermission(Manifest.permission.CAMERA) suspend fun setFps(fps: Int) { withContext(defaultDispatcher) { controllerMutex.withLock { @@ -343,12 +370,15 @@ internal class CameraController( this@CameraController.fps = fps + val captureRequestBuilder = getCaptureRequestBuilder() + + val range = fpsRange + val minFrameDuration = 1_000_000_000 / range.upper.toLong() + captureRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range) + captureRequestBuilder.set(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) + if (isActiveFlow.value) { - val range = fpsRange - val minFrameDuration = 1_000_000_000 / range.upper.toLong() - setSetting(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range) - setSetting(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) - setRepeatingSession() + applyRepeatingSessionUnsafe() } } } @@ -378,34 +408,65 @@ internal class CameraController( } /** - * Sets a repeating session sync with the current capture request. + * Adds a capture callback listener to the current capture session. * - * It returns only when the capture callback has been called for the first time. + * The listener is removed when it returns true or [removeCaptureCallbackListener] is called. */ - suspend fun setRepeatingSessionSync() { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setRepeatingSessionSync() + suspend fun addCaptureCallbackListener(listener: CaptureResultListener) { + sessionCallback.addListener(listener) } + /** + * Removes a capture callback listener from the current capture session. + * + * @param listener The listener to remove + */ + suspend fun removeCaptureCallbackListener(listener: CaptureResultListener) { + sessionCallback.removeListener(listener) + } + + /** + * Sets a repeating session with the current capture request. + * + * @param tag A tag to associate with the session. + */ + @RequiresPermission(Manifest.permission.CAMERA) + private suspend fun applyRepeatingSessionUnsafe(tag: Any? = null) { + val sessionController = getSessionController() + val captureRequestBuilder = getCaptureRequestBuilder() + tag?.let { + captureRequestBuilder.setTag(it) + } + sessionController.applyRepeatingSession(captureRequestBuilder) + } + + /** * Sets a repeating session with the current capture request. + * + * @param tag A tag to associate with the session. */ - suspend fun setRepeatingSession() { - val sessionController = requireNotNull(sessionController) { "SessionController is null" } - sessionController.setRepeatingSession() + @RequiresPermission(Manifest.permission.CAMERA) + suspend fun applyRepeatingSession(tag: Any? = null) { + withContext(defaultDispatcher) { + controllerMutex.withLock { + applyRepeatingSessionUnsafe(tag) + } + } } private suspend fun closeControllers() { sessionController?.close() + sessionController = null deviceController?.close() deviceController = null + captureRequestBuilder = null } suspend fun release() { withContext(defaultDispatcher) { controllerMutex.withLock { closeControllers() - sessionController = null } } outputs.clear() diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraDeviceController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraDeviceController.kt index 200a146ce..24dcf9220 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraDeviceController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraDeviceController.kt @@ -26,6 +26,7 @@ import android.view.Surface import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.ICameraCaptureSessionCompat import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -54,10 +55,9 @@ internal class CameraDeviceController private constructor( val id = cameraDevice.id - fun createCaptureRequest( - templateType: Int - ): CaptureRequest.Builder { - return cameraDevice.createCaptureRequest(templateType) + fun createCaptureRequestBuilder(template: Int = CameraDevice.TEMPLATE_RECORD): CaptureRequestWithTargetsBuilder { + val captureRequest = cameraDevice.createCaptureRequest(template) + return CaptureRequestWithTargetsBuilder(captureRequest) } fun createCaptureSession( @@ -66,7 +66,6 @@ internal class CameraDeviceController private constructor( sessionCompat.createCaptureSession(cameraDevice, targets, callback) } - fun createCaptureSessionByOutputConfiguration( outputConfigurations: List, callback: CameraCaptureSession.StateCallback diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt index fc3843e3f..44f50ac90 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/controllers/CameraSessionController.kt @@ -20,34 +20,28 @@ import android.hardware.camera2.CameraCaptureSession.CaptureCallback import android.hardware.camera2.CaptureFailure import android.hardware.camera2.CaptureRequest import android.hardware.camera2.TotalCaptureResult -import android.util.Range import android.view.Surface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.sessioncompat.ICameraCaptureSessionCompat +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSessionCallback import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraSurface import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CameraUtils import io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils.CaptureRequestWithTargetsBuilder import io.github.thibaultbee.streampack.core.logger.Logger -import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.first -import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.util.concurrent.atomic.AtomicLong -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine internal class CameraSessionController private constructor( - private val coroutineScope: CoroutineScope, - private val captureRequestBuilder: CaptureRequestWithTargetsBuilder, - private val sessionCompat: ICameraCaptureSessionCompat, + private val coroutineDispatcher: CoroutineDispatcher, private val captureSession: CameraCaptureSession, - private val outputs: List, - val dynamicRange: Long, + private val sessionCallback: CameraSessionCallback, + private val sessionCompat: ICameraCaptureSessionCompat, val cameraIsClosedFlow: StateFlow, val isClosedFlow: StateFlow ) { @@ -56,10 +50,6 @@ internal class CameraSessionController private constructor( val isClosed: Boolean get() = isClosedFlow.value || cameraIsClosedFlow.value - private val requestTargetMutex = Mutex() - - private val nextSessionUpdateId = AtomicLong(0) - /** * A default capture callback that logs the failure reason. */ @@ -72,167 +62,11 @@ internal class CameraSessionController private constructor( } } - private val sessionCallback = CameraControlSessionCallback(coroutineScope) - private val captureCallbacks = - mutableSetOf(captureCallback, sessionCallback) - - suspend fun isEmpty() = withContext(coroutineScope.coroutineContext) { - requestTargetMutex.withLock { captureRequestBuilder.isEmpty() } - } - - /** - * Whether the current capture request has a target - * - * @param surface The target to check - * @return true if the target is in the current capture request, false otherwise - */ - suspend fun hasTarget(surface: Surface) = withContext(coroutineScope.coroutineContext) { - requestTargetMutex.withLock { - captureRequestBuilder.hasTarget(surface) - } - } - - /** - * Whether the current capture request has a target - * - * @param cameraSurface The target to check - * @return true if the target is in the current capture request, false otherwise - */ - suspend fun hasTarget(cameraSurface: CameraSurface) = - withContext(coroutineScope.coroutineContext) { - requestTargetMutex.withLock { - captureRequestBuilder.hasTarget(cameraSurface) - } - } - - /** - * Adds targets to the current capture session - * - * @param targets The targets to add - */ - suspend fun addTargets(targets: List): Boolean { - require(targets.isNotEmpty()) { "At least one target is required" } - require(targets.all { it.surface.isValid }) { "All targets must be valid" } - require(targets.all { outputs.contains(it) }) { "Targets must be in the current capture session: $targets ($outputs)" } - - val res = withContext(coroutineScope.coroutineContext) { - requestTargetMutex.withLock { - val res = targets.map { - captureRequestBuilder.addTarget(it) - }.all { it } - setRepeatingSession() - res - } - } - - return res - } - - /** - * Adds a target to the current capture session - * - * @param name The name of target to add - */ - suspend fun addTarget(name: String): Boolean { - require(outputs.any { it.name == name }) { "Target type must be in the current capture session: $name ($outputs)" } - - val res = withContext(coroutineScope.coroutineContext) { - requestTargetMutex.withLock { - val target = outputs.first { it.name == name } - val res = captureRequestBuilder.addTarget(target) - setRepeatingSession() - res - } - } - return res - } - - /** - * Adds a target to the current capture session - * - * @param target The target to add - */ - suspend fun addTarget(target: CameraSurface): Boolean { - require(target.surface.isValid) { "Target must be valid: $target" } - require(outputs.contains(target)) { "Target must be in the current capture session: $target ($outputs)" } - - val res = withContext(coroutineScope.coroutineContext) { - requestTargetMutex.withLock { - val res = captureRequestBuilder.addTarget(target) - setRepeatingSession() - res - } - } - return res - } - - /** - * Removes targets from the current capture session - * - * @param targets The targets to remove - */ - suspend fun removeTargets(targets: List) { - withContext(coroutineScope.coroutineContext) { - requestTargetMutex.withLock { - targets.forEach { - captureRequestBuilder.removeTarget(it) - } - if (captureRequestBuilder.isEmpty()) { - stopRepeatingSession() - } else { - setRepeatingSession() - } - } - } - } - - /** - * Removes a target from the current capture session - * - * @param name The name of target to remove - */ - suspend fun removeTarget(name: String) { - withContext(coroutineScope.coroutineContext) { - requestTargetMutex.withLock { - val target = outputs.firstOrNull { it.name == name } - target?.let { - captureRequestBuilder.removeTarget(it) - } ?: Logger.w( - TAG, - "Target type $name not found in current outputs $outputs" - ) - - if (captureRequestBuilder.isEmpty()) { - stopRepeatingSession() - } else { - setRepeatingSession() - } - } - } - } - - /** - * Removes a target from the current capture session - * - * @param target The target to remove - */ - suspend fun removeTarget(target: CameraSurface) { - withContext(coroutineScope.coroutineContext) { - requestTargetMutex.withLock { - captureRequestBuilder.removeTarget(target) - - if (captureRequestBuilder.isEmpty()) { - stopRepeatingSession() - } else { - setRepeatingSession() - } - } - } - } + setOf(captureCallback, sessionCallback) suspend fun close() { - withContext(coroutineScope.coroutineContext) { + withContext(coroutineDispatcher) { captureSessionMutex.withLock { if (isClosed) { Logger.w(TAG, "Session already closed") @@ -250,77 +84,32 @@ internal class CameraSessionController private constructor( } } - suspend fun removeCaptureCallback( - cameraCaptureCallback: CaptureCallback - ) { - withContext(coroutineScope.coroutineContext) { - captureSessionMutex.withLock { - if (isClosed) { - Logger.w(TAG, "Camera session controller is released") - return@withContext - } - - captureCallbacks.remove(cameraCaptureCallback) - } - } - } - /** - * Sets a repeating session with the current capture request. + * Sets or stops a repeating session with the current capture request. + * + * If the [captureRequestBuilder] does not hold a [Surface], it will stop the repeating session. + * + * @param captureRequestBuilder The capture request builder to use */ - suspend fun setRepeatingSessionSync() { + suspend fun applyRepeatingSession(captureRequestBuilder: CaptureRequestWithTargetsBuilder) { if (captureRequestBuilder.isEmpty()) { - Logger.w(TAG, "Capture request is empty") - return - } - withContext(coroutineScope.coroutineContext) { - captureSessionMutex.withLock { - if (isClosed) { - Logger.w(TAG, "Camera session controller is released") - return@withContext - } - - suspendCoroutine { continuation -> - val nextSessionId = nextSessionUpdateId.getAndIncrement() - - val captureCallback = object : CaptureResultListener { - override fun onCaptureResult(result: TotalCaptureResult): Boolean { - val tag = result.request.tag as? TagBundle - val keyId = tag?.keyId ?: return false - if (keyId >= nextSessionId) { - continuation.resume(Unit) - return true - } - return false - } - } - - sessionCallback.addListener(captureCallback) - - captureRequestBuilder.setTag( - TagBundle().apply { - keyId = nextSessionId - } - ) - sessionCompat.setRepeatingSingleRequest( - captureSession, - captureRequestBuilder.build(), - MultiCaptureCallback(captureCallbacks) - ) - } - } + stopRepeatingSession() + } else { + setRepeatingSession(captureRequestBuilder) } } /** * Sets a repeating session with the current capture request. + * + * @param captureRequestBuilder The capture request builder to use */ - suspend fun setRepeatingSession() { + private suspend fun setRepeatingSession(captureRequestBuilder: CaptureRequestWithTargetsBuilder) { if (captureRequestBuilder.isEmpty()) { Logger.w(TAG, "Capture request is empty") return } - withContext(coroutineScope.coroutineContext) { + withContext(coroutineDispatcher) { captureSessionMutex.withLock { if (isClosed) { Logger.w(TAG, "Camera session controller is released") @@ -337,7 +126,7 @@ internal class CameraSessionController private constructor( } private suspend fun stopRepeatingSession() { - withContext(coroutineScope.coroutineContext) { + withContext(coroutineDispatcher) { captureSessionMutex.withLock { if (isClosed) { Logger.w(TAG, "Camera session controller is released") @@ -349,22 +138,6 @@ internal class CameraSessionController private constructor( } } - /** - * Gets a setting from the current capture request. - */ - fun getSetting(key: CaptureRequest.Key) = captureRequestBuilder.get(key) - - /** - * Sets a setting to the current capture request. - * - * Don't forget to call [setRepeatingSession] to apply the setting. - * - * @param key The setting key - * @param value The setting value - */ - fun setSetting(key: CaptureRequest.Key, value: T) = - captureRequestBuilder.set(key, value) - /** * Creates a new capture session with the given outputs. * @@ -379,69 +152,44 @@ internal class CameraSessionController private constructor( cameraDeviceController: CameraDeviceController, outputs: List, dynamicRange: Long, - fpsRange: Range - ): CameraSessionController = withContext(coroutineScope.coroutineContext) { - requestTargetMutex.withLock { - require(outputs.isNotEmpty()) { "At least one output is required" } - require(outputs.all { it.surface.isValid }) { "All outputs $outputs must be valid but ${outputs.filter { !it.surface.isValid }} is invalid" } - - if ((dynamicRange == this@CameraSessionController.dynamicRange) && (outputs == this@CameraSessionController.outputs) && !isClosed) { - Logger.w(TAG, "Same dynamic range and outputs, returning the same controller") - return@withContext this@CameraSessionController - } - - // Re-add targets that are in the new outputs (identified by their name) - val targets = outputs.filter { - captureRequestBuilder.hasTarget(it.name) - } - captureRequestBuilder.clearTargets() - captureRequestBuilder.addTargets(targets) - val minFrameDuration = 1_000_000_000 / fpsRange.upper.toLong() - captureRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange) - captureRequestBuilder.set(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) - - // Close current session - close() - - val isClosedFlow = MutableStateFlow(false) - val newCaptureSession = - CameraUtils.createCaptureSession( - cameraDeviceController, - outputs.map { it.surface }, - dynamicRange, - isClosedFlow - ) - - val controller = CameraSessionController( - coroutineScope, - captureRequestBuilder, - sessionCompat, - newCaptureSession, - outputs, + ): CameraSessionController = withContext(coroutineDispatcher) { + require(outputs.isNotEmpty()) { "At least one output is required" } + require(outputs.all { it.surface.isValid }) { "All outputs $outputs must be valid but ${outputs.filter { !it.surface.isValid }} is invalid" } + + // Close current session + close() + + val isClosedFlow = MutableStateFlow(false) + val newCaptureSession = + CameraUtils.createCaptureSession( + cameraDeviceController, + outputs.map { it.surface }, dynamicRange, - cameraDeviceController.isClosedFlow, - isClosedFlow.asStateFlow() + isClosedFlow ) - if (!captureRequestBuilder.isEmpty()) { - controller.setRepeatingSession() - } + val controller = CameraSessionController( + coroutineDispatcher, + newCaptureSession, + sessionCallback, + sessionCompat, + cameraDeviceController.isClosedFlow, + isClosedFlow.asStateFlow() + ) - controller - } + controller } companion object { private const val TAG = "CameraSessionController" suspend fun create( - coroutineScope: CoroutineScope, - sessionCompat: ICameraCaptureSessionCompat, cameraDeviceController: CameraDeviceController, + sessionCallback: CameraSessionCallback, + sessionCompat: ICameraCaptureSessionCompat, outputs: List, dynamicRange: Long, - fpsRange: Range, - defaultRequestBuilder: CaptureRequestWithTargetsBuilder.() -> Unit = {} + coroutineDispatcher: CoroutineDispatcher ): CameraSessionController { require(outputs.isNotEmpty()) { "At least one output is required" } require(outputs.all { it.surface.isValid }) { "All outputs $outputs must be valid but ${outputs.filter { !it.surface.isValid }} is invalid" } @@ -455,21 +203,11 @@ internal class CameraSessionController private constructor( isClosedFlow ) - val captureRequestBuilder = CaptureRequestWithTargetsBuilder.create( - cameraDeviceController - ).apply { - val minFrameDuration = 1_000_000_000 / fpsRange.upper.toLong() - set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange) - set(CaptureRequest.SENSOR_FRAME_DURATION, minFrameDuration) - defaultRequestBuilder() - } return CameraSessionController( - coroutineScope, - captureRequestBuilder, - sessionCompat, + coroutineDispatcher, captureSession, - outputs, - dynamicRange, + sessionCallback, + sessionCompat, cameraDeviceController.isClosedFlow, isClosedFlow.asStateFlow() ) @@ -513,68 +251,4 @@ internal class CameraSessionController private constructor( } } } - - private class TagBundle { - private val tagMap = mutableMapOf() - - var keyId: Long? - get() = tagMap[TAG_KEY_ID] as? Long - set(value) { - tagMap[TAG_KEY_ID] = value - } - - val keys: Set - get() = tagMap.keys - - fun setTag(key: String, value: Any?) { - tagMap[key] = value - } - - companion object { - private const val TAG_KEY_ID = "TAG_KEY_ID" - } - } - - interface CaptureResultListener { - /** - * Called when a capture result is received. - * - * @param result The capture result. - * @return true if the listener is finished and should be removed, false otherwise. - */ - fun onCaptureResult(result: TotalCaptureResult): Boolean - } - - private class CameraControlSessionCallback(private val coroutineScope: CoroutineScope) : - CaptureCallback() { - /* synthetic accessor */ - private val resultListeners = mutableSetOf() - - fun addListener(listener: CaptureResultListener) { - resultListeners.add(listener) - } - - fun removeListener(listener: CaptureResultListener) { - resultListeners.remove(listener) - } - - override fun onCaptureCompleted( - session: CameraCaptureSession, - request: CaptureRequest, - result: TotalCaptureResult - ) { - coroutineScope.launch { - val removeSet = mutableSetOf() - for (listener in resultListeners) { - val isFinished: Boolean = listener.onCaptureResult(result) - if (isFinished) { - removeSet.add(listener) - } - } - if (!removeSet.isEmpty()) { - resultListeners.removeAll(removeSet) - } - } - } - } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt new file mode 100644 index 000000000..67b8b086d --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CameraSessionCallback.kt @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils + +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCaptureSession.CaptureCallback +import android.hardware.camera2.CaptureRequest +import android.hardware.camera2.TotalCaptureResult +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** + * A capture callback that wraps multiple [CaptureResultListener]. + * + * @param coroutineScope The coroutine scope to use. + */ +internal class CameraSessionCallback(private val coroutineScope: CoroutineScope) : + CaptureCallback() { + /* synthetic accessor */ + private val resultListeners = mutableSetOf() + private val mutex = Mutex() + + /** + * Adds a capture result listener. + * + * The listener is removed when [removeListener] is explicitly called or when [CaptureResultListener] returns true. + */ + suspend fun addListener(listener: CaptureResultListener) { + mutex.withLock { + resultListeners.add(listener) + } + } + + suspend fun removeListener(listener: CaptureResultListener) { + mutex.withLock { + resultListeners.remove(listener) + } + } + + override fun onCaptureCompleted( + session: CameraCaptureSession, + request: CaptureRequest, + result: TotalCaptureResult + ) { + coroutineScope.launch { + val removeSet = mutableSetOf() + mutex.withLock { + for (listener in resultListeners) { + val isFinished: Boolean = listener.onCaptureResult(result) + if (isFinished) { + removeSet.add(listener) + } + } + } + if (!removeSet.isEmpty()) { + mutex.withLock { + resultListeners.removeAll(removeSet) + } + } + } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt index 8104e437b..1bd541b3f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureRequestWithTargetsBuilder.kt @@ -15,28 +15,27 @@ */ package io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils -import android.hardware.camera2.CameraDevice import android.hardware.camera2.CaptureRequest import android.hardware.camera2.CaptureRequest.Builder import android.view.Surface -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.controllers.CameraDeviceController -import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock /** * A class to store for [CaptureRequest] with targets */ -internal class CaptureRequestWithTargetsBuilder private constructor( - private val captureRequestBuilder: Builder +internal class CaptureRequestWithTargetsBuilder( + private var captureRequestBuilder: Builder ) { private val mutableTargets = mutableSetOf() private val mutex = Mutex() /** - * The targets of the CaptureRequest + * Gets the targets from the CaptureRequest + * + * @return The targets */ - private val targets = mutableTargets.toList() + suspend fun getTargets() = mutex.withLock { mutableTargets.toList() } /** * Whether the CaptureRequest has no target @@ -64,12 +63,14 @@ internal class CaptureRequestWithTargetsBuilder private constructor( } /** - * Adds a target to the CaptureRequest + * Adds a target to the CaptureRequest from a [CameraSurface] * * @param cameraSurface The surface to add * @return true if the surface was added, false otherwise */ suspend fun addTarget(cameraSurface: CameraSurface): Boolean = mutex.withLock { + require(cameraSurface.surface.isValid) { "Target $cameraSurface must be valid" } + val wasAdded = mutableTargets.add(cameraSurface) if (wasAdded) { captureRequestBuilder.addTarget(cameraSurface.surface) @@ -84,6 +85,9 @@ internal class CaptureRequestWithTargetsBuilder private constructor( * @return true if the surface was added, false otherwise */ suspend fun addTargets(cameraSurfaces: List) = mutex.withLock { + require(cameraSurfaces.isNotEmpty()) { "At least one target is required" } + require(cameraSurfaces.all { it.surface.isValid }) { "All targets must be valid" } + cameraSurfaces.forEach { val wasAdded = mutableTargets.add(it) if (wasAdded) { @@ -137,19 +141,6 @@ internal class CaptureRequestWithTargetsBuilder private constructor( fun build() = captureRequestBuilder.build() override fun toString(): String { - return "$captureRequestBuilder with targets: $targets" - } - - companion object { - /** - * Create a CaptureRequestBuilderWithTargets - * - * @param cameraDeviceController The camera device controller - * @param template The template to use - */ - fun create( - cameraDeviceController: CameraDeviceController, - template: Int = CameraDevice.TEMPLATE_RECORD, - ) = CaptureRequestWithTargetsBuilder(cameraDeviceController.createCaptureRequest(template)) + return "$captureRequestBuilder with targets: $mutableTargets" } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt new file mode 100644 index 000000000..c97eeda2f --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/sources/video/camera/utils/CaptureResultListener.kt @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.sources.video.camera.utils + +import android.hardware.camera2.TotalCaptureResult + +interface CaptureResultListener { + /** + * Called when a capture result is received. + * + * @param result The capture result. + * @return true if the listener is finished and should be removed, false otherwise. + */ + fun onCaptureResult(result: TotalCaptureResult): Boolean +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/ChannelWithCloseableData.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/ChannelWithCloseableData.kt index 130cf4450..35a803271 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/ChannelWithCloseableData.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/ChannelWithCloseableData.kt @@ -34,10 +34,14 @@ import java.io.Closeable */ class ChannelWithCloseableData( capacity: Int = RENDEZVOUS, - onBufferOverflow: BufferOverflow = BufferOverflow.SUSPEND + onBufferOverflow: BufferOverflow = BufferOverflow.SUSPEND, + onUndeliveredElement: ((T) -> Unit) = {} ) : ReceiveChannel> { private val channel = - Channel>(capacity, onBufferOverflow, onUndeliveredElement = { it.close() }) + Channel>(capacity, onBufferOverflow, onUndeliveredElement = { + it.close() + onUndeliveredElement(it.data) + }) /** * Sends data along with a close action to the channel. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/CoroutineScheduler.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/CoroutineScheduler.kt index 2c4e88c51..250349654 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/CoroutineScheduler.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/CoroutineScheduler.kt @@ -22,8 +22,10 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.launch +import kotlin.time.Duration + class CoroutineScheduler( - private val delayTimeInMs: Long, + private val delayTime: Duration, coroutineDispatcher: CoroutineDispatcher, private val action: suspend CoroutineScope.() -> Unit ) { @@ -36,7 +38,7 @@ class CoroutineScheduler( } job = coroutineScope.launch { while (true) { - delay(delayTimeInMs) + delay(delayTime) launch { action() } } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt index 75281c053..0b0c0fd3f 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/avc/AVCDecoderConfigurationRecord.kt @@ -19,8 +19,8 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.buffer.ByteBuffer import io.github.thibaultbee.streampack.core.elements.utils.av.video.ChromaFormat import io.github.thibaultbee.streampack.core.elements.utils.extensions.put import io.github.thibaultbee.streampack.core.elements.utils.extensions.putShort -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.shl +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize import java.nio.ByteBuffer @@ -33,8 +33,8 @@ data class AVCDecoderConfigurationRecord( private val sps: List, private val pps: List ) : ByteBufferWriter() { - private val spsNoStartCode: List = sps.map { it.removeStartCode() } - private val ppsNoStartCode: List = pps.map { it.removeStartCode() } + private val spsNoStartCode: List = sps.map { it.skipStartCode() } + private val ppsNoStartCode: List = pps.map { it.skipStartCode() } override val size: Int = getSize(spsNoStartCode, ppsNoStartCode) @@ -86,11 +86,13 @@ data class AVCDecoderConfigurationRecord( sps: List, pps: List ): AVCDecoderConfigurationRecord { - val spsNoStartCode = sps.map { it.removeStartCode() } - val ppsNoStartCode = pps.map { it.removeStartCode() } - val profileIdc: Byte = spsNoStartCode[0].get(1) - val profileCompatibility = spsNoStartCode[0].get(2) - val levelIdc = spsNoStartCode[0].get(3) + val spsNoStartCode = sps.map { it.skipStartCode() } + val ppsNoStartCode = pps.map { it.skipStartCode() } + val firstSpsNoStartCode = spsNoStartCode[0] + val firstSpsNoStartCodePosition = firstSpsNoStartCode.position() + val profileIdc: Byte = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 1) + val profileCompatibility = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 2) + val levelIdc = firstSpsNoStartCode.get(firstSpsNoStartCodePosition + 3) return AVCDecoderConfigurationRecord( profileIdc = profileIdc, profileCompatibility = profileCompatibility, @@ -112,7 +114,8 @@ data class AVCDecoderConfigurationRecord( size += 2 + it.remaining() - it.startCodeSize } val spsStartCodeSize = sps[0].startCodeSize - val profileIdc = sps[0].get(spsStartCodeSize + 1).toInt() + val spsPosition = sps[0].position() + val profileIdc = sps[0].get(spsPosition + spsStartCodeSize + 1).toInt() if ((profileIdc == 100) || (profileIdc == 110) || (profileIdc == 122) || (profileIdc == 144)) { size += 4 } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt index 28a11669f..7bd86d908 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt @@ -20,7 +20,7 @@ import io.github.thibaultbee.streampack.core.elements.utils.av.video.ChromaForma import io.github.thibaultbee.streampack.core.elements.utils.extensions.put import io.github.thibaultbee.streampack.core.elements.utils.extensions.putLong48 import io.github.thibaultbee.streampack.core.elements.utils.extensions.putShort -import io.github.thibaultbee.streampack.core.elements.utils.extensions.removeStartCode +import io.github.thibaultbee.streampack.core.elements.utils.extensions.skipStartCode import io.github.thibaultbee.streampack.core.elements.utils.extensions.shl import io.github.thibaultbee.streampack.core.elements.utils.extensions.shr import io.github.thibaultbee.streampack.core.elements.utils.extensions.startCodeSize @@ -195,7 +195,7 @@ data class HEVCDecoderConfigurationRecord( } data class NalUnit(val type: Type, val data: ByteBuffer, val completeness: Boolean = true) { - val noStartCodeData: ByteBuffer = data.removeStartCode() + val noStartCodeData: ByteBuffer = data.skipStartCode() fun write(buffer: ByteBuffer) { buffer.put((completeness shl 7) or type.value.toInt()) // array_completeness + reserved 1bit + naluType 6 bytes diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt index 66da15b18..09cee1ff1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensions.kt @@ -15,6 +15,7 @@ */ package io.github.thibaultbee.streampack.core.elements.utils.extensions +import io.github.thibaultbee.streampack.core.elements.utils.pool.IBufferPool import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.charset.StandardCharsets @@ -86,12 +87,14 @@ fun ByteBuffer.put3x3Matrix(matrix: IntArray) { matrix.forEach { putInt(it) } } -fun ByteBuffer.put(buffer: ByteBuffer, offset: Int, length: Int) { - val limit = buffer.limit() - buffer.position(offset) - buffer.limit(offset + length) - this.put(buffer) - buffer.limit(limit) +fun ByteBuffer.put(src: ByteBuffer, offset: Int, length: Int) { + val limit = src.limit() + if (offset != 0) { + src.position(src.position() + offset) + } + src.limit(src.position() + offset + length) + this.put(src) + src.limit(limit) } fun ByteBuffer.getString(size: Int = this.remaining()): String { @@ -111,22 +114,38 @@ fun ByteBuffer.getLong(isLittleEndian: Boolean): Long { return value } -fun ByteBuffer.indicesOf(prefix: ByteArray): List { - if (prefix.isEmpty()) { - return emptyList() +/** + * Finds all occurrences of the given [needle] byte array within the ByteBuffer. + * @param needle The byte array sequence to search for. + * @return A list of starting indices for every match found. + */ +fun ByteBuffer.indicesOf(needle: ByteArray): List { + if (needle.isEmpty()) return emptyList() + + val results = mutableListOf() + val end = limit() - needle.size + var i = position() + + while (i <= end) { + if (match(i, needle)) { + results.add(i) + // Move forward by the needle's length to find the next non-overlapping match + // Or use i++ if you want to allow overlapping matches (e.g., "AAA" in "AAAA") + i += needle.size + } else { + i++ + } } + return results +} - val indices = mutableListOf() - - outer@ for (i in 0 until this.limit() - prefix.size + 1) { - for (j in prefix.indices) { - if (this.get(i + j) != prefix[j]) { - continue@outer - } +private fun ByteBuffer.match(start: Int, needle: ByteArray): Boolean { + for (idx in needle.indices) { + if (get(start + idx) != needle[idx]) { + return false } - indices.add(i) } - return indices + return true } /** @@ -137,18 +156,18 @@ fun ByteBuffer.slices(prefix: ByteArray): List { // Get all occurrence of prefix in buffer val indexes = this.indicesOf(prefix) + // Get slices indexes.forEachIndexed { index, i -> val nextPosition = if (indexes.indices.contains(index + 1)) { - indexes[index + 1] - 1 + indexes[index + 1] } else { - this.limit() - 1 + this.limit() } slices.add(Pair(i, nextPosition)) } - val array = this.array() return slices.map { - ByteBuffer.wrap(array.sliceArray(IntRange(it.first, it.second))) + this.slice(from = it.first, to = it.second) } } @@ -235,18 +254,37 @@ fun ByteBuffer.toByteArray(): ByteArray { } /** - * Clone [ByteBuffer]. + * Deep copy of [ByteBuffer]. * The position of the original [ByteBuffer] will be 0 after the clone. */ -fun ByteBuffer.clone(): ByteBuffer { +@Deprecated("Use ByteBufferPool instead") +fun ByteBuffer.deepCopy(): ByteBuffer { val originalPosition = this.position() try { - val clone = if (isDirect) { + val copy = if (isDirect) { ByteBuffer.allocateDirect(this.remaining()) } else { ByteBuffer.allocate(this.remaining()) } - return clone.put(this).apply { rewind() } + return copy.put(this).apply { rewind() } + } finally { + this.position(originalPosition) + } +} + +/** + * Deep copy of [ByteBuffer] from [IBufferPool]. + * + * Don't forget to put the returned [ByteBuffer] to the buffer pool when you are done with it. + * + * @param pool [IBufferPool] to use + * @return [ByteBuffer] deep copy + */ +fun ByteBuffer.deepCopy(pool: IBufferPool): ByteBuffer { + val originalPosition = this.position() + try { + val copy = pool.get(this.remaining()) + return copy.put(this).apply { rewind() } } finally { this.position(originalPosition) } @@ -258,51 +296,72 @@ fun ByteBuffer.clone(): ByteBuffer { /** - * Get start code size of [ByteBuffer]. + * Get start code size of [ByteBuffer] from the current position */ val ByteBuffer.startCodeSize: Int - get() { - return if (this.get(0) == 0x00.toByte() && this.get(1) == 0x00.toByte() - && this.get(2) == 0x00.toByte() && this.get(3) == 0x01.toByte() - ) { - 4 - } else if (this.get(0) == 0x00.toByte() && this.get(1) == 0x00.toByte() - && this.get(2) == 0x01.toByte() - ) { - 3 - } else { - 0 - } + get() = getStartCodeSize(this.position()) + +/** + * Get start code size of [ByteBuffer] from the given [position]. + */ +fun ByteBuffer.getStartCodeSize( + position: Int +): Int { + return if (remaining() >= 4 && this.get(position) == 0x00.toByte() && this.get(position + 1) == 0x00.toByte() + && this.get(position + 2) == 0x00.toByte() && this.get(position + 3) == 0x01.toByte() + ) { + 4 + } else if (remaining() >= 3 && this.get(position) == 0x00.toByte() && this.get(position + 1) == 0x00.toByte() + && this.get(position + 2) == 0x01.toByte() + ) { + 3 + } else { + 0 } +} -fun ByteBuffer.removeStartCode(): ByteBuffer { +/** + * Moves the position after the start code. + */ +fun ByteBuffer.skipStartCode(): ByteBuffer { val startCodeSize = this.startCodeSize - this.position(startCodeSize) - return this.slice() + if (startCodeSize > 0) { + this.position(this.position() + startCodeSize) + } + return this } +private val emulationPreventionThreeByte = byteArrayOf(0x00, 0x00, 0x03) + +/** + * Removes all emulation prevention three bytes from [ByteBuffer]. + * + * @param headerLength [Int] of the header length before writing the [ByteBuffer]. + * @return [ByteBuffer] without emulation prevention three bytes + */ fun ByteBuffer.extractRbsp(headerLength: Int): ByteBuffer { - val rbsp = ByteBuffer.allocateDirect(this.remaining()) + val indices = this.indicesOf(emulationPreventionThreeByte) - val indices = this.indicesOf(byteArrayOf(0x00, 0x00, 0x03)) + val rbspSize = + this.remaining() - indices.size // remove 0x3 bytes for each emulation prevention three bytes + val rbsp = ByteBuffer.allocateDirect(rbspSize) - rbsp.put(this, this.startCodeSize, headerLength) + // Write header to new buffer + rbsp.put(this, 0, headerLength + this.startCodeSize) - var previous = this.position() indices.forEach { - rbsp.put(this, previous, it + 2 - previous) - previous = it + 3 // skip emulation_prevention_three_byte + rbsp.put(this, 0, it + 2 - this.position()) + this.position(this.position() + 1) // skip emulation_prevention_three_byte } - rbsp.put(this, previous, this.limit() - previous) + rbsp.put(this, 0, this.limit() - this.position()) - rbsp.limit(rbsp.position()) rbsp.rewind() return rbsp } /** * Remove all [prefixes] from [ByteBuffer] whatever their order. - * It slices [ByteBuffer] so it does not copy data. + * It moves the [position] of the [ByteBuffer]. * * Once a prefix is found, it is removed from the [prefixes] list. * @@ -324,7 +383,7 @@ fun ByteBuffer.removePrefixes(prefixes: List): ByteBuffer { } } - return this.slice().order(this.order()) + return this } /** @@ -343,4 +402,21 @@ val ByteBuffer.isAvcc: Boolean get() { val size = this.getInt(0) return size == (this.remaining() - 4) - } \ No newline at end of file + } + +/** + * Slices the buffer from [from] position to [to] position. + * + * @param from start position + * @param to end position + */ +fun ByteBuffer.slice(from: Int, to: Int): ByteBuffer { + val currentPosition = this.position() + val currentLimit = this.limit() + this.position(from) + this.limit(to) + val newBuffer = this.slice() + this.position(currentPosition) + this.limit(currentLimit) + return newBuffer +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/Extensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/Extensions.kt index 5d5fba851..9f59bc79a 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/Extensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/Extensions.kt @@ -43,16 +43,16 @@ internal fun Any.numOfBits(): Int { } } -internal fun > T.clamp(min: T, max: T): T { - return if (max >= min) { - if (this < min) min else if (this > max) max else this - } else { - if (this < max) max else if (this > min) min else this +internal fun > T.coerceIn(min: T, max: T): T { + return when { + this < min -> min + this > max -> max + else -> this } } -internal fun > T.clamp(range: Range) = - this.clamp(range.lower, range.upper) +internal fun > T.coerceIn(range: Range) = + this.coerceIn(range.lower, range.upper) internal val PointF.isNormalized: Boolean get() = x in 0f..1f && y in 0f..1f diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt index ce748def0..f3036b9b8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ByteBufferPool.kt @@ -87,4 +87,4 @@ class ByteBufferPool(private val isDirect: Boolean) : IBufferPool, C buffers.clear() } } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt new file mode 100644 index 000000000..3c94714f5 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/FramePool.kt @@ -0,0 +1,132 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.data.Extra +import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.WithClosable +import java.nio.ByteBuffer + + +/** + * A pool of [MutableFrame]. + */ +internal class FramePool : IClearableObjectPool { + private val pool = ObjectPoolImpl() + + fun get( + rawBuffer: ByteBuffer, + ptsInUs: Long, + dtsInUs: Long?, + isKeyFrame: Boolean, + extra: Extra?, + format: MediaFormat, + onClosed: (Frame) -> Unit = {} + ): Frame { + val frame = pool.get() + + val onClosedHook = { frame: MutableFrame -> + onClosed(frame) + pool.put(frame) + } + + return if (frame != null) { + frame.rawBuffer = rawBuffer + frame.ptsInUs = ptsInUs + frame.dtsInUs = dtsInUs + frame.isKeyFrame = isKeyFrame + frame.extra = extra + frame.format = format + frame.onClosed = onClosedHook + frame + } else { + MutableFrame( + rawBuffer = rawBuffer, + ptsInUs = ptsInUs, + dtsInUs = dtsInUs, + isKeyFrame = isKeyFrame, + extra = extra, + format = format, + onClosed = onClosedHook + ) + } + } + + override fun clear() = pool.clear() + + override fun close() = pool.close() + + /** + * A mutable [Frame] internal representation. + * + * The purpose is to get reusable [Frame] + */ + private data class MutableFrame( + /** + * Contains an audio or video frame data. + */ + override var rawBuffer: ByteBuffer, + + /** + * Presentation timestamp in µs + */ + override var ptsInUs: Long, + + /** + * Decoded timestamp in µs (not used). + */ + override var dtsInUs: Long?, + + /** + * `true` if frame is a key frame (I-frame for AVC/HEVC and audio frames) + */ + override var isKeyFrame: Boolean, + + /** + * Contains csd buffers for key frames and audio frames only. + * Could be (SPS, PPS, VPS, etc.) for key video frames, null for non-key video frames. + * ESDS for AAC frames,... + */ + override var extra: Extra?, + + /** + * Contains frame format.. + * TODO: to remove + */ + override var format: MediaFormat, + + /** + * A callback to call when frame is closed. + */ + override var onClosed: (MutableFrame) -> Unit = {} + ) : Frame, WithClosable { + override fun close() { + try { + onClosed(this) + } catch (_: Throwable) { + // Nothing to do + } + } + } + + companion object { + /** + * The default frame pool. + */ + internal val default by lazy { FramePool() } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IFrameFactory.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IFrameFactory.kt deleted file mode 100644 index 759419483..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/IFrameFactory.kt +++ /dev/null @@ -1,31 +0,0 @@ -package io.github.thibaultbee.streampack.core.elements.utils.pool - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame - -interface IReadOnlyRawFrameFactory { - /** - * Creates a [RawFrame]. - * - * The returned frame must be released by calling [RawFrame.close] when it is not used anymore. - * - * @param bufferSize the buffer size - * @param timestampInUs the frame timestamp in µs - * @return a frame - */ - fun create(bufferSize: Int, timestampInUs: Long): RawFrame -} - -/** - * A pool of frames. - */ -interface IRawFrameFactory : IReadOnlyRawFrameFactory { - /** - * Clears the factory. - */ - fun clear() - - /** - * Closes the factory. - */ - fun close() -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPoolImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPoolImpl.kt new file mode 100644 index 000000000..7817192dd --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/ObjectPoolImpl.kt @@ -0,0 +1,97 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import java.io.Closeable +import java.util.ArrayDeque +import java.util.concurrent.atomic.AtomicBoolean + +interface IClearableObjectPool : Closeable { + fun clear() +} + +interface IObjectPool : IClearableObjectPool { + fun put(any: T) +} + +/** + * A pool of objects. + * + * The pool is thread-safe. + * + * The implementation is required to add a `get` methods. + * + * @param T the type of object to pool + */ +internal open class ObjectPoolImpl : IObjectPool { + private val pool = ArrayDeque() + + private val isClosed = AtomicBoolean(false) + + fun get(): T? { + if (isClosed.get()) { + throw IllegalStateException("ObjectPool is closed") + } + + return synchronized(pool) { + if (!pool.isEmpty()) { + pool.removeFirst() + } else { + null + } + } + } + + /** + * Puts an object in the pool. + * + * @param any the object to put + */ + override fun put(any: T) { + if (isClosed.get()) { + throw IllegalStateException("ObjectPool is closed") + } + synchronized(pool) { + pool.addLast(any) + } + } + + /** + * Clears the pool. + */ + override fun clear() { + if (isClosed.get()) { + return + } + synchronized(pool) { + pool.clear() + } + } + + /** + * Closes the pool. + * + * After a pool is closed, it cannot be used anymore. + */ + override fun close() { + if (isClosed.getAndSet(true)) { + return + } + synchronized(pool) { + pool.clear() + } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt deleted file mode 100644 index dc4b3032b..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFrameFactory.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2025 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.elements.utils.pool - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.logger.Logger - -/** - * A factory to create [RawFrame]. - */ -fun RawFrameFactory(isDirect: Boolean): RawFrameFactory { - return RawFrameFactory(ByteBufferPool(isDirect)) -} - -/** - * A factory to create [RawFrame]. - */ -class RawFrameFactory(private val bufferPool: ByteBufferPool) : IRawFrameFactory { - override fun create(bufferSize: Int, timestampInUs: Long): RawFrame { - return RawFrame(bufferPool.get(bufferSize), timestampInUs) { rawFrame -> - try { - bufferPool.put(rawFrame.rawBuffer) - } catch (t: Throwable) { - Logger.w(TAG, "Error while putting buffer in pool: $t") - } - } - } - - override fun clear() { - bufferPool.clear() - } - - override fun close() { - bufferPool.close() - } - - companion object { - private const val TAG = "RawFramePool" - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt new file mode 100644 index 000000000..87e09b2e8 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/elements/utils/pool/RawFramePool.kt @@ -0,0 +1,95 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.elements.utils.pool + +import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.WithClosable +import java.nio.ByteBuffer + + +/** + * A pool of [MutableRawFrame]. + */ +internal class RawFramePool : IClearableObjectPool { + private val pool = ObjectPoolImpl() + + fun get( + rawBuffer: ByteBuffer, + timestampInUs: Long, + onClosed: (RawFrame) -> Unit = {} + ): RawFrame { + val frame = pool.get() + + val onClosedHook = { frame: MutableRawFrame -> + onClosed(frame) + pool.put(frame) + } + + return if (frame != null) { + frame.rawBuffer = rawBuffer + frame.timestampInUs = timestampInUs + frame.onClosed = onClosedHook + frame + } else { + MutableRawFrame( + rawBuffer = rawBuffer, + timestampInUs = timestampInUs, + onClosed = onClosedHook + ) + } + } + + override fun clear() = pool.clear() + + override fun close() = pool.close() + + /** + * A mutable [RawFrame] internal representation. + * + * The purpose is to get reusable [RawFrame] + */ + private data class MutableRawFrame( + /** + * Contains an audio or video frame data. + */ + override var rawBuffer: ByteBuffer, + + /** + * Presentation timestamp in µs + */ + override var timestampInUs: Long, + /** + * A callback to call when frame is closed. + */ + override var onClosed: (MutableRawFrame) -> Unit = {} + ) : RawFrame, WithClosable { + override fun close() { + try { + onClosed(this) + } catch (_: Throwable) { + // Nothing to do + } + } + } + + + companion object { + /** + * The default frame pool. + */ + internal val default by lazy { RawFramePool() } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt index 60a476805..80ae9b9e4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/ISource.kt @@ -36,7 +36,7 @@ interface IWithAudioSource { /** * The audio input to access to advanced settings. */ - val audioInput: IAudioInput? + val audioInput: IAudioInput /** * Sets a new audio source. @@ -44,8 +44,7 @@ interface IWithAudioSource { * @param audioSourceFactory The new audio source factory. */ suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) { - val audio = requireNotNull(audioInput) { "Audio input is not available" } - audio.setSource(audioSourceFactory) + audioInput.setSource(audioSourceFactory) } } @@ -63,9 +62,9 @@ interface IWithVideoRotation { */ interface IWithVideoSource { /** - * The audio input to access to advanced settings. + * The video input to access to advanced settings. */ - val videoInput: IVideoInput? + val videoInput: IVideoInput /** * Sets the video source. @@ -73,8 +72,7 @@ interface IWithVideoSource { * The previous video source will be released unless its preview is still running. */ suspend fun setVideoSource(videoSourceFactory: IVideoSourceInternal.Factory) { - val video = requireNotNull(videoInput) { "Video input is not available" } - video.setSource(videoSourceFactory) + videoInput.setSource(videoSourceFactory) } } @@ -87,8 +85,7 @@ interface IWithVideoSource { */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.setCameraId(cameraId: String) { - val video = requireNotNull(videoInput) { "Video input is not available" } - video.setSource(CameraSourceFactory(cameraId)) + videoInput.setSource(CameraSourceFactory(cameraId)) } @@ -96,7 +93,7 @@ suspend fun IWithVideoSource.setCameraId(cameraId: String) { * Whether the video source has a preview. */ val IWithVideoSource.isPreviewable: Boolean - get() = videoInput?.sourceFlow?.value is IPreviewableSource + get() = videoInput.sourceFlow?.value is IPreviewableSource /** * Sets the preview surface. @@ -105,7 +102,7 @@ val IWithVideoSource.isPreviewable: Boolean * @throws [IllegalStateException] if the video source is not previewable */ suspend fun IWithVideoSource.setPreview(surface: Surface) { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.setPreview(surface) + (videoInput.sourceFlow.value as? IPreviewableSource)?.setPreview(surface) ?: throw IllegalStateException("Video source is not previewable") } @@ -142,7 +139,7 @@ suspend fun IWithVideoSource.setPreview(textureView: TextureView) = * @throws [IllegalStateException] if the video source is not previewable */ suspend fun IWithVideoSource.startPreview() { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.startPreview() + (videoInput.sourceFlow.value as? IPreviewableSource)?.startPreview() ?: throw IllegalStateException("Video source is not previewable") } @@ -154,7 +151,7 @@ suspend fun IWithVideoSource.startPreview() { * @see [IWithVideoSource.stopPreview] */ suspend fun IWithVideoSource.startPreview(surface: Surface) { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.startPreview(surface) + (videoInput.sourceFlow.value as? IPreviewableSource)?.startPreview(surface) ?: throw IllegalStateException("Video source is not previewable") } @@ -195,5 +192,5 @@ suspend fun IWithVideoSource.startPreview(textureView: TextureView) = * Stops video preview. */ suspend fun IWithVideoSource.stopPreview() { - (videoInput?.sourceFlow?.value as? IPreviewableSource)?.stopPreview() + (videoInput.sourceFlow.value as? IPreviewableSource)?.stopPreview() } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/IStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/IStreamer.kt index a296a6c3d..b875931ac 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/IStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/interfaces/IStreamer.kt @@ -19,6 +19,8 @@ import android.net.Uri import androidx.core.net.toUri import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.runBlocking @@ -59,11 +61,13 @@ interface IStreamer { /** * Clean and reset the streamer synchronously. * + * @param dispatcher The coroutine dispatcher to use * @see [IStreamer.release] */ -fun IStreamer.releaseBlocking() = runBlocking { - release() -} +fun IStreamer.releaseBlocking(dispatcher: CoroutineDispatcher = Dispatchers.Default) = + runBlocking(dispatcher) { + release() + } /** * An interface for a component that can be closed. diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt index bb745019c..00ffea90c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/StreamerPipeline.kt @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.core.pipelines import android.content.Context import io.github.thibaultbee.streampack.core.elements.data.RawFrame +import io.github.thibaultbee.streampack.core.elements.data.copy import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory @@ -70,7 +71,6 @@ import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.io.Closeable import java.util.concurrent.atomic.AtomicBoolean /** @@ -79,7 +79,7 @@ import java.util.concurrent.atomic.AtomicBoolean * @param context the application context * @param withAudio whether the streamer has audio. It will create necessary audio components. * @param withVideo whether the streamer has video. It will create necessary video components. - * @param audioOutputMode the audio output mode. It can be [AudioOutputMode.PUSH] or [AudioOutputMode.CALLBACK]. Only use [AudioOutputMode.CALLBACK] when you have a single output and its implements [IAudioCallbackPipelineOutputInternal]. By default, it is [AudioOutputMode.PUSH]. + * @param audioInputMode the audio output mode. It can be [AudioInputMode.PUSH] or [AudioInputMode.CALLBACK]. Only use [AudioInputMode.CALLBACK] when you have a single output and its implements [IAudioCallbackPipelineOutputInternal]. By default, it is [AudioInputMode.PUSH]. * @param surfaceProcessorFactory the factory to create the surface processor * @param dispatcherProvider the coroutine dispatcher */ @@ -87,7 +87,7 @@ open class StreamerPipeline( protected val context: Context, val withAudio: Boolean = true, val withVideo: Boolean = true, - private val audioOutputMode: AudioOutputMode = AudioOutputMode.PUSH, + private val audioInputMode: AudioInputMode = AudioInputMode.PUSH, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), protected val dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : IWithVideoSource, IWithVideoRotation, IWithAudioSource, IStreamer { @@ -100,14 +100,14 @@ open class StreamerPipeline( // INPUTS private val inputMutex = Mutex() private val _audioInput = if (withAudio) { - when (audioOutputMode) { - AudioOutputMode.PUSH -> AudioInput( + when (audioInputMode) { + AudioInputMode.PUSH -> AudioInput( context, AudioInput.PushConfig(::queueAudioFrame), dispatcherProvider ) - AudioOutputMode.CALLBACK -> AudioInput( + AudioInputMode.CALLBACK -> AudioInput( context, AudioInput.CallbackConfig(), dispatcherProvider @@ -116,7 +116,8 @@ open class StreamerPipeline( } else { null } - override val audioInput: IAudioInput? = _audioInput + override val audioInput: IAudioInput + get() = requireNotNull(_audioInput) { "Audio input is not available" } private val _videoInput = if (withVideo) { VideoInput(context, surfaceProcessorFactory, dispatcherProvider) { @@ -126,7 +127,8 @@ open class StreamerPipeline( null } - override val videoInput: IVideoInput? = _videoInput + override val videoInput: IVideoInput + get() = requireNotNull(_videoInput) { "Video input is not available" } private val _isStreamingFlow = MutableStateFlow(false) @@ -246,25 +248,17 @@ open class StreamerPipeline( Logger.e(TAG, "Error while queueing audio frame to output: $t") } } else { - // Hook to close frame when all outputs have processed it - var numOfClosed = 0 - val onClosed = { frame: Closeable -> - numOfClosed++ - if (numOfClosed == audioStreamingOutput.size) { - frame.close() - } - } audioStreamingOutput.forEachIndexed { index, output -> try { output.queueAudioFrame( - frame.copy( - rawBuffer = if (index == audioStreamingOutput.lastIndex) { - frame.rawBuffer - } else { - frame.rawBuffer.duplicate() - }, - onClosed = onClosed - ) + if (index == audioStreamingOutput.lastIndex) { + frame + } else { + frame.copy( + rawBuffer = + frame.rawBuffer.duplicate() + ) + } ) } catch (t: Throwable) { Logger.e(TAG, "Error while queueing audio frame to output $output: $t") @@ -492,7 +486,7 @@ open class StreamerPipeline( if (output is IAudioPipelineOutputInternal) { if (withAudio) { val audioInput = requireNotNull(_audioInput) { "Audio input is not set" } - if (audioOutputMode == AudioOutputMode.CALLBACK) { + if (audioInputMode == AudioInputMode.CALLBACK) { require(output is IAudioCallbackPipelineOutputInternal) { "Output $output must be an audio callback output" } @@ -1049,7 +1043,7 @@ open class StreamerPipeline( /** * Audio output mode. */ - enum class AudioOutputMode { + enum class AudioInputMode { /** * The audio is pushed to the output. */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt index 05812e4d8..0ec5ade18 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt @@ -24,10 +24,12 @@ import io.github.thibaultbee.streampack.core.elements.processing.RawFramePullPus import io.github.thibaultbee.streampack.core.elements.processing.audio.AudioFrameProcessor import io.github.thibaultbee.streampack.core.elements.processing.audio.IAudioFrameProcessor import io.github.thibaultbee.streampack.core.elements.sources.audio.AudioSourceConfig +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioFrameSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.utils.ConflatedJob -import io.github.thibaultbee.streampack.core.elements.utils.pool.IRawFrameFactory +import io.github.thibaultbee.streampack.core.elements.utils.pool.ByteBufferPool +import io.github.thibaultbee.streampack.core.elements.utils.pool.RawFramePool import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Companion.THREAD_NAME_AUDIO_PREPROCESSING import io.github.thibaultbee.streampack.core.pipelines.IAudioDispatcherProvider @@ -55,6 +57,12 @@ interface IAudioInput { */ val isStreamingFlow: StateFlow + /** + * Whether the audio input is capturing. + * Capturing means the audio source and processor are running but frames might not be pushed to the encoder. + */ + val isCapturingFlow: StateFlow + /** * The audio source. * It allows access to advanced audio settings. @@ -79,6 +87,16 @@ interface IAudioInput { */ suspend fun setSource(audioSourceFactory: IAudioSourceInternal.Factory) + /** + * Starts the audio capture for dry-mode (does not push to encoders). + */ + suspend fun startCapture() + + /** + * Stops the audio capture. + */ + suspend fun stopCapture() + /** * Whether the audio input has a configuration. * It is true if the audio source has been configured. @@ -130,16 +148,35 @@ internal class AudioInput( } } + // STATE + /** + * Whether the audio input is streaming. + */ + private val _isStreamingFlow = MutableStateFlow(false) + override val isStreamingFlow = _isStreamingFlow.asStateFlow() + + private val _isCapturingFlow = MutableStateFlow(false) + override val isCapturingFlow = _isCapturingFlow.asStateFlow() + // PROCESSOR + private val bufferPool = ByteBufferPool(true) + /** * The audio processor. */ - private val frameProcessorInternal = AudioFrameProcessor() - override val processor: IAudioFrameProcessor = frameProcessorInternal + private val processorInternal = AudioFrameProcessor(bufferPool, dispatcherProvider.default) + override val processor: IAudioFrameProcessor = processorInternal private val port = if (config is PushConfig) { - PushAudioPort(frameProcessorInternal, config, dispatcherProvider) + val wrappedConfig = PushConfig { frame -> + if (_isStreamingFlow.value) { + config.onFrame(frame) + } else { + frame.close() + } + } + PushAudioPort(processorInternal, wrappedConfig, bufferPool, dispatcherProvider) } else { - CallbackAudioPort(frameProcessorInternal) // No threading needed, called from encoder thread + CallbackAudioPort(processorInternal) // No threading needed, called from encoder thread } // CONFIG @@ -156,13 +193,6 @@ internal class AudioInput( override val withConfig: Boolean get() = sourceConfig != null - // STATE - /** - * Whether the audio input is streaming. - */ - private val _isStreamingFlow = MutableStateFlow(false) - override val isStreamingFlow = _isStreamingFlow.asStateFlow() - /** * Sets a new audio source. * @@ -175,6 +205,10 @@ internal class AudioInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } + val previousAudioSource = sourceInternalFlow.value val isStreaming = previousAudioSource?.isStreamingFlow?.value ?: false @@ -193,22 +227,23 @@ internal class AudioInput( isStreamingJob += coroutineScope.launch { newAudioSource.isStreamingFlow.collect { isStreaming -> - if ((!isStreaming) && isStreamingFlow.value) { + if ((!isStreaming) && (isStreamingFlow.value || isCapturingFlow.value)) { Logger.i(TAG, "Audio source has been stopped.") - stopStream() + try { + stopStream() + } catch (_: Throwable) { + // Nothing to do, input might be already released + } + try { + stopCapture() + } catch (_: Throwable) { + // Nothing to do, input might be already released + } } } } - when (port) { - is PushAudioPort -> { - port.setInput(newAudioSource::getAudioFrame) - } - - is CallbackAudioPort -> { - port.setInput(newAudioSource::fillAudioFrame) - } - } + port.setInput(newAudioSource) // Replace audio source sourceInternalFlow.emit(newAudioSource) @@ -227,11 +262,15 @@ internal class AudioInput( internal suspend fun setSourceConfig(newAudioSourceConfig: AudioSourceConfig) { if (isReleaseRequested.get()) { - throw IllegalStateException("Pipeline is released") + throw IllegalStateException("Input is released") } withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released in lock") + } + if (sourceConfig == newAudioSourceConfig) { Logger.i( TAG, @@ -274,60 +313,132 @@ internal class AudioInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { - val source = requireNotNull(sourceInternalFlow.value) { - "Audio source is not set yet" + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released in lock") } if (isStreamingFlow.value) { Logger.w(TAG, "Stream is already running") return@withContext } - if (!withConfig) { - Logger.w(TAG, "Audio source configuration is not set yet") + start("startStream") + _isStreamingFlow.emit(true) + } + } + } + + override suspend fun startCapture() { + require(port is PushAudioPort) { "Audio capture is not supported in this mode: ${port::class.java}" } + + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } + + withContext(dispatcherProvider.default) { + sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released in lock") } - source.startStream() - try { - port.startStream() - } catch (t: Throwable) { - Logger.w( - TAG, - "startStream: Can't start audio processor: ${t.message}" - ) - source.stopStream() - throw t + if (isCapturingFlow.value) { + Logger.w(TAG, "Capture is already running") + return@withContext } - _isStreamingFlow.emit(true) + start("startCapture") + _isCapturingFlow.emit(true) } } } + private suspend fun start(caller: String) { + val source = requireNotNull(sourceInternalFlow.value) { + "Audio source is not set yet" + } + if (!withConfig) { + Logger.w(TAG, "Audio source configuration is not set yet") + } + source.startStream() + try { + port.startStream() + } catch (t: Throwable) { + Logger.w( + TAG, + "$caller: Can't start audio processor: ${t.message}" + ) + source.stopStream() + throw t + } + } + internal suspend fun stopStream() { if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") + Logger.w(TAG, "Input is released") + return } withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + Logger.w(TAG, "Input is released in lock") + return@withContext + } + if (!isStreamingFlow.value) { + Logger.w(TAG, "Stream is already stopped") + return@withContext + } _isStreamingFlow.emit(false) - try { - port.stopStream() - } catch (t: Throwable) { - Logger.w( - TAG, - "stopStream: Can't stop audio processor: ${t.message}" - ) + if (isCapturingFlow.value) { + Logger.w(TAG, "Stopping capture before stopping stream") + return@withContext } - try { - sourceInternalFlow.value?.stopStream() - } catch (t: Throwable) { - Logger.w( - TAG, - "stopStream: Can't stop audio source: ${t.message}" - ) + stopUnsafe("stopStream") + } + } + } + + override suspend fun stopCapture() { + if (isReleaseRequested.get()) { + Logger.w(TAG, "Input is released") + return + } + + withContext(dispatcherProvider.default) { + sourceMutex.withLock { + if (isReleaseRequested.get()) { + Logger.w(TAG, "Input is released in lock") + return@withContext + } + if (!isCapturingFlow.value) { + Logger.w(TAG, "Capture is already stopped") + return@withContext + } + _isCapturingFlow.emit(false) + if (isStreamingFlow.value) { + Logger.w(TAG, "Stopping stream before stopping capture") + return@withContext } + stopUnsafe("stopCapture") } } } + private suspend fun stopUnsafe(caller: String) { + try { + port.stopStream() + } catch (t: Throwable) { + Logger.w( + TAG, + "$caller: Can't stop audio processor: ${t.message}" + ) + } + try { + sourceInternalFlow.value?.stopStream() + } catch (t: Throwable) { + Logger.w( + TAG, + "$caller: Can't stop audio source: ${t.message}" + ) + } + } + internal suspend fun release() { if (isReleaseRequested.getAndSet(true)) { Logger.w(TAG, "Already released") @@ -362,6 +473,15 @@ internal class AudioInput( ) } + try { + processorInternal.close() + } catch (t: Throwable) { + Logger.w( + TAG, + "release: Can't close audio processor: ${t.message}" + ) + } + isStreamingJob.cancel() } coroutineScope.coroutineContext.cancelChildren() @@ -380,27 +500,29 @@ internal class AudioInput( internal class CallbackConfig : Config() } -private sealed interface IAudioPort : Streamable, Releasable { - suspend fun setInput(getFrame: T) +private sealed interface IAudioPort : Streamable, Releasable { + suspend fun setInput(source: IAudioFrameSourceInternal) suspend fun removeInput() } private class PushAudioPort( audioFrameProcessor: AudioFrameProcessor, config: PushConfig, + bufferPool: ByteBufferPool, dispatcherProvider: IAudioDispatcherProvider -) : IAudioPort<(frameFactory: IRawFrameFactory) -> RawFrame> { +) : IAudioPort { private val audioPullPush = RawFramePullPush( audioFrameProcessor, config.onFrame, + bufferPool, dispatcherProvider.createAudioDispatcher( 1, THREAD_NAME_AUDIO_PREPROCESSING ) ) - override suspend fun setInput(getFrame: (frameFactory: IRawFrameFactory) -> RawFrame) { - audioPullPush.setInput(getFrame) + override suspend fun setInput(source: IAudioFrameSourceInternal) { + audioPullPush.setInput(source) } override suspend fun removeInput() { @@ -421,32 +543,35 @@ private class PushAudioPort( } private class CallbackAudioPort(private val audioFrameProcessor: AudioFrameProcessor) : - IAudioPort<(frame: RawFrame) -> RawFrame> { - private var getFrame: ((frame: RawFrame) -> RawFrame)? = null + IAudioPort { private val mutex = Mutex() + private val pool = RawFramePool() + + private var source: IAudioFrameSourceInternal? = null var audioFrameRequestedListener: OnFrameRequestedListener = object : OnFrameRequestedListener { override suspend fun onFrameRequested(buffer: ByteBuffer): RawFrame { val frame = mutex.withLock { - val getFrame = requireNotNull(getFrame) { + val source = requireNotNull(source) { "Audio frame requested listener is not set yet" } - getFrame(RawFrame(buffer, 0)) + val timestampInUs = source.fillAudioFrame(buffer) + pool.get(buffer, timestampInUs) } - return audioFrameProcessor.processFrame(frame) + return audioFrameProcessor.process(frame) } } - override suspend fun setInput(getFrame: (frame: RawFrame) -> RawFrame) { + override suspend fun setInput(source: IAudioFrameSourceInternal) { mutex.withLock { - this.getFrame = getFrame + this.source = source } } override suspend fun removeInput() { mutex.withLock { - this.getFrame = null + this.source = null } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt index 54e7bfa3b..091bf0167 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt @@ -19,6 +19,7 @@ import android.content.Context import android.graphics.Bitmap import android.view.Surface import androidx.annotation.IntRange +import io.github.thibaultbee.streampack.core.elements.interfaces.ISnapshotable import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput @@ -45,18 +46,13 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.io.File -import java.io.FileOutputStream -import java.io.OutputStream import java.util.concurrent.atomic.AtomicBoolean -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine /** * The public interface for the video input. * It provides access to the video source, the video processor, and the streaming state. */ -interface IVideoInput { +interface IVideoInput : ISnapshotable { /** * Whether the video input is streaming. @@ -85,64 +81,6 @@ interface IVideoInput { * The video processor for adding effects to the video frames. */ val processor: ISurfaceProcessorInternal - - /** - * Takes a snapshot of the current video frame. - * - * The snapshot is returned as a [Bitmap]. - * - * @param rotationDegrees The rotation to apply to the snapshot, in degrees. 0 means no rotation. - * @return The snapshot as a [Bitmap]. - */ - suspend fun takeSnapshot(@IntRange(from = 0, to = 359) rotationDegrees: Int = 0): Bitmap -} - -/** - * Takes a JPEG snapshot of the current video frame. - * - * The snapshot is saved to the specified file. - * - * @param filePathString The path of the file to save the snapshot to. - * @param quality The quality of the JPEG, from 0 to 100. - * @param rotationDegrees The rotation to apply to the snapshot, in degrees. - */ -suspend fun IVideoInput.takeJpegSnapshot( - filePathString: String, - @IntRange(from = 0, to = 100) quality: Int = 100, - @IntRange(from = 0, to = 359) rotationDegrees: Int = 0 -) = takeJpegSnapshot(FileOutputStream(filePathString), quality, rotationDegrees) - - -/** - * Takes a JPEG snapshot of the current video frame. - * - * The snapshot is saved to the specified file. - * - * @param file The file to save the snapshot to. - * @param quality The quality of the JPEG, from 0 to 100. - * @param rotationDegrees The rotation to apply to the snapshot, in degrees. - */ -suspend fun IVideoInput.takeJpegSnapshot( - file: File, - @IntRange(from = 0, to = 100) quality: Int = 100, - @IntRange(from = 0, to = 359) rotationDegrees: Int = 0 -) = takeJpegSnapshot(FileOutputStream(file), quality, rotationDegrees) - -/** - * Takes a snapshot of the current video frame. - * - * The snapshot is saved as a JPEG to the specified output stream. - * @param outputStream The output stream to save the snapshot to. - * @param quality The quality of the JPEG, from 0 to 100. - * @param rotationDegrees The rotation to apply to the snapshot, in degrees. - */ -suspend fun IVideoInput.takeJpegSnapshot( - outputStream: OutputStream, - @IntRange(from = 0, to = 100) quality: Int = 100, - @IntRange(from = 0, to = 359) rotationDegrees: Int = 0 -) { - val bitmap = takeSnapshot(rotationDegrees) - bitmap.compress(Bitmap.CompressFormat.JPEG, quality, outputStream) } /** @@ -213,6 +151,9 @@ internal class VideoInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } val previousVideoSource = sourceInternalFlow.value val isStreaming = previousVideoSource?.isStreamingFlow?.value ?: false @@ -300,7 +241,11 @@ internal class VideoInput( newVideoSource.isStreamingFlow.collect { isStreaming -> if ((!isStreaming) && isStreamingFlow.value) { Logger.i(TAG, "Video source has been stopped.") - stopStream() + try { + stopStream() + } catch (_: Throwable) { + // Nothing to do, input might be already released + } } } } @@ -351,6 +296,9 @@ internal class VideoInput( withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } if (sourceConfig == newVideoSourceConfig) { Logger.i( TAG, @@ -436,6 +384,13 @@ internal class VideoInput( return newSurfaceProcessor } + /** + * Takes a snapshot of the current video frame. + * + * It starts the video source if needed. + * + * The snapshot is returned as a [Bitmap]. + */ override suspend fun takeSnapshot( @IntRange( from = 0, @@ -445,14 +400,16 @@ internal class VideoInput( if (isReleaseRequested.get()) { throw IllegalStateException("Input is released") } - return startStreamForBlock { - suspendCoroutine { continuation -> - val listener = processor.snapshot(rotationDegrees) - try { - val bitmap = listener.get() - continuation.resume(bitmap) - } catch (e: Exception) { - continuation.resumeWith(Result.failure(e)) + return withContext(dispatcherProvider.default) { + sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } + val processor = processor as? ISnapshotable? + ?: throw IllegalStateException("Processor is not a snapshotable") + + startStreamForBlockUnsafe { + processor.takeSnapshot(rotationDegrees) } } } @@ -499,6 +456,9 @@ internal class VideoInput( } sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } val sourceConfig = sourceConfig val infoProvider = source?.infoProviderFlow?.value if ((sourceConfig == null) || (infoProvider == null)) { @@ -521,21 +481,17 @@ internal class VideoInput( } internal fun addOutputSurfaceUnsafe(output: ISurfaceOutput) { - if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") + try { + processor.addOutputSurface(output) + } catch (t: Throwable) { + Logger.w(TAG, "addOutputSurface: Can't add output surface: ${t.message}") } - - processor.addOutputSurface(output) } private suspend fun addOutputSurfacesUnsafe( infoProvider: ISourceInfoProvider? = source?.infoProviderFlow?.value, videoSourceConfig: VideoSourceConfig? = sourceConfig ) { - if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") - } - if ((videoSourceConfig == null) || (infoProvider == null)) { Logger.w( TAG, @@ -562,10 +518,6 @@ internal class VideoInput( infoProvider: ISourceInfoProvider? = source?.infoProviderFlow?.value, videoSourceConfig: VideoSourceConfig? = sourceConfig ) { - if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") - } - processor.removeAllOutputSurfaces() addOutputSurfacesUnsafe(infoProvider, videoSourceConfig) } @@ -579,6 +531,9 @@ internal class VideoInput( } sourceMutex.withLock { + if (isReleaseRequested.get()) { + throw IllegalStateException("Input is released") + } updateOutputSurfacesUnsafe(infoProvider, videoSourceConfig) } } @@ -605,23 +560,19 @@ internal class VideoInput( * * If the stream was already running, it will not be stopped after the block. */ - private suspend fun startStreamForBlock(block: suspend () -> T): T { + private suspend fun startStreamForBlockUnsafe(block: suspend () -> T): T { if (isReleaseRequested.get()) { throw IllegalStateException("Input is released") } - return withContext(dispatcherProvider.default) { - sourceMutex.withLock { - val wasStreaming = isStreamingFlow.value - if (!wasStreaming) { - startStreamUnsafe() - } - try { - block() - } finally { - if (!wasStreaming) { - stopStreamUnsafe() - } - } + val wasStreaming = isStreamingFlow.value + if (!wasStreaming) { + startStreamUnsafe() + } + return try { + block() + } finally { + if (!wasStreaming) { + stopStreamUnsafe() } } } @@ -659,10 +610,15 @@ internal class VideoInput( suspend fun stopStream() { if (isReleaseRequested.get()) { - throw IllegalStateException("Input is released") + Logger.w(TAG, "Input is released") + return } withContext(dispatcherProvider.default) { sourceMutex.withLock { + if (isReleaseRequested.get()) { + Logger.w(TAG, "Input is released") + return@withContext + } _isStreamingFlow.emit(false) stopStreamUnsafe() } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt index cedbad83b..e6749ee68 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/EncodingPipelineOutput.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.core.pipelines.outputs.encoding import android.content.Context import android.view.Surface import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.data.RawFrame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig @@ -50,6 +50,7 @@ import io.github.thibaultbee.streampack.core.pipelines.outputs.SurfaceDescriptor import io.github.thibaultbee.streampack.core.pipelines.outputs.isStreaming import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.consumeEach @@ -89,7 +90,7 @@ internal class EncodingPipelineOutput( ) : IConfigurableAudioVideoEncodingPipelineOutput, IEncodingPipelineOutputInternal, IVideoSurfacePipelineOutputInternal, IAudioSyncPipelineOutputInternal, IAudioCallbackPipelineOutputInternal { - private val coroutineScope = CoroutineScope(dispatcherProvider.default) + private val coroutineScope = CoroutineScope(dispatcherProvider.default + SupervisorJob()) private val coroutineDispatcher = dispatcherProvider.default /** @@ -101,6 +102,28 @@ internal class EncodingPipelineOutput( private val isReleaseRequested = AtomicBoolean(false) private var bitrateRegulatorController: IBitrateRegulatorController? = null + override var bitrateRegulatorControllerFactory: IBitrateRegulatorController.Factory? = null + set(value) { + field = value + if (isReleaseRequested.get()) { + throw IllegalStateException("Output is released") + } + bitrateRegulatorController?.stop() + bitrateRegulatorController = + value?.newBitrateRegulatorController(this, dispatcherProvider.default) + + if (bitrateRegulatorController != null) { + Logger.d( + TAG, + "Bitrate regulator controller added: ${bitrateRegulatorController!!.javaClass.simpleName}" + ) + if (isStreaming) { + bitrateRegulatorController?.start() + } + } else { + Logger.d(TAG, "Bitrate regulator controller removed") + } + } private var audioStreamId: Int? = null private var videoStreamId: Int? = null @@ -239,7 +262,7 @@ internal class EncodingPipelineOutput( } override val outputChannel = - Channel(Channel.UNLIMITED, onUndeliveredElement = { + Channel(Channel.UNLIMITED, onUndeliveredElement = { it.close() }) } @@ -250,19 +273,29 @@ internal class EncodingPipelineOutput( } override val outputChannel = - Channel(Channel.UNLIMITED, onUndeliveredElement = { + Channel(Channel.UNLIMITED, onUndeliveredElement = { it.close() }) } init { + coroutineScope.launch { + isStreamingFlow.collect { isStreaming -> + if (isStreaming) { + bitrateRegulatorController?.start() + } else { + bitrateRegulatorController?.stop() + } + } + } + if (withAudio) { coroutineScope.launch(audioOutputDispatcher) { // Audio - audioEncoderListener.outputChannel.consumeEach { closeableFrame -> + audioEncoderListener.outputChannel.consumeEach { frame -> try { audioStreamId?.let { - endpointInternal.write(closeableFrame, it) + endpointInternal.write(frame, it) } ?: Logger.w(TAG, "Audio frame received but audio stream is not set") } catch (t: Throwable) { onInternalError(t) @@ -273,10 +306,10 @@ internal class EncodingPipelineOutput( if (withVideo) { coroutineScope.launch(videoOutputDispatcher) { // Video - videoEncoderListener.outputChannel.consumeEach { closeableFrame -> + videoEncoderListener.outputChannel.consumeEach { frame -> try { videoStreamId?.let { - endpointInternal.write(closeableFrame, it) + endpointInternal.write(frame, it) } ?: Logger.w(TAG, "Video frame received but video stream is not set") } catch (t: Throwable) { onInternalError(t) @@ -397,16 +430,28 @@ internal class EncodingPipelineOutput( } } - override suspend fun queueAudioFrame(frame: RawFrame) = mutex.withLock { - val input = try { - requireNotNull(audioInput) { "Audio input is null" } - } catch (t: Throwable) { + override suspend fun queueAudioFrame(frame: RawFrame) { + /** + * Avoid deadlock when trying to start/stop the stream. + */ + if (!mutex.tryLock()) { + Logger.e(TAG, "queueAudioFrame: not locked") frame.close() - throw t + return + } + try { + val input = try { + requireNotNull(audioInput) { "Audio input is null" } + } catch (t: Throwable) { + frame.close() + throw t + } + input.queueInputFrame( + frame + ) + } finally { + mutex.unlock() } - input.queueInputFrame( - frame - ) } @@ -624,8 +669,6 @@ internal class EncodingPipelineOutput( videoEncoderJob?.join() endpointInternal.startStream() - - bitrateRegulatorController?.start() } catch (t: Throwable) { stopStreamUnsafe() throw t @@ -677,7 +720,11 @@ internal class EncodingPipelineOutput( return } - streamEventListener?.onStopStream() + try { + streamEventListener?.onStopStream() + } catch (t: Throwable) { + Logger.w(TAG, "Can't stop stream: ${t.message}") + } stopStreamElements() @@ -699,12 +746,6 @@ internal class EncodingPipelineOutput( * @see [stopStream] */ private suspend fun stopStreamElements() { - try { - bitrateRegulatorController?.stop() - } catch (t: Throwable) { - Logger.w(TAG, "Can't stop bitrate regulator controller: ${t.message}") - } - // Encoders val audioEncoderJob = audioEncoderInternal?.let { coroutineScope.launch { @@ -809,40 +850,6 @@ internal class EncodingPipelineOutput( coroutineScope.cancel() } - /** - * Adds a bitrate regulator controller. - * - * Limitation: it is only available for SRT for now. - */ - override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) { - if (isReleaseRequested.get()) { - throw IllegalStateException("Output is released") - } - bitrateRegulatorController?.stop() - bitrateRegulatorController = - controllerFactory.newBitrateRegulatorController(this, dispatcherProvider.default) - .apply { - if (isStreaming) { - this.start() - } - Logger.d( - TAG, "Bitrate regulator controller added: ${this.javaClass.simpleName}" - ) - } - } - - /** - * Removes the bitrate regulator controller. - */ - override fun removeBitrateRegulatorController() { - if (isReleaseRequested.get()) { - throw IllegalStateException("Output is released") - } - bitrateRegulatorController?.stop() - bitrateRegulatorController = null - Logger.d(TAG, "Bitrate regulator controller removed") - } - private suspend fun setTargetRotationInternal(@RotationValue newTargetRotation: Int) { if (shouldUpdateRotation(newTargetRotation)) { updateVideoEncoderForTransformation() diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/IEncodingPipelineOutput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/IEncodingPipelineOutput.kt index 9c8b3f247..5af6c9bb9 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/IEncodingPipelineOutput.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/outputs/encoding/IEncodingPipelineOutput.kt @@ -42,12 +42,7 @@ interface IEncodingPipelineOutput : IPipelineOutput, IOpenableStreamer { /** * Adds a bitrate regulator controller to the streamer. */ - fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) - - /** - * Removes the bitrate regulator controller from the streamer. - */ - fun removeBitrateRegulatorController() + var bitrateRegulatorControllerFactory: IBitrateRegulatorController.Factory? } /** @@ -120,4 +115,18 @@ interface IConfigurableAudioVideoEncodingPipelineOutput : internal interface IEncodingPipelineOutputInternal : IConfigurableAudioVideoEncodingPipelineOutput, IConfigurableAudioPipelineOutputInternal, IConfigurableVideoPipelineOutputInternal, - IPipelineEventOutputInternal \ No newline at end of file + IPipelineEventOutputInternal + +/** + * Adds a bitrate regulator controller to the streamer. + */ +fun IEncodingPipelineOutput.addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) { + this.bitrateRegulatorControllerFactory = controllerFactory +} + +/** + * Removes the bitrate regulator controller from the streamer. + */ +fun IEncodingPipelineOutput.removeBitrateRegulatorController() { + this.bitrateRegulatorControllerFactory = null +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/BitrateRegulator.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/BitrateRegulator.kt index ba7c86fb4..1edb6a28c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/BitrateRegulator.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/BitrateRegulator.kt @@ -16,6 +16,8 @@ package io.github.thibaultbee.streampack.core.regulator import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig +import io.github.thibaultbee.streampack.core.elements.metrics.EndpointMetricsTracker +import io.github.thibaultbee.streampack.core.elements.utils.extensions.coerceIn /** * Abstract class for the bitrate regulation implementation. @@ -23,12 +25,23 @@ import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfi * If you want to implement your custom bitrate regulator, it must inherit from this class. * The bitrate regulator object is created by streamers through the [IBitrateRegulator.Factory]. * + * @param metricsTracker endpoint metrics tracker * @param bitrateRegulatorConfig bitrate regulation configuration * @param onVideoTargetBitrateChange call when you have to change video bitrate * @param onAudioTargetBitrateChange call when you have to change audio bitrate */ abstract class BitrateRegulator( + override val metricsTracker: EndpointMetricsTracker, protected val bitrateRegulatorConfig: BitrateRegulatorConfig, - protected val onVideoTargetBitrateChange: ((Int) -> Unit), - protected val onAudioTargetBitrateChange: ((Int) -> Unit) -) : IBitrateRegulator + onVideoTargetBitrateChange: ((Int) -> Unit), + onAudioTargetBitrateChange: ((Int) -> Unit) +) : IBitrateRegulator { + + protected val onVideoTargetBitrateChange: ((Int) -> Unit) = { + onVideoTargetBitrateChange(it.coerceIn(bitrateRegulatorConfig.videoBitrateRange)) + } + + protected val onAudioTargetBitrateChange: ((Int) -> Unit) = { + onAudioTargetBitrateChange(it.coerceIn(bitrateRegulatorConfig.audioBitrateRange)) + } +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/IBitrateRegulator.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/IBitrateRegulator.kt index 367af6973..2285505ec 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/IBitrateRegulator.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/IBitrateRegulator.kt @@ -16,19 +16,24 @@ package io.github.thibaultbee.streampack.core.regulator import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig +import io.github.thibaultbee.streampack.core.elements.metrics.EndpointMetricsTracker /** * Interface to implement a bitrate regulator. */ interface IBitrateRegulator { /** - * Calls regularly to get new stats + * Tracker for endpoint metrics + */ + val metricsTracker: EndpointMetricsTracker + + /** + * Called regularly to get new metrics * - * @param stats transmission stats * @param currentVideoBitrate current video bitrate target in bits/s. * @param currentAudioBitrate current audio bitrate target in bits/s. */ - fun update(stats: Any, currentVideoBitrate: Int, currentAudioBitrate: Int) + fun update(currentVideoBitrate: Int, currentAudioBitrate: Int) /** * Factory interface you must use to create a [BitrateRegulator] object. @@ -39,12 +44,14 @@ interface IBitrateRegulator { /** * Creates a [BitrateRegulator] object from given parameters * + * @param metricsTracker endpoint metrics tracker * @param bitrateRegulatorConfig bitrate regulation configuration * @param onVideoTargetBitrateChange call when you have to change video bitrate * @param onAudioTargetBitrateChange call when you have to change audio bitrate * @return a [BitrateRegulator] object */ fun newBitrateRegulator( + metricsTracker: EndpointMetricsTracker, bitrateRegulatorConfig: BitrateRegulatorConfig, onVideoTargetBitrateChange: ((Int) -> Unit), onAudioTargetBitrateChange: ((Int) -> Unit) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/SimpleBitrateRegulator.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/SimpleBitrateRegulator.kt new file mode 100644 index 000000000..58108a39d --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/SimpleBitrateRegulator.kt @@ -0,0 +1,107 @@ +package io.github.thibaultbee.streampack.core.regulator + +import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig +import io.github.thibaultbee.streampack.core.elements.metrics.EndpointMetricsTracker +import io.github.thibaultbee.streampack.core.elements.metrics.writtenBitrateInBps +import kotlin.math.max +import kotlin.math.min + +/** + * A [BitrateRegulator] that reduce video bitrate when packets are lost. + * + * @param bitrateRegulatorConfig bitrate regulation configuration + * @param onVideoTargetBitrateChange call when you have to change video bitrate + * @param onAudioTargetBitrateChange call when you have to change audio bitrate + */ +class SimpleBitrateRegulator( + metricsTracker: EndpointMetricsTracker, + bitrateRegulatorConfig: BitrateRegulatorConfig, + onVideoTargetBitrateChange: ((Int) -> Unit), + onAudioTargetBitrateChange: ((Int) -> Unit) +) : BitrateRegulator( + metricsTracker, + bitrateRegulatorConfig, + onVideoTargetBitrateChange, + onAudioTargetBitrateChange +) { + companion object { + private const val MIN_DECREASE_STEP = 100000 // b/s + private const val MAX_INCREASE_STEP = 200000 // b/s + + private const val MAX_PERCENTAGE_DECREASE = 85 // % + private const val MIN_PERCENTAGE_DECREASE = 20 // % + } + + /** + * Called regularly to get new endpoint metrics + * + * @param currentVideoBitrate current video bitrate target in bits/s. + * @param currentAudioBitrate current audio bitrate target in bits/s. + */ + override fun update( + currentVideoBitrate: Int, + currentAudioBitrate: Int + ) { + val metrics = metricsTracker.instant + val packetsLostOrDropped = metrics.packetsWriteDropped + metrics.packetsWriteLost + val writtenBitrate = metrics.writtenBitrateInBps + + if (packetsLostOrDropped > 0) { + // Detected packet dropped or loss - we should reduce the bitrate with multiplicative decrease + // How critical? + val percentageReduction = if (metrics.packetsWritten == 0L) { + MAX_PERCENTAGE_DECREASE + } else { + (packetsLostOrDropped * 100 / metrics.packetsWritten).toInt().coerceIn(MIN_PERCENTAGE_DECREASE, MAX_PERCENTAGE_DECREASE) + } + + // Reduce current bitrate by percentageReduction % + val newVideoBitrate = currentVideoBitrate - max( + currentVideoBitrate * percentageReduction / 100, + MIN_DECREASE_STEP // getting down by 100000 b/s minimum + ) + onVideoTargetBitrateChange(newVideoBitrate) + } else if (currentVideoBitrate < bitrateRegulatorConfig.videoBitrateRange.upper) { + // Only increase if we are successfully sending at near the current target bitrate + // This prevents us from increasing the target when the network is already saturated + if (writtenBitrate > currentVideoBitrate * 0.9) { + // Additive increase + val newVideoBitrate = min( + currentVideoBitrate + MAX_INCREASE_STEP, + bitrateRegulatorConfig.videoBitrateRange.upper + ) + onVideoTargetBitrateChange(newVideoBitrate) + } + } + } + + /** + * Factory interface you must use to create a [SimpleBitrateRegulator] object. + * If you want to create a custom RTMP bitrate regulation implementation, create a factory that + * implements this interface. + */ + class Factory : IBitrateRegulator.Factory { + /** + * Creates a [SimpleBitrateRegulator] object from given parameters + * + * @param metricsTracker endpoint metrics tracker + * @param bitrateRegulatorConfig bitrate regulation configuration + * @param onVideoTargetBitrateChange call when you have to change video bitrate + * @param onAudioTargetBitrateChange call when you have to change audio bitrate + * @return a [SimpleBitrateRegulator] object + */ + override fun newBitrateRegulator( + metricsTracker: EndpointMetricsTracker, + bitrateRegulatorConfig: BitrateRegulatorConfig, + onVideoTargetBitrateChange: ((Int) -> Unit), + onAudioTargetBitrateChange: ((Int) -> Unit) + ): SimpleBitrateRegulator { + return SimpleBitrateRegulator( + metricsTracker, + bitrateRegulatorConfig, + onVideoTargetBitrateChange, + onAudioTargetBitrateChange + ) + } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/BitrateRegulatorController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/BitrateRegulatorController.kt index 2ba195240..7f116dd4d 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/BitrateRegulatorController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/BitrateRegulatorController.kt @@ -17,7 +17,7 @@ package io.github.thibaultbee.streampack.core.regulator.controllers import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder -import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint +import io.github.thibaultbee.streampack.core.elements.metrics.WithEndpointMetrics import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutput import io.github.thibaultbee.streampack.core.regulator.IBitrateRegulator import kotlinx.coroutines.CoroutineDispatcher @@ -27,16 +27,16 @@ import kotlinx.coroutines.CoroutineDispatcher * * @param audioEncoder the audio [IEncoder] * @param videoEncoder the video [IEncoder] - * @param endpoint the [IEndpoint] implementation + * @param metricsProvider the [WithEndpointMetrics] implementation * @param bitrateRegulatorFactory the [IBitrateRegulator.Factory] implementation. Use it to make your own bitrate regulator. * @param bitrateRegulatorConfig bitrate regulator configuration */ abstract class BitrateRegulatorController( - private val audioEncoder: IEncoder?, - private val videoEncoder: IEncoder?, - private val endpoint: IEndpoint, - private val bitrateRegulatorFactory: IBitrateRegulator.Factory, - private val bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig() + protected val audioEncoder: IEncoder?, + protected val videoEncoder: IEncoder?, + protected val metricsProvider: WithEndpointMetrics<*>, + protected val bitrateRegulatorFactory: IBitrateRegulator.Factory, + protected val bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig() ) : IBitrateRegulatorController { abstract class Factory : IBitrateRegulatorController.Factory { /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/BitrateRegulatorControllers.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/BitrateRegulatorControllers.kt new file mode 100644 index 000000000..7bfbacd1d --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/BitrateRegulatorControllers.kt @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.regulator.controllers + +import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig +import io.github.thibaultbee.streampack.core.regulator.IBitrateRegulator +import io.github.thibaultbee.streampack.core.regulator.SimpleBitrateRegulator +import io.github.thibaultbee.streampack.core.regulator.controllers.IntervalBitrateRegulatorController.Companion.DEFAULT_POLLING_TIME +import io.github.thibaultbee.streampack.core.regulator.controllers.IntervalBitrateRegulatorController.Factory +import kotlin.time.Duration + +/** + * A [IntervalBitrateRegulatorController.Factory] for [IBitrateRegulator]. + * + * @param bitrateRegulatorFactory the [IBitrateRegulator.Factory] implementation. Use it to make your own bitrate regulator. + * @param bitrateRegulatorConfig bitrate regulator configuration + * @param pollingTime delay between each call to [IBitrateRegulator.update] + */ +fun intervalBitrateRegulatorControllerFactory( + bitrateRegulatorFactory: IBitrateRegulator.Factory = SimpleBitrateRegulator.Factory(), + bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig(), + pollingTime: Duration = DEFAULT_POLLING_TIME +) = Factory( + bitrateRegulatorFactory, bitrateRegulatorConfig, pollingTime +) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/DummyBitrateRegulatorController.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/IntervalBitrateRegulatorController.kt similarity index 75% rename from core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/DummyBitrateRegulatorController.kt rename to core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/IntervalBitrateRegulatorController.kt index a05a69ad7..6a3f99ca1 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/DummyBitrateRegulatorController.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/regulator/controllers/IntervalBitrateRegulatorController.kt @@ -17,56 +17,61 @@ package io.github.thibaultbee.streampack.core.regulator.controllers import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder -import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint +import io.github.thibaultbee.streampack.core.elements.metrics.WithEndpointMetrics +import io.github.thibaultbee.streampack.core.elements.metrics.createMetricsTracker import io.github.thibaultbee.streampack.core.elements.utils.CoroutineScheduler import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioEncodingPipelineOutput import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutput import io.github.thibaultbee.streampack.core.regulator.IBitrateRegulator import kotlinx.coroutines.CoroutineDispatcher +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds /** - * A [BitrateRegulatorController] implementation that triggers [IBitrateRegulator.update] every [delayTimeInMs]. + * A [BitrateRegulatorController] implementation that triggers [IBitrateRegulator.update] every [pollingTime]. * * @param audioEncoder the audio [IEncoder] * @param videoEncoder the video [IEncoder] - * @param endpoint the [IEndpoint] implementation + * @param metricsProvider the [WithEndpointMetrics] implementation * @param bitrateRegulatorFactory the [IBitrateRegulator.Factory] implementation. Use it to make your own bitrate regulator. * @param bitrateRegulatorConfig bitrate regulator configuration - * @param delayTimeInMs delay between each call to [IBitrateRegulator.update] + * @param pollingTime delay between each call to [IBitrateRegulator.update] */ -open class DummyBitrateRegulatorController( +class IntervalBitrateRegulatorController( audioEncoder: IEncoder?, videoEncoder: IEncoder, - endpoint: IEndpoint, + metricsProvider: WithEndpointMetrics<*>, bitrateRegulatorFactory: IBitrateRegulator.Factory, coroutineDispatcher: CoroutineDispatcher, bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig(), - delayTimeInMs: Long = 500 + pollingTime: Duration = DEFAULT_POLLING_TIME ) : BitrateRegulatorController( audioEncoder, videoEncoder, - endpoint, + metricsProvider, bitrateRegulatorFactory, bitrateRegulatorConfig ) { + private val metricsTracker = metricsProvider.createMetricsTracker() + /** * Bitrate regulator. Calls regularly by [scheduler]. Don't call it otherwise or you might break regulation. */ private val bitrateRegulator = bitrateRegulatorFactory.newBitrateRegulator( + metricsTracker, bitrateRegulatorConfig, - { + onVideoTargetBitrateChange = { videoEncoder.bitrate = it }, - { /* Do nothing for audio */ } + onAudioTargetBitrateChange = { /* Do nothing for audio */ } ) /** * Scheduler for bitrate regulation */ - private val scheduler = CoroutineScheduler(delayTimeInMs, coroutineDispatcher) { + private val scheduler = CoroutineScheduler(pollingTime, coroutineDispatcher) { bitrateRegulator.update( - endpoint.metrics, videoEncoder.bitrate, audioEncoder?.bitrate ?: 0 ) @@ -80,10 +85,14 @@ open class DummyBitrateRegulatorController( scheduler.stop() } + companion object { + val DEFAULT_POLLING_TIME = 500.milliseconds + } + class Factory( private val bitrateRegulatorFactory: IBitrateRegulator.Factory, private val bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig(), - private val delayTimeInMs: Long = 500 + private val pollingTime: Duration = DEFAULT_POLLING_TIME ) : BitrateRegulatorController.Factory() { override fun newBitrateRegulatorController( pipelineOutput: IEncodingPipelineOutput, @@ -102,14 +111,15 @@ open class DummyBitrateRegulatorController( } else { null } - return DummyBitrateRegulatorController( + val endpoint = pipelineOutput.endpoint as WithEndpointMetrics<*> + return IntervalBitrateRegulatorController( audioEncoder, videoEncoder, - pipelineOutput.endpoint, + endpoint, bitrateRegulatorFactory, coroutineDispatcher, bitrateRegulatorConfig, - delayTimeInMs + pollingTime ) } } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt index 72c92ffd0..e889d3aeb 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamer.kt @@ -36,26 +36,15 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.exten import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation -import io.github.thibaultbee.streampack.core.elements.utils.extensions.isCompatibleWith import io.github.thibaultbee.streampack.core.interfaces.setCameraId import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline -import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioOutputMode -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioVideoEncodingPipelineOutput -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal -import io.github.thibaultbee.streampack.core.pipelines.utils.MultiThrowable +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.single.AudioConfig import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.cancel -import kotlinx.coroutines.flow.SharingStarted -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.combineTransform -import kotlinx.coroutines.flow.merge -import kotlinx.coroutines.flow.stateIn -import kotlinx.coroutines.runBlocking /** * Creates a [DualStreamer] with a default audio source. @@ -66,6 +55,8 @@ import kotlinx.coroutines.runBlocking * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraDualStreamer( @@ -74,19 +65,22 @@ suspend fun cameraDualStreamer( audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( - context, - withAudio = true, - withVideo = true, - firstEndpointFactory, - secondEndpointFactory, - defaultRotation + context = context, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setCameraId(cameraId) if (audioSourceFactory != null) { - streamer.audioInput!!.setSource(audioSourceFactory) + streamer.setAudioSource(audioSourceFactory) } return streamer } @@ -99,6 +93,8 @@ suspend fun cameraDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresApi(Build.VERSION_CODES.Q) suspend fun audioVideoMediaProjectionDualStreamer( @@ -106,18 +102,20 @@ suspend fun audioVideoMediaProjectionDualStreamer( mediaProjection: MediaProjection, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - streamer.videoInput!!.setSource(MediaProjectionVideoSourceFactory(mediaProjection)) + streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) streamer.setAudioSource(MediaProjectionAudioSourceFactory(mediaProjection)) return streamer } @@ -131,6 +129,8 @@ suspend fun audioVideoMediaProjectionDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionDualStreamer( context: Context, @@ -138,16 +138,19 @@ suspend fun videoMediaProjectionDualStreamer( audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) if (audioSourceFactory != null) { streamer.setAudioSource(audioSourceFactory) @@ -164,6 +167,8 @@ suspend fun videoMediaProjectionDualStreamer( * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun DualStreamer( context: Context, @@ -171,16 +176,19 @@ suspend fun DualStreamer( videoSourceFactory: IVideoSourceInternal.Factory, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): DualStreamer { val streamer = DualStreamer( context = context, - withAudio = true, - withVideo = true, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + streamer.setAudioSource(audioSourceFactory) streamer.setVideoSource(videoSourceFactory) return streamer @@ -192,238 +200,93 @@ suspend fun DualStreamer( * For example, you can use it to live stream and record simultaneously. * * @param context the application context - * @param withAudio `true` to capture audio. It can't be changed after instantiation. - * @param withVideo `true` to capture video. It can't be changed after instantiation. * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ -open class DualStreamer( - protected val context: Context, - val withAudio: Boolean = true, - val withVideo: Boolean = true, +class DualStreamer( + context: Context, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), - dispatcherProvider: IDispatcherProvider = DispatcherProvider(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : IDualStreamer, IAudioDualStreamer, IVideoDualStreamer { - private val coroutineScope = CoroutineScope(dispatcherProvider.default) - - private val pipeline = StreamerPipeline( - context, - withAudio, - withVideo, - AudioOutputMode.PUSH, - surfaceProcessorFactory, - dispatcherProvider - ) - - private val firstPipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, withVideo, firstEndpointFactory, defaultRotation - ) as IEncodingPipelineOutputInternal - } - - - /** - * First output of the streamer. - */ - override val first = firstPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput - - private val secondPipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, withVideo, secondEndpointFactory, defaultRotation - ) as IEncodingPipelineOutputInternal - } - - /** - * Second output of the streamer. - */ - override val second = secondPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput - - override val throwableFlow: StateFlow = merge( - pipeline.throwableFlow, - firstPipelineOutput.throwableFlow, - secondPipelineOutput.throwableFlow - ).stateIn( - coroutineScope, - SharingStarted.Eagerly, - null - ) - - /** - * Whether any of the output is opening. - */ - override val isOpenFlow: StateFlow = combineTransform( - firstPipelineOutput.isOpenFlow, secondPipelineOutput.isOpenFlow - ) { isOpens -> - emit(isOpens.any { it }) - }.stateIn( - coroutineScope, - SharingStarted.Eagerly, - false + private val streamer = DualStreamerImpl( + context = context, + withAudio = true, + withVideo = true, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - /** - * Whether any of the output is streaming. - */ - override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow - - /** - * Closes the outputs. - * Same as calling [first.close] and [second.close]. - */ - override suspend fun close() { - firstPipelineOutput.close() - secondPipelineOutput.close() - } + override val first = streamer.first as IConfigurableVideoEncodingPipelineOutput + override val second = streamer.second as IConfigurableVideoEncodingPipelineOutput - // SOURCES - override val audioInput = pipeline.audioInput + override val throwableFlow = streamer.throwableFlow + override val isOpenFlow = streamer.isOpenFlow + override val isStreamingFlow = streamer.isStreamingFlow - // PROCESSORS - override val videoInput = pipeline.videoInput + override val audioInput: IAudioInput = streamer.audioInput + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. * * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) */ - override suspend fun setTargetRotation(@RotationValue rotation: Int) { - pipeline.setTargetRotation(rotation) - } + override suspend fun setTargetRotation(@RotationValue rotation: Int) = + streamer.setTargetRotation(rotation) - /** - * Sets audio configuration. - * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig]. - * - * @param audioConfig the audio configuration to set - */ @RequiresPermission(Manifest.permission.RECORD_AUDIO) - override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) { - val throwables = mutableListOf() + override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) = + streamer.setAudioConfig(audioConfig) - val firstAudioCodecConfig = firstPipelineOutput.audioCodecConfigFlow.value - if ((firstAudioCodecConfig != null) && (!firstAudioCodecConfig.isCompatibleWith(audioConfig.firstAudioConfig))) { - firstPipelineOutput.invalidateAudioCodecConfig() - } - val secondAudioCodecConfig = secondPipelineOutput.audioCodecConfigFlow.value - if ((secondAudioCodecConfig != null) && (!secondAudioCodecConfig.isCompatibleWith( - audioConfig.secondAudioConfig - )) - ) { - secondPipelineOutput.invalidateAudioCodecConfig() - } + override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) = + streamer.setVideoConfig(videoConfig) - try { - firstPipelineOutput.setAudioCodecConfig(audioConfig.firstAudioConfig) - } catch (t: Throwable) { - throwables += t - } - try { - audioConfig.secondAudioConfig.let { secondPipelineOutput.setAudioCodecConfig(it) } - } catch (t: Throwable) { - throwables += t - } - if (throwables.isNotEmpty()) { - if (throwables.size == 1) { - throw throwables.first() - } else { - throw MultiThrowable(throwables) - } - } - } + override suspend fun close() = streamer.close() - /** - * Sets video configuration. - * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig]. - * To only set video configuration for a specific output, use [first.setVideoCodecConfig] or - * [second.setVideoCodecConfig] outputs. - * In that case, you call [first.setVideoCodecConfig] or [second.setVideoCodecConfig] explicitly, - * make sure that the frame rate for both configurations is the same. - * - * @param videoConfig the video configuration to set - */ - override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) { - val throwables = mutableListOf() - - val firstVideoCodecConfig = firstPipelineOutput.videoCodecConfigFlow.value - if ((firstVideoCodecConfig != null) && (!firstVideoCodecConfig.isCompatibleWith(videoConfig.firstVideoConfig))) { - firstPipelineOutput.invalidateVideoCodecConfig() - } + override suspend fun startStream() = streamer.startStream() - val secondVideoCodecConfig = secondPipelineOutput.videoCodecConfigFlow.value - if ((secondVideoCodecConfig != null) && (!secondVideoCodecConfig.isCompatibleWith( - videoConfig.secondVideoConfig - )) - ) { - secondPipelineOutput.invalidateVideoCodecConfig() - } - - try { - firstPipelineOutput.setVideoCodecConfig(videoConfig.firstVideoConfig) - } catch (t: Throwable) { - throwables += t - } - try { - secondPipelineOutput.setVideoCodecConfig(videoConfig.secondVideoConfig) - } catch (t: Throwable) { - throwables += t - } - if (throwables.isNotEmpty()) { - if (throwables.size == 1) { - throw throwables.first() - } else { - throw MultiThrowable(throwables) - } - } - } + override suspend fun stopStream() = streamer.stopStream() - /** - * Configures both video and audio settings. - * - * It must be call when both stream and audio and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param audioConfig Audio configuration to set - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - * @see [DualStreamer.release] - */ - @RequiresPermission(Manifest.permission.RECORD_AUDIO) - suspend fun setConfig( - audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig - ) { - setAudioConfig(audioConfig) - setVideoConfig(videoConfig) - } - - - override suspend fun startStream() = pipeline.startStream() - - override suspend fun stopStream() = pipeline.stopStream() + override suspend fun release() = streamer.release() +} - override suspend fun release() { - pipeline.release() - coroutineScope.cancel() - } +/** + * Configures both video and audio settings. + * + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [DualStreamer.release] + */ +@RequiresPermission(Manifest.permission.RECORD_AUDIO) +suspend fun DualStreamer.setConfig( + audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig +) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) } /** * Sets audio configuration. * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig] when both + * It is a shortcut for [DualStreamer.setVideoConfig] when both * outputs use the same audio configuration. * * @param audioConfig the audio configuration to set @@ -436,7 +299,7 @@ suspend fun DualStreamer.setAudioConfig(audioConfig: AudioConfig) { /** * Sets video configuration. * - * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig] when both + * It is a shortcut for [DualStreamer.setVideoConfig] when both * outputs use the same video configuration. * * @param videoConfig the video configuration to set diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt new file mode 100644 index 000000000..d0c76ba13 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/DualStreamerImpl.kt @@ -0,0 +1,283 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.streamers.dual + +import android.Manifest +import android.content.Context +import android.view.Surface +import androidx.annotation.RequiresPermission +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal.Factory +import io.github.thibaultbee.streampack.core.elements.utils.RotationValue +import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation +import io.github.thibaultbee.streampack.core.elements.utils.extensions.isCompatibleWith +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioInputMode +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioVideoEncodingPipelineOutput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.pipelines.utils.MultiThrowable +import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.combineTransform +import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking + + +/** + * A class that handles 2 outputs. + * + * For example, you can use it to live stream and record simultaneously. + * + * @param context the application context + * @param withAudio `true` to capture audio. It can't be changed after instantiation. + * @param withVideo `true` to capture video. It can't be changed after instantiation. + * @param firstEndpointFactory the [IEndpointInternal] implementation of the first output. By default, it is a [DynamicEndpoint]. + * @param secondEndpointFactory the [IEndpointInternal] implementation of the second output. By default, it is a [DynamicEndpoint]. + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. + */ +internal class DualStreamerImpl( + private val context: Context, + withAudio: Boolean = true, + withVideo: Boolean = true, + firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider(), +) : IDualStreamer, IAudioDualStreamer, IVideoDualStreamer { + private val coroutineScope = CoroutineScope(dispatcherProvider.default) + + private val pipeline = StreamerPipeline( + context, + withAudio, + withVideo, + AudioInputMode.PUSH, + surfaceProcessorFactory, + dispatcherProvider + ) + + private val firstPipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, withVideo, firstEndpointFactory, defaultRotation + ) as IEncodingPipelineOutputInternal + } + + + /** + * First output of the streamer. + */ + override val first = firstPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput + + private val secondPipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, withVideo, secondEndpointFactory, defaultRotation + ) as IEncodingPipelineOutputInternal + } + + /** + * Second output of the streamer. + */ + override val second = secondPipelineOutput as IConfigurableAudioVideoEncodingPipelineOutput + + override val throwableFlow: StateFlow = merge( + pipeline.throwableFlow, + firstPipelineOutput.throwableFlow, + secondPipelineOutput.throwableFlow + ).stateIn( + coroutineScope, + SharingStarted.Eagerly, + null + ) + + /** + * Whether any of the output is opening. + */ + override val isOpenFlow: StateFlow = combineTransform( + firstPipelineOutput.isOpenFlow, secondPipelineOutput.isOpenFlow + ) { isOpens -> + emit(isOpens.any { it }) + }.stateIn( + coroutineScope, + SharingStarted.Eagerly, + false + ) + + /** + * Whether any of the output is streaming. + */ + override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + + /** + * Closes the outputs. + * Same as calling [first.close] and [second.close]. + */ + override suspend fun close() { + firstPipelineOutput.close() + secondPipelineOutput.close() + } + + // SOURCES + override val audioInput = pipeline.audioInput + + // PROCESSORS + override val videoInput = pipeline.videoInput + + /** + * Sets the target rotation. + * + * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) + */ + override suspend fun setTargetRotation(@RotationValue rotation: Int) { + pipeline.setTargetRotation(rotation) + } + + /** + * Sets audio configuration. + * + * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setAudioCodecConfig]. + * + * @param audioConfig the audio configuration to set + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + override suspend fun setAudioConfig(audioConfig: DualStreamerAudioConfig) { + val throwables = mutableListOf() + + val firstAudioCodecConfig = firstPipelineOutput.audioCodecConfigFlow.value + if ((firstAudioCodecConfig != null) && (!firstAudioCodecConfig.isCompatibleWith(audioConfig.firstAudioConfig))) { + firstPipelineOutput.invalidateAudioCodecConfig() + } + val secondAudioCodecConfig = secondPipelineOutput.audioCodecConfigFlow.value + if ((secondAudioCodecConfig != null) && (!secondAudioCodecConfig.isCompatibleWith( + audioConfig.secondAudioConfig + )) + ) { + secondPipelineOutput.invalidateAudioCodecConfig() + } + + try { + firstPipelineOutput.setAudioCodecConfig(audioConfig.firstAudioConfig) + } catch (t: Throwable) { + throwables += t + } + try { + audioConfig.secondAudioConfig.let { secondPipelineOutput.setAudioCodecConfig(it) } + } catch (t: Throwable) { + throwables += t + } + if (throwables.isNotEmpty()) { + if (throwables.size == 1) { + throw throwables.first() + } else { + throw MultiThrowable(throwables) + } + } + } + + /** + * Sets video configuration. + * + * It is a shortcut for [IConfigurableAudioVideoEncodingPipelineOutput.setVideoCodecConfig]. + * To only set video configuration for a specific output, use [first.setVideoCodecConfig] or + * [second.setVideoCodecConfig] outputs. + * In that case, you call [first.setVideoCodecConfig] or [second.setVideoCodecConfig] explicitly, + * make sure that the frame rate for both configurations is the same. + * + * @param videoConfig the video configuration to set + */ + override suspend fun setVideoConfig(videoConfig: DualStreamerVideoConfig) { + val throwables = mutableListOf() + + val firstVideoCodecConfig = firstPipelineOutput.videoCodecConfigFlow.value + if ((firstVideoCodecConfig != null) && (!firstVideoCodecConfig.isCompatibleWith(videoConfig.firstVideoConfig))) { + firstPipelineOutput.invalidateVideoCodecConfig() + } + + val secondVideoCodecConfig = secondPipelineOutput.videoCodecConfigFlow.value + if ((secondVideoCodecConfig != null) && (!secondVideoCodecConfig.isCompatibleWith( + videoConfig.secondVideoConfig + )) + ) { + secondPipelineOutput.invalidateVideoCodecConfig() + } + + try { + firstPipelineOutput.setVideoCodecConfig(videoConfig.firstVideoConfig) + } catch (t: Throwable) { + throwables += t + } + try { + secondPipelineOutput.setVideoCodecConfig(videoConfig.secondVideoConfig) + } catch (t: Throwable) { + throwables += t + } + if (throwables.isNotEmpty()) { + if (throwables.size == 1) { + throw throwables.first() + } else { + throw MultiThrowable(throwables) + } + } + } + + /** + * Configures both video and audio settings. + * + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [DualStreamer.release] + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + suspend fun setConfig( + audioConfig: DualStreamerAudioConfig, videoConfig: DualStreamerVideoConfig + ) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) + } + + + override suspend fun startStream() = pipeline.startStream() + + override suspend fun stopStream() = pipeline.stopStream() + + override suspend fun release() { + pipeline.release() + coroutineScope.cancel() + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt index 979ffe9a8..b98472b80 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/IDualStreamer.kt @@ -243,9 +243,9 @@ internal constructor( val dynamicRangeProfile = firstVideoConfig.dynamicRangeProfile } -interface IAudioDualStreamer : IAudioStreamer +interface IAudioDualStreamer : IAudioStreamer, IDualStreamer -interface IVideoDualStreamer : IVideoStreamer +interface IVideoDualStreamer : IVideoStreamer, IDualStreamer interface IDualStreamer : ICloseableStreamer { /** diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt index 31b89d58d..b4664f6b7 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/dual/VideoOnlyDualStreamer.kt @@ -23,14 +23,20 @@ import android.view.Surface import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal.Factory import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.setCameraId +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput +import io.github.thibaultbee.streampack.core.streamers.single.VideoConfig /** @@ -41,6 +47,8 @@ import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigu * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraVideoOnlyDualStreamer( @@ -48,10 +56,17 @@ suspend fun cameraVideoOnlyDualStreamer( cameraId: String = context.defaultCameraId, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( - context, firstEndpointFactory, secondEndpointFactory, defaultRotation + context = context, + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) return streamer @@ -65,19 +80,25 @@ suspend fun cameraVideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionVideoOnlyDualStreamer( context: Context, mediaProjection: MediaProjection, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) return streamer @@ -91,19 +112,25 @@ suspend fun videoMediaProjectionVideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun VideoOnlyDualStreamer( context: Context, videoSourceFactory: IVideoSourceInternal.Factory, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlyDualStreamer { val streamer = VideoOnlyDualStreamer( context = context, firstEndpointFactory = firstEndpointFactory, secondEndpointFactory = secondEndpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(videoSourceFactory) return streamer @@ -116,20 +143,26 @@ suspend fun VideoOnlyDualStreamer( * @param firstEndpointFactory the [IEndpointInternal.Factory] implementation of the first output. By default, it is a [DynamicEndpointFactory]. * @param secondEndpointFactory the [IEndpointInternal.Factory] implementation of the second output. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation to use to create the video processor. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class VideoOnlyDualStreamer( context: Context, firstEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), secondEndpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : IDualStreamer, IVideoDualStreamer { - private val streamer = DualStreamer( + private val streamer = DualStreamerImpl( context = context, - firstEndpointFactory = firstEndpointFactory, - secondEndpointFactory = secondEndpointFactory, withAudio = false, withVideo = true, - defaultRotation = defaultRotation + firstEndpointFactory = firstEndpointFactory, + secondEndpointFactory = secondEndpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) override val first = streamer.first as IConfigurableVideoEncodingPipelineOutput @@ -139,7 +172,7 @@ class VideoOnlyDualStreamer( override val isOpenFlow = streamer.isOpenFlow override val isStreamingFlow = streamer.isStreamingFlow - override val videoInput: IVideoInput = streamer.videoInput!! + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. @@ -159,4 +192,17 @@ class VideoOnlyDualStreamer( override suspend fun stopStream() = streamer.stopStream() override suspend fun release() = streamer.release() -} \ No newline at end of file +} + +/** + * Sets video configuration. + * + * It is a shortcut for [VideoOnlyDualStreamer.setVideoConfig] when both + * outputs use the same video configuration. + * + * @param videoConfig the video configuration to set + */ +suspend fun VideoOnlyDualStreamer.setVideoConfig(videoConfig: VideoConfig) { + setVideoConfig(DualStreamerVideoConfig(videoConfig)) +} + diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerActivityLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerActivityLifeCycleObserver.kt deleted file mode 100644 index abab6d9d0..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerActivityLifeCycleObserver.kt +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2022 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.streamers.lifecycle - -import androidx.lifecycle.DefaultLifecycleObserver -import androidx.lifecycle.LifecycleOwner -import io.github.thibaultbee.streampack.core.interfaces.IStreamer -import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking - -/** - * A [DefaultLifecycleObserver] to control a streamer on [Activity] lifecycle. - * - * It stops streamer when application goes to background and release it when application is destroyed. - * - * To use it, call: - * - `lifeCycle.addObserver(StreamerActivityLifeCycleObserver(streamer))` - * - * @param streamer The streamer to control - */ -open class StreamerActivityLifeCycleObserver(streamer: IStreamer) : - StreamerViewModelLifeCycleObserver(streamer) { - - override fun onDestroy(owner: LifecycleOwner) { - streamer.releaseBlocking() - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt new file mode 100644 index 000000000..4d5f08b98 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerLifeCycleObserver.kt @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2022 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.streamers.lifecycle + +import androidx.lifecycle.DefaultLifecycleObserver +import androidx.lifecycle.LifecycleOwner +import androidx.lifecycle.lifecycleScope +import io.github.thibaultbee.streampack.core.interfaces.ICloseableStreamer +import io.github.thibaultbee.streampack.core.interfaces.IStreamer +import io.github.thibaultbee.streampack.core.interfaces.IWithAudioSource +import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking +import io.github.thibaultbee.streampack.core.logger.Logger +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +/** + * A [DefaultLifecycleObserver] to control a streamer on [Activity] lifecycle. + * + * It stops streamer when application goes to background and release it when application is destroyed. + * + * To use it, call: + * - `lifeCycle.addObserver(StreamerActivityLifeCycleObserver(streamer))` + * + * @param streamer The streamer to control + * @param releaseOnDestroy Whether to release the streamer when application is destroyed. If a view model is used, this should be set to false. + * @param startAudioCaptureOnResume Whether to start audio capture when application is resumed. + */ +open class StreamerLifeCycleObserver( + private val streamer: IStreamer, + private val releaseOnDestroy: Boolean = false, + private val startAudioCaptureOnResume: Boolean = false, + private val coroutineDispatcher: CoroutineDispatcher = Dispatchers.Default +) : DefaultLifecycleObserver { + override fun onResume(owner: LifecycleOwner) { + if (startAudioCaptureOnResume) { + owner.lifecycleScope.launch(coroutineDispatcher) { + withContext(NonCancellable) { + if (streamer is IWithAudioSource) { + // Wait for audio input to be ready + streamer.audioInput.sourceFlow.filterNotNull().first() + try { + streamer.audioInput.startCapture() + } catch (t: Throwable) { + Logger.e(TAG, "Error while starting audio capture: $t") + } + } + } + } + } + } + + override fun onPause(owner: LifecycleOwner) { + owner.lifecycleScope.launch(coroutineDispatcher) { + withContext(NonCancellable) { + try { + streamer.stopStream() + } catch (t: Throwable) { + Logger.e(TAG, "Error while stopping streamer: $t") + } + if (streamer is ICloseableStreamer) { + try { + streamer.close() + } catch (t: Throwable) { + Logger.e(TAG, "Error while closing streamer: $t") + } + } + if (streamer is IWithAudioSource) { + try { + streamer.audioInput.stopCapture() + } catch (t: Throwable) { + Logger.e(TAG, "Error while stopping audio capture: $t") + } + } + } + } + } + + override fun onDestroy(owner: LifecycleOwner) { + if (releaseOnDestroy) { + try { + streamer.releaseBlocking(coroutineDispatcher) + } catch (t: Throwable) { + Logger.e(TAG, "Error while releasing streamer: $t") + } + } + } + + companion object { + private const val TAG = "StreamerLifeCycleObserver" + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerViewModelLifeCycleObserver.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerViewModelLifeCycleObserver.kt deleted file mode 100644 index ad42887a8..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/lifecycle/StreamerViewModelLifeCycleObserver.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2024 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.streamers.lifecycle - -import androidx.lifecycle.DefaultLifecycleObserver -import androidx.lifecycle.LifecycleOwner -import androidx.lifecycle.lifecycleScope -import io.github.thibaultbee.streampack.core.interfaces.ICloseableStreamer -import io.github.thibaultbee.streampack.core.interfaces.IStreamer -import kotlinx.coroutines.NonCancellable -import kotlinx.coroutines.launch - -/** - * A [DefaultLifecycleObserver] to control a streamer on [Activity] lifecycle in a ViewModel. - * - * It stops streamer when application goes to background. - * - * To use it, call: - * - `lifeCycle.addObserver(StreamerActivityLifeCycleObserver(streamer))` - * - * @param streamer The streamer to control - */ -open class StreamerViewModelLifeCycleObserver(protected val streamer: IStreamer) : - DefaultLifecycleObserver { - - override fun onPause(owner: LifecycleOwner) { - owner.lifecycleScope.launch(NonCancellable) { - streamer.stopStream() - if (streamer is ICloseableStreamer) { - streamer.close() - } - } - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt index d8043700c..e10b6d8f0 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/AudioOnlySingleStreamer.kt @@ -25,8 +25,12 @@ import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSourceFactory +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioInputMode +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline.AudioInputMode.CALLBACK import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput -import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo /** @@ -34,16 +38,22 @@ import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo * * @param context the application context * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If parameter is null, no audio source are set. It can be set later with [AudioOnlySingleStreamer.setAudioSource]. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun AudioOnlySingleStreamer( context: Context, audioSourceFactory: IAudioSourceInternal.Factory = MicrophoneSourceFactory(), - endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory() + audioInputMode: AudioInputMode = CALLBACK, + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): AudioOnlySingleStreamer { val streamer = AudioOnlySingleStreamer( context = context, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, + dispatcherProvider = dispatcherProvider ) streamer.setAudioSource(audioSourceFactory) return streamer @@ -53,17 +63,23 @@ suspend fun AudioOnlySingleStreamer( * A [ISingleStreamer] implementation for audio only (without video). * * @param context the application context + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class AudioOnlySingleStreamer( context: Context, - endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory() + audioInputMode: AudioInputMode = CALLBACK, + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : ISingleStreamer, IAudioSingleStreamer { - private val streamer = SingleStreamer( + private val streamer = SingleStreamerImpl( context = context, - endpointFactory = endpointFactory, withAudio = true, - withVideo = false + withVideo = false, + audioInputMode = audioInputMode, + endpointFactory = endpointFactory, + dispatcherProvider = dispatcherProvider ) override val throwableFlow = streamer.throwableFlow override val isOpenFlow = streamer.isOpenFlow @@ -74,8 +90,8 @@ class AudioOnlySingleStreamer( get() = streamer.info override val audioConfigFlow = streamer.audioConfigFlow - override val audioInput: IAudioInput = streamer.audioInput!! - + override val audioInput: IAudioInput = streamer.audioInput + override val audioEncoder: IEncoder? get() = streamer.audioEncoder @@ -97,12 +113,4 @@ class AudioOnlySingleStreamer( override suspend fun stopStream() = streamer.stopStream() override suspend fun release() = streamer.release() - - override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) { - throw UnsupportedOperationException("Audio single streamer does not support bitrate regulator controller") - } - - override fun removeBitrateRegulatorController() { - // Do nothing - } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt index d19ffbfc3..830680e2c 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/ISingleStreamer.kt @@ -55,22 +55,25 @@ interface ISingleStreamer : IOpenableStreamer { * Gets configuration information */ fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo +} +val ISingleStreamer.withAudio: Boolean /** - * Adds a bitrate regulator controller to the streamer. + * Whether the streamer has an audio source. */ - fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) + get() = this is IAudioSingleStreamer +val ISingleStreamer.withVideo: Boolean /** - * Removes the bitrate regulator controller from the streamer. + * Whether the streamer has a video source. */ - fun removeBitrateRegulatorController() -} + get() = this is IVideoSingleStreamer + /** * An audio single Streamer */ -interface IAudioSingleStreamer : IAudioStreamer { +interface IAudioSingleStreamer : IAudioStreamer, ISingleStreamer { /** * The audio configuration flow. */ @@ -85,7 +88,7 @@ interface IAudioSingleStreamer : IAudioStreamer { /** * A video single streamer. */ -interface IVideoSingleStreamer : IVideoStreamer { +interface IVideoSingleStreamer : IVideoStreamer, ISingleStreamer { /** * The video configuration flow. */ @@ -95,5 +98,9 @@ interface IVideoSingleStreamer : IVideoStreamer { * Advanced settings for the video encoder. */ val videoEncoder: IEncoder? -} + /** + * Adds a bitrate regulator controller to the streamer. + */ + var bitrateRegulatorControllerFactory: IBitrateRegulatorController.Factory? +} diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt index d7c1138da..84bc5a475 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamer.kt @@ -25,7 +25,6 @@ import androidx.annotation.RequiresApi import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder -import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal @@ -35,7 +34,6 @@ import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MediaProjectionAudioSourceFactory import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSourceFactory import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal -import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue @@ -44,18 +42,10 @@ import io.github.thibaultbee.streampack.core.interfaces.setCameraId import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController -import io.github.thibaultbee.streampack.core.streamers.infos.CameraStreamerConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo -import io.github.thibaultbee.streampack.core.streamers.infos.StreamerConfigurationInfo -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.cancel -import kotlinx.coroutines.flow.SharingStarted -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.merge -import kotlinx.coroutines.flow.stateIn -import kotlinx.coroutines.runBlocking /** @@ -64,19 +54,30 @@ import kotlinx.coroutines.runBlocking * @param context the application context * @param cameraId the camera id to use. By default, it is the default camera. * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If set to null, you will have to set it later explicitly. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraSingleStreamer( context: Context, cameraId: String = context.defaultCameraId, audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( - context, withAudio = true, withVideo = true, endpointFactory, defaultRotation + context = context, + audioInputMode = audioInputMode, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) if (audioSourceFactory != null) { @@ -90,22 +91,29 @@ suspend fun cameraSingleStreamer( * * @param context the application context * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresApi(Build.VERSION_CODES.Q) suspend fun audioVideoMediaProjectionSingleStreamer( context: Context, mediaProjection: MediaProjection, + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) @@ -119,22 +127,29 @@ suspend fun audioVideoMediaProjectionSingleStreamer( * @param context the application context * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. * @param audioSourceFactory the audio source factory. By default, it is the default microphone source factory. If set to null, you will have to set it later explicitly. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionSingleStreamer( context: Context, mediaProjection: MediaProjection, audioSourceFactory: IAudioSourceInternal.Factory? = MicrophoneSourceFactory(), + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, - withAudio = true, - withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) @@ -150,22 +165,29 @@ suspend fun videoMediaProjectionSingleStreamer( * @param context the application context * @param audioSourceFactory the audio source factory. * @param videoSourceFactory the video source factory. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun SingleStreamer( context: Context, audioSourceFactory: IAudioSourceInternal.Factory, videoSourceFactory: IVideoSourceInternal.Factory, + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): SingleStreamer { val streamer = SingleStreamer( context = context, - withAudio = true, - withVideo = true, + audioInputMode = audioInputMode, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setAudioSource(audioSourceFactory) streamer.setVideoSource(videoSourceFactory) @@ -173,249 +195,106 @@ suspend fun SingleStreamer( } /** - * A [ISingleStreamer] implementation for audio and video. + * A [ISingleStreamer] implementation for both audio and video. * * @param context the application context - * @param withAudio `true` to capture audio. It can't be changed after instantiation. - * @param withVideo `true` to capture video. It can't be changed after instantiation. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ -open class SingleStreamer( - protected val context: Context, - val withAudio: Boolean = true, - val withVideo: Boolean = true, +class SingleStreamer( + context: Context, + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), @RotationValue defaultRotation: Int = context.displayRotation, surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), dispatcherProvider: IDispatcherProvider = DispatcherProvider(), ) : ISingleStreamer, IAudioSingleStreamer, IVideoSingleStreamer { - private val coroutineScope: CoroutineScope = CoroutineScope(dispatcherProvider.default) - - private val pipeline = StreamerPipeline( - context, - withAudio, - withVideo, - audioOutputMode = StreamerPipeline.AudioOutputMode.CALLBACK, - surfaceProcessorFactory, - dispatcherProvider + private val streamer = SingleStreamerImpl( + context = context, + withAudio = true, + withVideo = true, + audioInputMode = audioInputMode, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) - private val pipelineOutput: IEncodingPipelineOutputInternal = - runBlocking(dispatcherProvider.default) { - pipeline.createEncodingOutput( - withAudio, - withVideo, - endpointFactory, - defaultRotation - ) as IEncodingPipelineOutputInternal - } - - override val throwableFlow: StateFlow = - merge(pipeline.throwableFlow, pipelineOutput.throwableFlow).stateIn( - coroutineScope, - SharingStarted.Eagerly, - null - ) - - override val isOpenFlow: StateFlow - get() = pipelineOutput.isOpenFlow - override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + override val throwableFlow = streamer.throwableFlow + override val isOpenFlow = streamer.isOpenFlow + override val isStreamingFlow = streamer.isStreamingFlow - // AUDIO - /** - * The audio input. - * It allows advanced audio source settings. - */ - override val audioInput = pipeline.audioInput + override val endpoint: IEndpoint + get() = streamer.endpoint + override val info: IConfigurationInfo + get() = streamer.info + override val audioConfigFlow = streamer.audioConfigFlow override val audioEncoder: IEncoder? - get() = pipelineOutput.audioEncoder - - override suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) = - pipeline.setAudioSource(audioSourceFactory) - - // VIDEO - /** - * The video input. - * It allows advanced video source settings. - */ - override val videoInput = pipeline.videoInput + get() = streamer.audioEncoder + override val audioInput: IAudioInput = streamer.audioInput!! + override val videoConfigFlow = streamer.videoConfigFlow override val videoEncoder: IEncoder? - get() = pipelineOutput.videoEncoder - - // ENDPOINT - override val endpoint: IEndpoint - get() = pipelineOutput.endpoint + get() = streamer.videoEncoder + override val videoInput: IVideoInput = streamer.videoInput!! /** * Sets the target rotation. * * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) */ - override suspend fun setTargetRotation(@RotationValue rotation: Int) { - pipeline.setTargetRotation(rotation) - } + override suspend fun setTargetRotation(@RotationValue rotation: Int) = + streamer.setTargetRotation(rotation) - /** - * Gets configuration information. - * - * Could throw an exception if the endpoint needs to infer the configuration from the - * [MediaDescriptor]. - * In this case, prefer using [getInfo] with the [MediaDescriptor] used in [open]. - */ - override val info: IConfigurationInfo - get() = if (videoInput?.sourceFlow?.value is CameraSource) { - CameraStreamerConfigurationInfo(endpoint.info) - } else { - StreamerConfigurationInfo(endpoint.info) - } - - /** - * Gets configuration information from [MediaDescriptor]. - * - * If the endpoint is not [DynamicEndpoint], [descriptor] is unused as the endpoint type is - * already known. - * - * @param descriptor the media descriptor - */ - override fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo { - val endpointInfo = try { - endpoint.info - } catch (_: Throwable) { - endpoint.getInfo(descriptor) - } - return if (videoInput?.sourceFlow?.value is CameraSource) { - CameraStreamerConfigurationInfo(endpointInfo) - } else { - StreamerConfigurationInfo(endpointInfo) - } - } - - // CONFIGURATION - /** - * The audio configuration flow. - */ - override val audioConfigFlow: StateFlow = pipelineOutput.audioCodecConfigFlow - - /** - * Configures audio settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and audio capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * @param audioConfig Audio configuration to set - * - * @throws [Throwable] if configuration can not be applied. - */ @RequiresPermission(Manifest.permission.RECORD_AUDIO) - override suspend fun setAudioConfig(audioConfig: AudioConfig) { - pipelineOutput.setAudioCodecConfig(audioConfig) - } + override suspend fun setAudioConfig(audioConfig: AudioConfig) = + streamer.setAudioConfig(audioConfig) - /** - * The video configuration flow. - */ - override val videoConfigFlow: StateFlow = pipelineOutput.videoCodecConfigFlow + override suspend fun setVideoConfig(videoConfig: VideoConfig) = + streamer.setVideoConfig(videoConfig) - /** - * Configures video settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - */ - override suspend fun setVideoConfig(videoConfig: VideoConfig) { - pipelineOutput.setVideoCodecConfig(videoConfig) - } + override fun getInfo(descriptor: MediaDescriptor) = streamer.getInfo(descriptor) - /** - * Configures both video and audio settings. - * It is the first method to call after a [SingleStreamer] instantiation. - * It must be call when both stream and audio and video capture are not running. - * - * Use [IConfigurationInfo] to get value limits. - * - * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks - * to video encoder default level and default profile. - * - * @param audioConfig Audio configuration to set - * @param videoConfig Video configuration to set - * - * @throws [Throwable] if configuration can not be applied. - * @see [IStreamer.release] - */ - @RequiresPermission(Manifest.permission.RECORD_AUDIO) - suspend fun setConfig(audioConfig: AudioConfig, videoConfig: VideoConfig) { - setAudioConfig(audioConfig) - setVideoConfig(videoConfig) - } - - /** - * Opens the streamer endpoint. - * - * @param descriptor Media descriptor to open - */ - override suspend fun open(descriptor: MediaDescriptor) = pipelineOutput.open(descriptor) + override suspend fun open(descriptor: MediaDescriptor) = streamer.open(descriptor) - /** - * Closes the streamer endpoint. - */ - override suspend fun close() = pipelineOutput.close() + override suspend fun close() = streamer.close() - /** - * Starts audio/video stream. - * Stream depends of the endpoint: Audio/video could be write to a file or send to a remote - * device. - * To avoid creating an unresponsive UI, do not call on main thread. - * - * @see [stopStream] - */ - override suspend fun startStream() = pipelineOutput.startStream() + override suspend fun startStream() = streamer.startStream() - /** - * Stops audio/video stream. - * - * Internally, it resets audio and video recorders and encoders to get them ready for another - * [startStream] session. It explains why preview could be restarted. - * - * @see [startStream] - */ - override suspend fun stopStream() = pipeline.stopStream() + override suspend fun stopStream() = streamer.stopStream() - /** - * Releases the streamer. - */ - override suspend fun release() { - pipeline.release() - coroutineScope.cancel() - } + override suspend fun release() = streamer.release() - /** - * Adds a bitrate regulator controller. - * - * Limitation: it is only available for SRT for now. - */ - override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) = - pipelineOutput.addBitrateRegulatorController(controllerFactory) + override var bitrateRegulatorControllerFactory: IBitrateRegulatorController.Factory? + get() = streamer.bitrateRegulatorControllerFactory + set(value) { + streamer.bitrateRegulatorControllerFactory = value + } +} - /** - * Removes the bitrate regulator controller. - */ - override fun removeBitrateRegulatorController() = - pipelineOutput.removeBitrateRegulatorController() - companion object { - const val TAG = "SingleStreamer" - } +/** + * Configures both video and audio settings. + * It is the first method to call after a [SingleStreamer] instantiation. + * It must be call when both stream and audio and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param audioConfig Audio configuration to set + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + * @see [SingleStreamer.release] + */ +@RequiresPermission(Manifest.permission.RECORD_AUDIO) +suspend fun SingleStreamer.setConfig(audioConfig: AudioConfig, videoConfig: VideoConfig) { + setAudioConfig(audioConfig) + setVideoConfig(videoConfig) } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt new file mode 100644 index 000000000..cb5a5f425 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/SingleStreamerImpl.kt @@ -0,0 +1,275 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.streamers.single + +import android.Manifest +import android.content.Context +import android.view.Surface +import androidx.annotation.RequiresPermission +import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor +import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint +import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal +import io.github.thibaultbee.streampack.core.elements.sources.audio.IAudioSourceInternal +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSource +import io.github.thibaultbee.streampack.core.elements.utils.RotationValue +import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.pipelines.inputs.IAudioInput +import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput +import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutputInternal +import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController +import io.github.thibaultbee.streampack.core.streamers.infos.CameraStreamerConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo +import io.github.thibaultbee.streampack.core.streamers.infos.StreamerConfigurationInfo +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking + +/** + * A [ISingleStreamer] implementation for audio and video. + * + * @param context the application context + * @param withAudio `true` to capture audio. It can't be changed after instantiation. + * @param withVideo `true` to capture video. It can't be changed after instantiation. + * @param audioInputMode the audio output mode. By default, it is [StreamerPipeline.AudioInputMode.CALLBACK]. Use [StreamerPipeline.AudioInputMode.PUSH] only to get processor (incl. vumeter) running outside a stream + * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + */ +internal class SingleStreamerImpl( + private val context: Context, + withAudio: Boolean, + withVideo: Boolean, + audioInputMode: StreamerPipeline.AudioInputMode = StreamerPipeline.AudioInputMode.CALLBACK, + endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider(), +) : ISingleStreamer, IAudioSingleStreamer, IVideoSingleStreamer { + private val coroutineScope: CoroutineScope = CoroutineScope(dispatcherProvider.default) + + private val pipeline = StreamerPipeline( + context, + withAudio, + withVideo, + audioInputMode = audioInputMode, + surfaceProcessorFactory, + dispatcherProvider + ) + private val pipelineOutput: IEncodingPipelineOutputInternal = + runBlocking(dispatcherProvider.default) { + pipeline.createEncodingOutput( + withAudio, + withVideo, + endpointFactory, + defaultRotation + ) as IEncodingPipelineOutputInternal + } + + override val throwableFlow: StateFlow = + merge(pipeline.throwableFlow, pipelineOutput.throwableFlow).stateIn( + coroutineScope, + SharingStarted.Eagerly, + null + ) + + override val isOpenFlow: StateFlow + get() = pipelineOutput.isOpenFlow + + override val isStreamingFlow: StateFlow = pipeline.isStreamingFlow + + // AUDIO + /** + * The audio input. + * It allows advanced audio source settings. + */ + override val audioInput: IAudioInput + get() = pipeline.audioInput + + override val audioEncoder: IEncoder? + get() = pipelineOutput.audioEncoder + + override suspend fun setAudioSource(audioSourceFactory: IAudioSourceInternal.Factory) = + pipeline.setAudioSource(audioSourceFactory) + + // VIDEO + /** + * The video input. + * It allows advanced video source settings. + */ + override val videoInput: IVideoInput + get() = pipeline.videoInput + + override val videoEncoder: IEncoder? + get() = pipelineOutput.videoEncoder + + // ENDPOINT + override val endpoint: IEndpoint + get() = pipelineOutput.endpoint + + /** + * Sets the target rotation. + * + * @param rotation the target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) + */ + override suspend fun setTargetRotation(@RotationValue rotation: Int) { + pipeline.setTargetRotation(rotation) + } + + /** + * Gets configuration information. + * + * Could throw an exception if the endpoint needs to infer the configuration from the + * [MediaDescriptor]. + * In this case, prefer using [getInfo] with the [MediaDescriptor] used in [open]. + */ + override val info: IConfigurationInfo + get() = if (videoInput.sourceFlow.value is CameraSource) { + CameraStreamerConfigurationInfo(endpoint.info) + } else { + StreamerConfigurationInfo(endpoint.info) + } + + /** + * Gets configuration information from [MediaDescriptor]. + * + * If the endpoint is not [DynamicEndpoint], [descriptor] is unused as the endpoint type is + * already known. + * + * @param descriptor the media descriptor + */ + override fun getInfo(descriptor: MediaDescriptor): IConfigurationInfo { + val endpointInfo = try { + endpoint.info + } catch (_: Throwable) { + endpoint.getInfo(descriptor) + } + return if (videoInput.sourceFlow.value is CameraSource) { + CameraStreamerConfigurationInfo(endpointInfo) + } else { + StreamerConfigurationInfo(endpointInfo) + } + } + + // CONFIGURATION + /** + * The audio configuration flow. + */ + override val audioConfigFlow: StateFlow = pipelineOutput.audioCodecConfigFlow + + /** + * Configures audio settings. + * It is the first method to call after a [SingleStreamerImpl] instantiation. + * It must be call when both stream and audio capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * @param audioConfig Audio configuration to set + * + * @throws [Throwable] if configuration can not be applied. + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + override suspend fun setAudioConfig(audioConfig: AudioConfig) { + pipelineOutput.setAudioCodecConfig(audioConfig) + } + + /** + * The video configuration flow. + */ + override val videoConfigFlow: StateFlow = pipelineOutput.videoCodecConfigFlow + + /** + * Configures video settings. + * It is the first method to call after a [SingleStreamerImpl] instantiation. + * It must be call when both stream and video capture are not running. + * + * Use [IConfigurationInfo] to get value limits. + * + * If video encoder does not support [VideoConfig.level] or [VideoConfig.profile], it fallbacks + * to video encoder default level and default profile. + * + * @param videoConfig Video configuration to set + * + * @throws [Throwable] if configuration can not be applied. + */ + override suspend fun setVideoConfig(videoConfig: VideoConfig) { + pipelineOutput.setVideoCodecConfig(videoConfig) + } + + /** + * Opens the streamer endpoint. + * + * @param descriptor Media descriptor to open + */ + override suspend fun open(descriptor: MediaDescriptor) = pipelineOutput.open(descriptor) + + /** + * Closes the streamer endpoint. + */ + override suspend fun close() = pipelineOutput.close() + + /** + * Starts audio/video stream. + * Stream depends of the endpoint: Audio/video could be write to a file or send to a remote + * device. + * To avoid creating an unresponsive UI, do not call on main thread. + * + * @see [stopStream] + */ + override suspend fun startStream() = pipelineOutput.startStream() + + /** + * Stops audio/video stream. + * + * Internally, it resets audio and video recorders and encoders to get them ready for another + * [startStream] session. It explains why preview could be restarted. + * + * @see [startStream] + */ + override suspend fun stopStream() = pipeline.stopStream() + + /** + * Releases the streamer. + */ + override suspend fun release() { + pipeline.release() + coroutineScope.cancel() + } + + /** + * Adds a bitrate regulator controller. + */ + override var bitrateRegulatorControllerFactory: IBitrateRegulatorController.Factory? + get() = pipelineOutput.bitrateRegulatorControllerFactory + set(value) { + pipelineOutput.bitrateRegulatorControllerFactory = value + } + + companion object Companion { + const val TAG = "SingleStreamer" + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt index fa8071750..298c40b35 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/single/VideoOnlySingleStreamer.kt @@ -26,12 +26,16 @@ import io.github.thibaultbee.streampack.core.elements.encoders.IEncoder import io.github.thibaultbee.streampack.core.elements.endpoints.DynamicEndpointFactory import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal +import io.github.thibaultbee.streampack.core.elements.processing.video.DefaultSurfaceProcessorFactory +import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal import io.github.thibaultbee.streampack.core.elements.sources.video.IVideoSourceInternal import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId import io.github.thibaultbee.streampack.core.elements.sources.video.mediaprojection.MediaProjectionVideoSourceFactory import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.setCameraId +import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider +import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.core.pipelines.inputs.IVideoInput import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo @@ -44,39 +48,53 @@ import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo * @param cameraId the camera id to use. By default, it is the default camera. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ @RequiresPermission(Manifest.permission.CAMERA) suspend fun cameraVideoOnlySingleStreamer( context: Context, cameraId: String = context.defaultCameraId, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( - context, endpointFactory, defaultRotation + context = context, + endpointFactory = endpointFactory, + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setCameraId(cameraId) return streamer } /** - * Creates a [SingleStreamer] with the screen as video source and no audio source. + * Creates a [VideoOnlySingleStreamer] with the screen as video source and no audio source. * * @param context the application context * @param mediaProjection the media projection. It can be obtained with [MediaProjectionManager.getMediaProjection]. Don't forget to call [MediaProjection.stop] when you are done. * @param endpointFactory the [IEndpointInternal.Factory] implementation * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun videoMediaProjectionVideoOnlySingleStreamer( context: Context, mediaProjection: MediaProjection, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( context = context, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) return streamer @@ -89,17 +107,23 @@ suspend fun videoMediaProjectionVideoOnlySingleStreamer( * @param videoSourceFactory the video source factory. If parameter is null, no audio source are set. It can be set later with [VideoOnlySingleStreamer.setVideoSource]. * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ suspend fun VideoOnlySingleStreamer( context: Context, videoSourceFactory: IVideoSourceInternal.Factory, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ): VideoOnlySingleStreamer { val streamer = VideoOnlySingleStreamer( context = context, endpointFactory = endpointFactory, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) streamer.setVideoSource(videoSourceFactory) return streamer @@ -111,19 +135,27 @@ suspend fun VideoOnlySingleStreamer( * @param context the application context * @param endpointFactory the [IEndpointInternal.Factory] implementation. By default, it is a [DynamicEndpointFactory]. * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param surfaceProcessorFactory the [ISurfaceProcessorInternal.Factory] implementation. By default, it is a [DefaultSurfaceProcessorFactory]. + * @param dispatcherProvider the [IDispatcherProvider] implementation. By default, it is a [DispatcherProvider]. */ class VideoOnlySingleStreamer( context: Context, endpointFactory: IEndpointInternal.Factory = DynamicEndpointFactory(), - @RotationValue defaultRotation: Int = context.displayRotation + @RotationValue defaultRotation: Int = context.displayRotation, + surfaceProcessorFactory: ISurfaceProcessorInternal.Factory = DefaultSurfaceProcessorFactory(), + dispatcherProvider: IDispatcherProvider = DispatcherProvider() ) : ISingleStreamer, IVideoSingleStreamer { - private val streamer = SingleStreamer( + private val streamer = SingleStreamerImpl( context = context, - endpointFactory = endpointFactory, withAudio = false, withVideo = true, - defaultRotation = defaultRotation + defaultRotation = defaultRotation, + endpointFactory = endpointFactory, + surfaceProcessorFactory = surfaceProcessorFactory, + dispatcherProvider = dispatcherProvider ) + override val throwableFlow = streamer.throwableFlow override val isOpenFlow = streamer.isOpenFlow override val isStreamingFlow = streamer.isStreamingFlow @@ -136,7 +168,7 @@ class VideoOnlySingleStreamer( override val videoConfigFlow = streamer.videoConfigFlow override val videoEncoder: IEncoder? get() = streamer.videoEncoder - override val videoInput: IVideoInput = streamer.videoInput!! + override val videoInput: IVideoInput = streamer.videoInput /** * Sets the target rotation. @@ -161,8 +193,9 @@ class VideoOnlySingleStreamer( override suspend fun release() = streamer.release() - override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) = - streamer.addBitrateRegulatorController(controllerFactory) - - override fun removeBitrateRegulatorController() = streamer.removeBitrateRegulatorController() + override var bitrateRegulatorControllerFactory: IBitrateRegulatorController.Factory? + get() = streamer.bitrateRegulatorControllerFactory + set(value) { + streamer.bitrateRegulatorControllerFactory = value + } } \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt index 2b8e3428b..44475954a 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/DynamicEndpointTest.kt @@ -6,7 +6,7 @@ import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor import io.github.thibaultbee.streampack.core.elements.utils.DescriptorUtils -import io.github.thibaultbee.streampack.core.elements.utils.FakeFramesWithCloseable +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrameFactory import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.test.runTest import org.junit.Assert.assertFalse @@ -85,7 +85,7 @@ class DynamicEndpointTest { val dynamicEndpoint = DynamicEndpoint(context, Dispatchers.Default, Dispatchers.IO) try { dynamicEndpoint.write( - FakeFramesWithCloseable.create(MediaFormat.MIMETYPE_AUDIO_AAC), + FakeFrameFactory.create(MediaFormat.MIMETYPE_AUDIO_AAC), 0 ) fail("Throwable expected") diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt index 1a5b3dc8d..4eaacf8db 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/TsMuxerTest.kt @@ -21,7 +21,7 @@ import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.TSConst import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.Utils.createFakeServiceInfo -import io.github.thibaultbee.streampack.core.elements.utils.FakeFramesWithCloseable +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrameFactory import io.github.thibaultbee.streampack.core.elements.utils.MockUtils import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -193,7 +193,7 @@ class TsMuxerTest { val tsMux = TsMuxer() try { tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), -1 ) fail() @@ -206,7 +206,7 @@ class TsMuxerTest { val tsMux = TsMuxer() try { tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), -1 ) fail() @@ -231,11 +231,11 @@ class TsMuxerTest { tsMux.addStreams(service, listOf(config))[config]!! tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid ) tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_VIDEO_AVC), streamPid ) } @@ -251,10 +251,10 @@ class TsMuxerTest { tsMux.addStreams(createFakeServiceInfo(), listOf(config))[config]!! tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid ) tsMux.write( - FakeFramesWithCloseable.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid + FakeFrameFactory.create(mimeType = MediaFormat.MIMETYPE_AUDIO_AAC), streamPid ) } } \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/packets/PesTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/packets/PesTest.kt index 389504af9..361cea1dd 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/packets/PesTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/endpoints/composites/muxers/ts/packets/PesTest.kt @@ -21,7 +21,7 @@ import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.data.Stream import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.utils.AssertEqualsBuffersMockMuxerListener -import io.github.thibaultbee.streampack.core.elements.utils.FakeFrames +import io.github.thibaultbee.streampack.core.elements.utils.FakeFrameFactory import io.github.thibaultbee.streampack.core.elements.utils.MockUtils import io.github.thibaultbee.streampack.core.elements.utils.ResourcesUtils import io.github.thibaultbee.streampack.core.elements.utils.pool.ByteBufferPool @@ -54,7 +54,7 @@ class PesTest { MockUtils.mockTimeUtils(1433034) val rawData = ResourcesUtils.readByteBuffer(TEST_SAMPLES_DIR + "pes-video1/raw") - val frame = FakeFrames.create( + val frame = FakeFrameFactory.create( buffer = rawData, pts = 1433334, dts = 1400000, @@ -84,7 +84,7 @@ class PesTest { MockUtils.mockTimeUtils(700000) val rawData = ResourcesUtils.readByteBuffer(TEST_SAMPLES_DIR + "pes-audio1/raw.aac") - val frame = FakeFrames.create( + val frame = FakeFrameFactory.create( buffer = rawData, pts = 1400000, dts = null, @@ -108,7 +108,7 @@ class PesTest { MockUtils.mockTimeUtils(700000) val rawData = ResourcesUtils.readByteBuffer(TEST_SAMPLES_DIR + "pes-audio2/raw.aac") - val frame = FakeFrames.create( + val frame = FakeFrameFactory.create( buffer = rawData, pts = 1400000, dts = null, diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt index 2c579cdcd..393c6a473 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/sources/AudioCaptureUnitTest.kt @@ -18,11 +18,11 @@ package io.github.thibaultbee.streampack.core.elements.sources import android.media.MediaRecorder import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSource import io.github.thibaultbee.streampack.core.elements.utils.StubLogger -import io.github.thibaultbee.streampack.core.elements.utils.StubRawFrameFactory import io.github.thibaultbee.streampack.core.logger.Logger import kotlinx.coroutines.test.runTest import org.junit.Assert import org.junit.Test +import java.nio.ByteBuffer class MicrophoneSourceUnitTest { init { @@ -38,7 +38,7 @@ class MicrophoneSourceUnitTest { } catch (_: Throwable) { } try { - microphoneSource.getAudioFrame(StubRawFrameFactory()) + microphoneSource.fillAudioFrame(ByteBuffer.allocate(microphoneSource.minBufferSize)) Assert.fail() } catch (_: Throwable) { } diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrameFactory.kt similarity index 72% rename from core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt rename to core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrameFactory.kt index 83139d45c..78888f82d 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrames.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/FakeFrameFactory.kt @@ -16,12 +16,12 @@ package io.github.thibaultbee.streampack.core.elements.utils import android.media.MediaFormat +import io.github.thibaultbee.streampack.core.elements.data.Extra import io.github.thibaultbee.streampack.core.elements.data.Frame -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable import java.nio.ByteBuffer import kotlin.random.Random -object FakeFrames { +object FakeFrameFactory { fun create( mimeType: String, buffer: ByteBuffer = ByteBuffer.wrap(Random.nextBytes(1024)), @@ -51,36 +51,32 @@ object FakeFrames { ) ) } - return Frame( + return FakeFrame( buffer, pts, dts, isKeyFrame, - listOf( - ByteBuffer.wrap( - Random.nextBytes(10) + Extra( + listOf( + ByteBuffer.wrap( + Random.nextBytes(10) + ) ) ), format = format ) } -} -object FakeFramesWithCloseable { - fun create( - mimeType: String, - buffer: ByteBuffer = ByteBuffer.wrap(Random.nextBytes(1024)), - pts: Long = Random.nextLong(), - dts: Long? = null, - isKeyFrame: Boolean = false - ) = FrameWithCloseable( - FakeFrames.create( - mimeType, - buffer, - pts, - dts, - isKeyFrame - ), - {/* Nothing to do */ } - ) -} \ No newline at end of file + data class FakeFrame( + override val rawBuffer: ByteBuffer, + override val ptsInUs: Long, + override val dtsInUs: Long?, + override val isKeyFrame: Boolean, + override val extra: Extra?, + override val format: MediaFormat, + ) : Frame { + override fun close() { + // Nothing to do + } + } +} diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt deleted file mode 100644 index a4d24568a..000000000 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/StubRawFrameFactory.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2025 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.elements.utils - -import io.github.thibaultbee.streampack.core.elements.data.RawFrame -import io.github.thibaultbee.streampack.core.elements.utils.pool.IGetOnlyBufferPool -import io.github.thibaultbee.streampack.core.elements.utils.pool.IReadOnlyRawFrameFactory -import java.nio.ByteBuffer - -/** - * Stub buffer pool for testing. - * - * It always returns a new allocated buffer. - */ -class StubRawFrameFactory(private val bufferPool: IGetOnlyBufferPool = StubBufferPool()) : - IReadOnlyRawFrameFactory { - override fun create(bufferSize: Int, timestampInUs: Long): RawFrame { - return RawFrame(bufferPool.get(bufferSize), timestampInUs) { rawFrame -> - // Do nothing - } - } -} \ No newline at end of file diff --git a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt index 9825800e9..489364031 100644 --- a/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt +++ b/core/src/test/java/io/github/thibaultbee/streampack/core/elements/utils/extensions/ByteBufferExtensionsKtTest.kt @@ -117,7 +117,7 @@ class ByteBufferExtensionsKtTest { ) testBuffer.position(2) - val clonedBuffer = testBuffer.clone() + val clonedBuffer = testBuffer.deepCopy() assertArrayEquals( testBuffer.toByteArray(), clonedBuffer.toByteArray() ) @@ -134,12 +134,16 @@ class ByteBufferExtensionsKtTest { val resultBuffers = testBuffer.slices( byteArrayOf(0, 0, 0, 1) ) + assertEquals( 3, resultBuffers.size ) - var resultArray = ByteArray(0) - resultBuffers.forEach { resultArray += it.array() } // concat all arrays - assertArrayEquals(testBuffer.array(), resultArray) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array(), resultBuffer.array()) } @Test @@ -172,7 +176,12 @@ class ByteBufferExtensionsKtTest { assertEquals( 1, resultBuffers.size ) - assertArrayEquals(testBuffer.array(), resultBuffers[0].array()) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array(), resultBuffer.array()) } @Test @@ -189,9 +198,12 @@ class ByteBufferExtensionsKtTest { assertEquals( 1, resultBuffers.size ) - assertArrayEquals( - testBuffer.array().sliceArray(IntRange(3, 8)), resultBuffers[0].array() - ) + + val resultBuffer = ByteBuffer.allocate(resultBuffers.sumOf { it.remaining() }) + resultBuffers.forEach { + resultBuffer.put(it) + } + assertArrayEquals(testBuffer.array().sliceArray(IntRange(3, 8)), resultBuffer.array()) } @Test @@ -263,6 +275,10 @@ class ByteBufferExtensionsKtTest { ) ) val expectedArray = byteArrayOf( + 0, + 0, + 0, + 1, 66, 1, 1, @@ -336,4 +352,19 @@ class ByteBufferExtensionsKtTest { assertEquals(0, prefixBuffer.position()) assertEquals(0, testBuffer.position()) } + + @Test + fun `slice from to test`() { + val testBuffer = ByteBuffer.wrap("ABCDE".toByteArray()) + val slice = testBuffer.slice(1, 4) + + assertArrayEquals( + "BCDE".toByteArray(), slice.toByteArray() + ) + assertEquals(0, slice.position()) + assertEquals(4, slice.limit()) + + assertEquals(0, testBuffer.position()) + assertEquals(5, testBuffer.limit()) + } } \ No newline at end of file diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/data/storage/DataStoreRepository.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/data/storage/DataStoreRepository.kt index 9feac88bd..a170631d3 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/data/storage/DataStoreRepository.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/data/storage/DataStoreRepository.kt @@ -135,11 +135,8 @@ class DataStoreRepository( val filename = preferences[stringPreferencesKey(context.getString(R.string.file_endpoint_key))] ?: "StreamPack" - UriMediaDescriptor( - context, - context.createVideoContentUri( - filename.appendIfNotEndsWith(FileExtension.fromEndpointType(endpointType).extension) - ) + context.createVideoContentUri( + filename.appendIfNotEndsWith(FileExtension.fromEndpointType(endpointType).extension) ) } @@ -167,7 +164,7 @@ class DataStoreRepository( EndpointType.RTMP -> { val url = preferences[stringPreferencesKey(context.getString(R.string.rtmp_server_url_key))] - ?: context.getString(R.string.default_rtmp_url) + ?: context.getString(R.string.rtmp_default_url) UriMediaDescriptor(context, url) } } @@ -175,19 +172,71 @@ class DataStoreRepository( val bitrateRegulatorConfigFlow: Flow = dataStore.data.map { preferences -> + val endpointTypeId = + preferences[stringPreferencesKey(context.getString(R.string.endpoint_type_key))]?.toInt() + ?: EndpointType.SRT.id + val isBitrateRegulatorEnable = - preferences[booleanPreferencesKey(context.getString(R.string.srt_server_enable_bitrate_regulation_key))] + preferences[booleanPreferencesKey( + context.getString( + when (endpointTypeId) { + EndpointType.SRT.id -> { + R.string.srt_server_enable_bitrate_regulation_key + } + + EndpointType.RTMP.id -> { + R.string.rtmp_server_enable_bitrate_regulation_key + } + + else -> { + throw IllegalArgumentException("Unknown endpoint type") + } + } + ) + )] ?: true if (!isBitrateRegulatorEnable) { return@map null } val videoMinBitrate = - preferences[intPreferencesKey(context.getString(R.string.srt_server_video_min_bitrate_key))]?.toInt() + preferences[intPreferencesKey( + context.getString( + when (endpointTypeId) { + EndpointType.SRT.id -> { + R.string.srt_server_video_min_bitrate_key + } + + EndpointType.RTMP.id -> { + R.string.rtmp_server_video_min_bitrate_key + } + + else -> { + throw IllegalArgumentException("Unknown endpoint type") + } + } + ) + )] ?.times(1000) ?: 300000 val videoMaxBitrate = - preferences[intPreferencesKey(context.getString(R.string.srt_server_video_target_bitrate_key))]?.toInt() + preferences[intPreferencesKey( + context.getString( + when (endpointTypeId) { + EndpointType.SRT.id -> { + R.string.srt_server_video_target_bitrate_key + } + + EndpointType.RTMP.id -> { + R.string.rtmp_server_video_target_bitrate_key + } + + else -> { + throw IllegalArgumentException("Unknown endpoint type") + } + } + ) + )] ?.times(1000) ?: 10000000 BitrateRegulatorConfig( diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt index acd8f2a14..3f4296514 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewFragment.kt @@ -34,15 +34,16 @@ import io.github.thibaultbee.streampack.app.R import io.github.thibaultbee.streampack.app.databinding.MainFragmentBinding import io.github.thibaultbee.streampack.app.utils.DialogUtils import io.github.thibaultbee.streampack.app.utils.PermissionManager +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource -import io.github.thibaultbee.streampack.core.streamers.lifecycle.StreamerViewModelLifeCycleObserver +import io.github.thibaultbee.streampack.core.streamers.lifecycle.StreamerLifeCycleObserver import io.github.thibaultbee.streampack.ui.views.PreviewView import kotlinx.coroutines.launch class PreviewFragment : Fragment(R.layout.main_fragment) { private lateinit var binding: MainFragmentBinding - private var previousStreamerLifecycleObserver: StreamerViewModelLifeCycleObserver? = null + private var previousStreamerLifecycleObserver: StreamerLifeCycleObserver? = null private val previewViewModel: PreviewViewModel by viewModels { PreviewViewModelFactory(requireActivity().application) @@ -110,7 +111,7 @@ class PreviewFragment : Fragment(R.layout.main_fragment) { if (previousStreamerLifecycleObserver != null) { lifecycle.removeObserver(previousStreamerLifecycleObserver!!) } - val newObserver = StreamerViewModelLifeCycleObserver(streamer).apply { + val newObserver = StreamerLifeCycleObserver(streamer).apply { previousStreamerLifecycleObserver = this } lifecycle.addObserver(newObserver) @@ -182,15 +183,17 @@ class PreviewFragment : Fragment(R.layout.main_fragment) { private fun inflateStreamerPreview(streamer: IWithVideoSource) { val preview = binding.preview // Set camera settings button when camera is started - preview.listener = object : PreviewView.Listener { + preview.listener = object : PreviewView.PreviewListener { override fun onPreviewStarted() { Log.i(TAG, "Preview started") } + } - override fun onZoomRationOnPinchChanged(zoomRatio: Float) { - previewViewModel.onZoomRationOnPinchChanged() + preview.setZoomListener(object : CameraSettings.Zoom.OnZoomChangedListener { + override fun onZoomChanged(zoomRatio: Float) { + previewViewModel.onZoomChanged() } - } + }) // Wait till streamer exists to set it to the SurfaceView. lifecycleScope.launch { diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index bf57eed0c..70d4b40b2 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.app.ui.main import android.Manifest +import android.annotation.SuppressLint import android.app.Application import android.content.Context import android.content.pm.PackageManager @@ -43,6 +44,10 @@ import io.github.thibaultbee.streampack.app.utils.setNextCameraId import io.github.thibaultbee.streampack.app.utils.toggleBackToFront import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor import io.github.thibaultbee.streampack.core.elements.endpoints.MediaSinkType +import io.github.thibaultbee.streampack.core.elements.metrics.WithEndpointMetrics +import io.github.thibaultbee.streampack.core.elements.metrics.metricsFlow +import io.github.thibaultbee.streampack.app.utils.formatBitrate +import io.github.thibaultbee.streampack.core.elements.metrics.writtenBitrateInBps import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.IAudioRecordSource import io.github.thibaultbee.streampack.core.elements.sources.audio.audiorecord.MicrophoneSourceFactory import io.github.thibaultbee.streampack.core.elements.sources.video.bitmap.BitmapSourceFactory @@ -52,43 +57,58 @@ import io.github.thibaultbee.streampack.core.elements.sources.video.camera.Camer import io.github.thibaultbee.streampack.core.elements.sources.video.camera.ICameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.cameraManager import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.defaultCameraId +import io.github.thibaultbee.streampack.core.interfaces.IWithAudioSource import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource import io.github.thibaultbee.streampack.core.interfaces.releaseBlocking import io.github.thibaultbee.streampack.core.interfaces.startStream +import io.github.thibaultbee.streampack.core.pipelines.StreamerPipeline +import io.github.thibaultbee.streampack.core.regulator.controllers.intervalBitrateRegulatorControllerFactory +import io.github.thibaultbee.streampack.core.streamers.single.IAudioSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.IVideoSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.withAudio +import io.github.thibaultbee.streampack.core.streamers.single.withVideo import io.github.thibaultbee.streampack.core.utils.extensions.isClosedException -import io.github.thibaultbee.streampack.ext.srt.regulator.controllers.DefaultSrtBitrateRegulatorController +import io.github.thibaultbee.streampack.ext.srt.regulator.controllers.intervalSrtBitrateRegulatorControllerFactory import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.Job +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.emptyFlow import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.flatMapLatest import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +@OptIn(ExperimentalCoroutinesApi::class) class PreviewViewModel(private val application: Application) : ObservableViewModel() { private val storageRepository = DataStoreRepository(application, application.dataStore) private val rotationRepository = RotationRepository.getInstance(application) private val defaultDispatcher = Dispatchers.IO - private val buildStreamerUseCase = BuildStreamerUseCase(application, storageRepository) + private val buildStreamerUseCase = BuildStreamerUseCase(application) private val streamerFlow = MutableStateFlow( - SingleStreamer( - application, - runBlocking { storageRepository.isAudioEnableFlow.first() }) // TODO avoid runBlocking + buildFirstStreamer(runBlocking { storageRepository.isAudioEnableFlow.first() }) ) - private val streamer: SingleStreamer + + private val streamer: IVideoSingleStreamer get() = streamerFlow.value + private val audioStreamer: IAudioSingleStreamer? + get() = streamer as? IAudioSingleStreamer? + val streamerLiveData = streamerFlow.asLiveData() /** @@ -111,10 +131,10 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod val requiredPermissions: List get() { val permissions = mutableListOf() - if (streamer.videoInput?.sourceFlow is ICameraSource) { + if (streamer.videoInput.sourceFlow is ICameraSource) { permissions.add(Manifest.permission.CAMERA) } - if (streamer.audioInput?.sourceFlow?.value is IAudioRecordSource) { + if (audioStreamer?.audioInput?.sourceFlow?.value is IAudioRecordSource) { permissions.add(Manifest.permission.RECORD_AUDIO) } storageRepository.endpointDescriptorFlow.asLiveData().value?.let { @@ -139,6 +159,12 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod private val _isTryingConnectionLiveData = MutableLiveData() val isTryingConnectionLiveData: LiveData = _isTryingConnectionLiveData + private val _instantBitrateLiveData = MutableLiveData("0 bps") + val instantBitrateLiveData: LiveData = _instantBitrateLiveData + + private val _packetLossLiveData = MutableLiveData("0 / 0 (0%)") + val packetLossLiveData: LiveData = _packetLossLiveData + private val videoSourceMutex = Mutex() private var startStreamJob: Job? = null @@ -149,7 +175,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod // Set audio source and video source if (streamer.withAudio) { Log.i(TAG, "Audio source is enabled. Setting audio source") - streamer.setAudioSource(MicrophoneSourceFactory()) + audioStreamer!!.setAudioSource(MicrophoneSourceFactory()) } else { Log.i(TAG, "Audio source is disabled") } @@ -167,7 +193,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod // TODO: cancel jobs linked to previous streamer viewModelScope.launch { - streamer.videoInput?.sourceFlow?.collect { + streamer.videoInput.sourceFlow.collect { notifySourceChanged() } } @@ -190,6 +216,12 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod Log.i(TAG, "Streamer is opened: $it") } } + viewModelScope.launch { + val audioInput = (streamer as IWithAudioSource).audioInput + audioInput.sourceFlow.filterNotNull().first() + delay(3000L) + audioInput.startCapture() + } viewModelScope.launch { streamer.isStreamingFlow .collect { isStreaming -> @@ -226,7 +258,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod ) == PackageManager.PERMISSION_GRANTED ) { try { - streamer.setAudioConfig(config) + audioStreamer!!.setAudioConfig(config) } catch (t: Throwable) { Log.e(TAG, "setAudioConfig failed", t) _streamerErrorLiveData.postValue("setAudioConfig: ${t.message ?: "Unknown error"}") @@ -245,9 +277,37 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } } + viewModelScope.launch { + streamerFlow.flatMapLatest { streamer -> + val endpointMetrics = streamer.endpoint as? WithEndpointMetrics<*> + endpointMetrics?.metricsFlow() ?: emptyFlow() + }.collect { metrics -> + val formattedBitrate = metrics.instant.writtenBitrateInBps.formatBitrate() + _instantBitrateLiveData.postValue(formattedBitrate) + val lost = + metrics.cumulative.packetsWriteDropped + metrics.cumulative.packetsWriteLost + val total = metrics.cumulative.packetsWritten + lost + val packetLossStr = if (total == 0L) { + "0 / 0 (0%)" + } else { + val percentage = (lost * 100f / total) + val formattedPercentage = String.format(java.util.Locale.US, "%.2f", percentage) + "$lost / $total ($formattedPercentage%)" + } + _packetLossLiveData.postValue(packetLossStr) + } + } } - fun onZoomRationOnPinchChanged() { + private fun buildFirstStreamer(isAudioEnable: Boolean): IVideoSingleStreamer { + return if (isAudioEnable) { + SingleStreamer(application, audioInputMode = StreamerPipeline.AudioInputMode.PUSH) + } else { + VideoOnlySingleStreamer(application) + } + } + + fun onZoomChanged() { notifyPropertyChanged(BR.zoomRatio) } @@ -255,7 +315,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun configureAudio() { viewModelScope.launch { try { - storageRepository.audioConfigFlow.first()?.let { streamer.setAudioConfig(it) } + storageRepository.audioConfigFlow.first()?.let { audioStreamer?.setAudioConfig(it) } ?: Log.i( TAG, "Audio is disabled" @@ -271,7 +331,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun initializeVideoSource() { viewModelScope.launch { videoSourceMutex.withLock { - if (streamer.videoInput?.sourceFlow?.value == null) { + if (streamer.videoInput.sourceFlow.value == null) { streamer.setVideoSource(CameraSourceFactory(defaultCameraId)) } else { Log.i(TAG, "Camera source already set") @@ -287,16 +347,32 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod val descriptor = storageRepository.endpointDescriptorFlow.first() streamer.startStream(descriptor) - if (descriptor.type.sinkType == MediaSinkType.SRT) { + if ((descriptor.type.sinkType == MediaSinkType.RTMP) || (descriptor.type.sinkType == MediaSinkType.SRT)) { val bitrateRegulatorConfig = storageRepository.bitrateRegulatorConfigFlow.first() if (bitrateRegulatorConfig != null) { Log.i(TAG, "Add bitrate regulator controller") - streamer.addBitrateRegulatorController( - DefaultSrtBitrateRegulatorController.Factory( - bitrateRegulatorConfig = bitrateRegulatorConfig - ) - ) + val controllerFactory = + when (descriptor.type.sinkType) { + MediaSinkType.RTMP -> { + intervalBitrateRegulatorControllerFactory( + bitrateRegulatorConfig = bitrateRegulatorConfig + ) + } + + MediaSinkType.SRT -> { + intervalSrtBitrateRegulatorControllerFactory( + bitrateRegulatorConfig = bitrateRegulatorConfig + ) + } + + else -> { + null + } + } + controllerFactory?.let { + streamer.bitrateRegulatorControllerFactory = it + } ?: Log.e(TAG, "Controller factory is null") } } } catch (e: CancellationException) { @@ -319,7 +395,6 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod try { streamer.stopStream() streamer.close() - streamer.removeBitrateRegulatorController() } catch (e: Throwable) { Log.e(TAG, "stopStream failed", e) } @@ -327,7 +402,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } fun setMute(isMuted: Boolean) { - streamer.audioInput?.isMuted = isMuted + audioStreamer?.audioInput?.isMuted = isMuted } @RequiresPermission(Manifest.permission.CAMERA) @@ -337,7 +412,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod * exception instead of crashing. You can either catch the exception or check if the * configuration is valid for the new camera with [Context.isFpsSupported]. */ - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is ICameraSource) { viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { @@ -357,7 +432,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod */ viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is ICameraSource) { streamer.setNextCameraId(application) } @@ -369,7 +444,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod fun toggleVideoSource() { viewModelScope.launch(defaultDispatcher) { videoSourceMutex.withLock { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value val nextSource = when (videoSource) { is ICameraSource -> { BitmapSourceFactory(testBitmap) @@ -390,9 +465,11 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } - val isCameraSource = streamer.videoInput?.sourceFlow?.map { it is ICameraSource }?.asLiveData() + val isCameraSource = streamer.videoInput.sourceFlow?.map { it is ICameraSource }?.asLiveData() val isFlashAvailable = MutableLiveData(false) + + @SuppressLint("MissingPermission") fun toggleFlash() { cameraSettings?.let { viewModelScope.launch { @@ -402,6 +479,8 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } val isAutoWhiteBalanceAvailable = MutableLiveData(false) + + @SuppressLint("MissingPermission") fun toggleAutoWhiteBalanceMode() { cameraSettings?.let { settings -> val awbModes = settings.whiteBalance.availableAutoModes @@ -429,6 +508,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod 0f } } + @SuppressLint("MissingPermission") set(value) { cameraSettings?.let { settings -> settings.exposure.let { @@ -472,6 +552,8 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } val isAutoFocusModeAvailable = MutableLiveData(false) + + @SuppressLint("MissingPermission") fun toggleAutoFocusMode() { cameraSettings?.let { val afModes = it.focus.availableAutoModes @@ -500,6 +582,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod 0f } } + @SuppressLint("MissingPermission") set(value) { cameraSettings?.let { settings -> settings.focus.let { @@ -514,7 +597,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } private fun notifySourceChanged() { - val videoSource = streamer.videoInput?.sourceFlow?.value ?: return + val videoSource = streamer.videoInput.sourceFlow.value ?: return if (videoSource is ICameraSource) { notifyCameraChanged(videoSource) } else { @@ -526,6 +609,7 @@ class PreviewViewModel(private val application: Application) : ObservableViewMod } } + @SuppressLint("MissingPermission") private fun notifyCameraChanged(videoSource: ICameraSource) { val settings = videoSource.settings // Set optical stabilization first diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt index 417f6614c..beef3a8f9 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/usecases/BuildStreamerUseCase.kt @@ -4,15 +4,17 @@ import android.Manifest import android.content.Context import android.content.pm.PackageManager import androidx.core.app.ActivityCompat -import io.github.thibaultbee.streampack.app.data.storage.DataStoreRepository +import io.github.thibaultbee.streampack.core.streamers.single.IAudioSingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.IVideoSingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.withAudio class BuildStreamerUseCase( - private val context: Context, - private val dataStoreRepository: DataStoreRepository + private val context: Context ) { /** - * Build a new [SingleStreamer] based on audio and video preferences. + * Build a new [IVideoSingleStreamer] based on audio and video preferences. * * Only create a new streamer if the previous one is not the same type. * @@ -21,28 +23,37 @@ class BuildStreamerUseCase( * @param previousStreamer Previous streamer to check if we need to create a new one. */ suspend operator fun invoke( - previousStreamer: SingleStreamer, + previousStreamer: IVideoSingleStreamer, isAudioEnable: Boolean - ): SingleStreamer { + ): IVideoSingleStreamer { if (previousStreamer.withAudio != isAudioEnable) { - return SingleStreamer(context, isAudioEnable).apply { - // Get previous streamer config if any + previousStreamer.release() + + val streamer = if (isAudioEnable) { + SingleStreamer(context) + } else { + VideoOnlySingleStreamer(context) + } + + // Get previous streamer config if any + if ((previousStreamer is IAudioSingleStreamer) && (streamer is IAudioSingleStreamer)) { val audioConfig = previousStreamer.audioConfigFlow.value - val videoConfig = previousStreamer.videoConfigFlow.value - if ((audioConfig != null && isAudioEnable)) { + if (audioConfig != null) { if (ActivityCompat.checkSelfPermission( context, Manifest.permission.RECORD_AUDIO ) == PackageManager.PERMISSION_GRANTED ) { - setAudioConfig(audioConfig) + streamer.setAudioConfig(audioConfig) } } - if (videoConfig != null) { - setVideoConfig(videoConfig) - } - previousStreamer.release() } + + val videoConfig = previousStreamer.videoConfigFlow.value + if (videoConfig != null) { + streamer.setVideoConfig(videoConfig) + } + return streamer } return previousStreamer } diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/settings/SettingsFragment.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/settings/SettingsFragment.kt index 2a8079eda..04a3b1b1b 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/settings/SettingsFragment.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/settings/SettingsFragment.kt @@ -131,18 +131,30 @@ class SettingsFragment : PreferenceFragmentCompat() { this.findPreference(getString(R.string.srt_server_port_key))!! } - private val serverEnableBitrateRegulationPreference: SwitchPreference by lazy { + private val srtServerEnableBitrateRegulationPreference: SwitchPreference by lazy { this.findPreference(getString(R.string.srt_server_enable_bitrate_regulation_key))!! } - private val serverTargetVideoBitratePreference: SeekBarPreference by lazy { + private val srtServerTargetVideoBitratePreference: SeekBarPreference by lazy { this.findPreference(getString(R.string.srt_server_video_target_bitrate_key))!! } - private val serverMinVideoBitratePreference: SeekBarPreference by lazy { + private val srtServerMinVideoBitratePreference: SeekBarPreference by lazy { this.findPreference(getString(R.string.srt_server_video_min_bitrate_key))!! } + private val rtmpServerEnableBitrateRegulationPreference: SwitchPreference by lazy { + this.findPreference(getString(R.string.rtmp_server_enable_bitrate_regulation_key))!! + } + + private val rtmpServerTargetVideoBitratePreference: SeekBarPreference by lazy { + this.findPreference(getString(R.string.rtmp_server_video_target_bitrate_key))!! + } + + private val rtmpServerMinVideoBitratePreference: SeekBarPreference by lazy { + this.findPreference(getString(R.string.rtmp_server_video_min_bitrate_key))!! + } + private val fileNamePreference: EditTextPreference by lazy { this.findPreference(getString(R.string.file_name_key))!! } @@ -417,26 +429,50 @@ class SettingsFragment : PreferenceFragmentCompat() { editText.filters = arrayOf(InputFilter.LengthFilter(5)) } - serverTargetVideoBitratePreference.isVisible = - serverEnableBitrateRegulationPreference.isChecked - serverMinVideoBitratePreference.isVisible = - serverEnableBitrateRegulationPreference.isChecked - serverEnableBitrateRegulationPreference.setOnPreferenceChangeListener { _, newValue -> - serverTargetVideoBitratePreference.isVisible = newValue as Boolean - serverMinVideoBitratePreference.isVisible = newValue + srtServerTargetVideoBitratePreference.isVisible = + srtServerEnableBitrateRegulationPreference.isChecked + srtServerMinVideoBitratePreference.isVisible = + srtServerEnableBitrateRegulationPreference.isChecked + srtServerEnableBitrateRegulationPreference.setOnPreferenceChangeListener { _, newValue -> + srtServerTargetVideoBitratePreference.isVisible = newValue as Boolean + srtServerMinVideoBitratePreference.isVisible = newValue + true + } + + srtServerTargetVideoBitratePreference.setOnPreferenceChangeListener { _, newValue -> + if ((newValue as Int) < srtServerMinVideoBitratePreference.value) { + srtServerMinVideoBitratePreference.value = newValue + } + true + } + + srtServerMinVideoBitratePreference.setOnPreferenceChangeListener { _, newValue -> + if ((newValue as Int) > srtServerTargetVideoBitratePreference.value) { + srtServerTargetVideoBitratePreference.value = newValue + } + true + } + + rtmpServerTargetVideoBitratePreference.isVisible = + rtmpServerEnableBitrateRegulationPreference.isChecked + rtmpServerMinVideoBitratePreference.isVisible = + rtmpServerEnableBitrateRegulationPreference.isChecked + rtmpServerEnableBitrateRegulationPreference.setOnPreferenceChangeListener { _, newValue -> + rtmpServerTargetVideoBitratePreference.isVisible = newValue as Boolean + rtmpServerMinVideoBitratePreference.isVisible = newValue true } - serverTargetVideoBitratePreference.setOnPreferenceChangeListener { _, newValue -> - if ((newValue as Int) < serverMinVideoBitratePreference.value) { - serverMinVideoBitratePreference.value = newValue + rtmpServerTargetVideoBitratePreference.setOnPreferenceChangeListener { _, newValue -> + if ((newValue as Int) < rtmpServerMinVideoBitratePreference.value) { + rtmpServerMinVideoBitratePreference.value = newValue } true } - serverMinVideoBitratePreference.setOnPreferenceChangeListener { _, newValue -> - if ((newValue as Int) > serverTargetVideoBitratePreference.value) { - serverTargetVideoBitratePreference.value = newValue + rtmpServerMinVideoBitratePreference.setOnPreferenceChangeListener { _, newValue -> + if ((newValue as Int) > rtmpServerTargetVideoBitratePreference.value) { + rtmpServerTargetVideoBitratePreference.value = newValue } true } @@ -459,7 +495,7 @@ class SettingsFragment : PreferenceFragmentCompat() { // Update file extension if (endpoint.hasFileCapabilities) { // Remove previous extension - FileExtension.entries.forEach { + FileExtension.entries.forEach { _ -> fileNamePreference.text = fileNamePreference.text?.substringBeforeLast(".") } // Add correct extension diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt index ec45c6943..c565ac65d 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/Extensions.kt @@ -18,13 +18,13 @@ package io.github.thibaultbee.streampack.app.utils import android.Manifest import android.content.ContentValues import android.content.Context -import android.net.Uri -import android.os.Build import android.provider.MediaStore import android.util.Range import androidx.annotation.RequiresPermission import androidx.datastore.preferences.preferencesDataStore import io.github.thibaultbee.streampack.app.ApplicationConstants.userPrefName +import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor +import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.videoUriMediaDescriptor import io.github.thibaultbee.streampack.core.elements.sources.video.camera.ICameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.backCameras import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.cameraManager @@ -37,7 +37,7 @@ import io.github.thibaultbee.streampack.core.interfaces.setCameraId @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.setNextCameraId(context: Context) { val cameras = context.cameraManager.cameras - val videoSource = videoInput?.sourceFlow?.value + val videoSource = videoInput.sourceFlow.value val newCameraId = if (videoSource is ICameraSource) { val currentCameraIndex = cameras.indexOf(videoSource.cameraId) @@ -52,7 +52,7 @@ suspend fun IWithVideoSource.setNextCameraId(context: Context) { @RequiresPermission(Manifest.permission.CAMERA) suspend fun IWithVideoSource.toggleBackToFront(context: Context) { val cameraManager = context.cameraManager - val videoSource = videoInput?.sourceFlow?.value + val videoSource = videoInput.sourceFlow.value val cameras = if (videoSource is ICameraSource) { if (cameraManager.isBackCamera(videoSource.cameraId)) { cameraManager.frontCameras @@ -77,7 +77,7 @@ val Context.dataStore by preferencesDataStore( name = userPrefName ) -fun Context.createVideoContentUri(name: String): Uri { +fun Context.createVideoContentUri(name: String): UriMediaDescriptor { val videoDetails = ContentValues().apply { put(MediaStore.Video.Media.TITLE, name) put( @@ -85,42 +85,7 @@ fun Context.createVideoContentUri(name: String): Uri { name ) } - - val resolver = this.contentResolver - val collection = - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - MediaStore.Video.Media.getContentUri( - MediaStore.VOLUME_EXTERNAL_PRIMARY - ) - } else { - MediaStore.Video.Media.EXTERNAL_CONTENT_URI - } - - return resolver.insert(collection, videoDetails) - ?: throw Exception("Unable to create video file: $name") -} - -fun Context.createAudioContentUri(name: String): Uri { - val audioDetails = ContentValues().apply { - put(MediaStore.Audio.Media.TITLE, name) - put( - MediaStore.Audio.Media.DISPLAY_NAME, - name - ) - } - - val resolver = this.contentResolver - val collection = - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - MediaStore.Audio.Media.getContentUri( - MediaStore.VOLUME_EXTERNAL_PRIMARY - ) - } else { - MediaStore.Audio.Media.EXTERNAL_CONTENT_URI - } - - return resolver.insert(collection, audioDetails) - ?: throw Exception("Unable to create audio file: $name") + return videoUriMediaDescriptor(this, videoDetails) } fun String.appendIfNotEndsWith(suffix: String): String { @@ -133,3 +98,11 @@ fun String.appendIfNotEndsWith(suffix: String): String { val Range<*>.isEmpty: Boolean get() = upper == lower + +fun Long.formatBitrate(): String { + return when { + this >= 1_000_000 -> String.format(java.util.Locale.US, "%.2f Mbps", this / 1_000_000.0) + this >= 1_000 -> String.format(java.util.Locale.US, "%.2f kbps", this / 1_000.0) + else -> "$this bps" + } +} diff --git a/demos/camera/src/main/res/layout/main_fragment.xml b/demos/camera/src/main/res/layout/main_fragment.xml index 03f03eef6..36a8a4bb5 100644 --- a/demos/camera/src/main/res/layout/main_fragment.xml +++ b/demos/camera/src/main/res/layout/main_fragment.xml @@ -1,7 +1,8 @@ + xmlns:app="http://schemas.android.com/apk/res-auto" + xmlns:tools="http://schemas.android.com/tools"> @@ -27,6 +28,31 @@ app:position="center" app:scaleMode="fill" /> + + + + + + + rtmp_server_key RTMP Server rtmp_server_url_key - rtmp://192.168.1.192/s/streamKey + rtmp://192.168.1.192/s/streamKey + rtmp_server_enable_bitrate_regulation_key + Enable bitrate regulation + rtmp_server_video_target_bitrate_key + Video target bitrate (kb/s) + rtmp_server_video_min_bitrate_key + Video minimum bitrate (kb/s) + URL file_endpoint_key diff --git a/demos/camera/src/main/res/xml/root_preferences.xml b/demos/camera/src/main/res/xml/root_preferences.xml index e47de9ad3..58a3a89b7 100644 --- a/demos/camera/src/main/res/xml/root_preferences.xml +++ b/demos/camera/src/main/res/xml/root_preferences.xml @@ -167,11 +167,33 @@ app:title="@string/rtmp_server"> + + + + + = _isOpenFlow.asStateFlow() @@ -126,12 +123,11 @@ sealed class FlvEndpoint( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame val startUpTimestamp = getStartUpTimestamp(frame.ptsInUs) val ts = (frame.ptsInUs - startUpTimestamp) / 1000 - flvTagBuilder.write(closeableFrame, ts.toInt(), streamPid) + flvTagBuilder.write(frame, ts.toInt(), streamPid) } override suspend fun addStreams(streamConfigs: List): Map { diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt index b0d1013b5..54df39c08 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvAudioDataFactory.kt @@ -24,6 +24,7 @@ import io.github.komedia.komuxer.flv.tags.audio.ExtendedAudioDataFactory import io.github.komedia.komuxer.flv.tags.audio.codedFrame import io.github.komedia.komuxer.flv.tags.audio.sequenceStart import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.get import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.utils.av.audio.opus.OpusCsdParser import io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.muxer.AudioFlvMuxerInfo @@ -72,7 +73,7 @@ internal class FlvAudioDataFactory { val flvDatas = mutableListOf() if (withSequenceStart) { - val decoderConfigurationRecordBuffer = frame.extra!![0] + val decoderConfigurationRecordBuffer = frame.extra!!.get(0) flvDatas.add( aacAudioDataFactory.sequenceStart( decoderConfigurationRecordBuffer @@ -155,7 +156,7 @@ internal class FlvAudioDataFactory { return FlvExtendedAudioDataFactory( ExtendedAudioDataFactory(AudioFourCC.AAC), onSequenceStart = { frame -> - frame.extra!![0] + frame.extra!!.get(0) } ) } @@ -165,7 +166,7 @@ internal class FlvAudioDataFactory { ExtendedAudioDataFactory(AudioFourCC.OPUS), onSequenceStart = { frame -> frame.extra?.let { - OpusCsdParser.findIdentificationHeader(it[0]) + OpusCsdParser.findIdentificationHeader(it.get(0)) } } ) diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt index 14df85e5b..afb98d8f9 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvTagBuilder.kt @@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.m import io.github.komedia.komuxer.flv.tags.FLVTag import io.github.komedia.komuxer.flv.tags.script.Metadata import io.github.komedia.komuxer.logger.KomuxerLogger -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.AudioCodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig @@ -106,21 +106,19 @@ class FlvTagBuilder(val channel: ChannelWithCloseableData) { } suspend fun write( - closeableFrame: FrameWithCloseable, + frame: Frame, ts: Int, streamPid: Int ) { if (ts < 0) { Logger.w( TAG, - "Negative timestamp $ts for frame ${closeableFrame.frame}. Frame will be dropped." + "Negative timestamp $ts for frame $frame. Frame will be dropped." ) - closeableFrame.close() + frame.close() return } - val frame = closeableFrame.frame - val flvDatas = when (streamPid) { AUDIO_STREAM_PID -> audioStream?.create(frame) ?: throw IllegalStateException("Audio stream not added") @@ -133,14 +131,14 @@ class FlvTagBuilder(val channel: ChannelWithCloseableData) { flvDatas.forEachIndexed { index, flvData -> if (index == flvDatas.lastIndex) { // Pass the close callback on the last element - channel.send(FLVTag(ts, flvData), { closeableFrame.close() }) + channel.send(FLVTag(ts, flvData), { frame.close() }) } else { channel.send(FLVTag(ts, flvData)) } } } - companion object Companion { + companion object { private const val TAG = "FlvTagBuilder" private const val AUDIO_STREAM_PID = 0 diff --git a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt index 6b541bc92..dccc05e0d 100644 --- a/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt +++ b/extensions/flv/src/main/java/io/github/thibaultbee/streampack/ext/flv/elements/endpoints/composites/muxer/utils/FlvVideoDataFactory.kt @@ -28,6 +28,8 @@ import io.github.komedia.komuxer.flv.tags.video.VideoFrameType import io.github.komedia.komuxer.flv.tags.video.codedFrame import io.github.komedia.komuxer.flv.tags.video.sequenceStart import io.github.thibaultbee.streampack.core.elements.data.Frame +import io.github.thibaultbee.streampack.core.elements.data.extra +import io.github.thibaultbee.streampack.core.elements.data.get import io.github.thibaultbee.streampack.core.elements.encoders.VideoCodecConfig import io.github.thibaultbee.streampack.core.elements.utils.av.video.avc.AVCDecoderConfigurationRecord import io.github.thibaultbee.streampack.core.elements.utils.av.video.hevc.HEVCDecoderConfigurationRecord @@ -64,10 +66,11 @@ internal class FlvVideoDataFactory { VideoFrameType.INTER } if (frame.isKeyFrame && withSequenceStart) { + val extra = frame.extra!!.extra val decoderConfigurationRecordBuffer = AVCDecoderConfigurationRecord.fromParameterSets( - frame.extra!![0], - frame.extra!![1] + extra[0], + extra[1] ).toByteBuffer() flvDatas.add( factory.sequenceStart( @@ -185,10 +188,11 @@ internal class FlvVideoDataFactory { private fun createHEVCFactory(): IVideoDataFactory { return FlvExtendedVideoDataFactory(HEVCExtendedVideoDataFactory()) { frame -> // Extra is VPS, SPS, PPS + val extra = frame.extra!!.extra HEVCDecoderConfigurationRecord.fromParameterSets( - frame.extra!![0], - frame.extra!![1], - frame.extra!![2] + extra[0], + extra[1], + extra[2] ).toByteBuffer() } } @@ -196,7 +200,7 @@ internal class FlvVideoDataFactory { private fun createAV1Factory(): IVideoDataFactory { return FlvExtendedVideoDataFactory(ExtendedVideoDataFactory(VideoFourCC.AV1)) { frame -> // Extra is AV1CodecConfigurationRecord - frame.extra!![0] + frame.extra!!.get(0) } } diff --git a/extensions/rtmp/build.gradle.kts b/extensions/rtmp/build.gradle.kts index beca775cf..1be6ad482 100644 --- a/extensions/rtmp/build.gradle.kts +++ b/extensions/rtmp/build.gradle.kts @@ -14,7 +14,7 @@ dependencies { implementation(project(":streampack-core")) implementation(project(":streampack-flv")) - implementation(libs.komedia.komuxer.rtmp) + api(libs.komedia.komuxer.rtmp) implementation(libs.kotlinx.coroutines.android) implementation(libs.androidx.core.ktx) diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/configuration/mediadescriptor/RtmpMediaDescriptor.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/configuration/mediadescriptor/RtmpMediaDescriptor.kt index 2815c09a8..a35683c69 100644 --- a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/configuration/mediadescriptor/RtmpMediaDescriptor.kt +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/configuration/mediadescriptor/RtmpMediaDescriptor.kt @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.ext.rtmp.configuration.mediadescriptor import android.net.Uri import androidx.core.net.toUri +import io.github.komedia.komuxer.rtmp.messages.command.ConnectObjectBuilder import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.UriMediaDescriptor import io.github.thibaultbee.streampack.core.elements.endpoints.MediaContainerType @@ -26,23 +27,22 @@ import java.security.InvalidParameterException /** * Creates a RTMP connection descriptor from an [descriptor]. * If the descriptor is already a [RtmpMediaDescriptor], it will be returned as is. - * If the descriptor is an [UriMediaDescriptor], it will be converted to a [RtmpMediaDescriptor] with default [TSServiceInfo]. + * If the descriptor is an [UriMediaDescriptor], it will be converted to a [RtmpMediaDescriptor]. * Otherwise, an [InvalidParameterException] will be thrown. */ fun RtmpMediaDescriptor(descriptor: MediaDescriptor) = - when (descriptor) { - is RtmpMediaDescriptor -> descriptor - is UriMediaDescriptor -> { - RtmpMediaDescriptor.fromUri(descriptor.uri) - } - - else -> throw InvalidParameterException("Invalid descriptor ${descriptor::class.java.simpleName} for RTMP") - } + RtmpMediaDescriptor.fromMediaDescriptor(descriptor) /** * Creates a RTMP connection descriptor from an [Uri] + * + * @param uri the server uri + * @param connectInfo configures the RTMP connect command sent during the handshake */ -fun RtmpMediaDescriptor(uri: Uri) = RtmpMediaDescriptor.fromUri(uri) +fun RtmpMediaDescriptor( + uri: Uri, + connectInfo: (ConnectObjectBuilder.() -> Unit)? = null +) = RtmpMediaDescriptor.fromUri(uri, connectInfo) /** * RTMP connection parameters @@ -52,10 +52,15 @@ fun RtmpMediaDescriptor(uri: Uri) = RtmpMediaDescriptor.fromUri(uri) * @param port the server port * @param app the application name * @param streamKey the stream key + * @param connectInfo configures the RTMP connect command sent during the handshake */ class RtmpMediaDescriptor( - val scheme: String, val host: String, val port: Int, val app: String?, val streamKey: String -) : MediaDescriptor(Type(MediaContainerType.FLV, MediaSinkType.RTMP)) { + val scheme: String, val host: String, val port: Int, val app: String?, val streamKey: String, + val connectInfo: (ConnectObjectBuilder.() -> Unit)? = null +) : MediaDescriptor( + Type(MediaContainerType.FLV, MediaSinkType.RTMP), + emptyList() +) { init { require(scheme == RTMP_SCHEME || scheme == RTMPS_SCHEME || scheme == RTMPT_SCHEME || scheme == RTMPE_SCHEME || scheme == RTMFP_SCHEME || scheme == RTMPTE_SCHEME || scheme == RTMPTS_SCHEME) { "Invalid scheme $scheme" } require(host.isNotBlank()) { "Invalid host $host" } @@ -65,7 +70,7 @@ class RtmpMediaDescriptor( require(streamKey.isNotBlank()) { "Invalid streamKey $streamKey" } } - override val uri = Uri.Builder() + override val uri: Uri = Uri.Builder() .scheme(scheme) .encodedAuthority("$host:$port") .apply { @@ -91,30 +96,63 @@ class RtmpMediaDescriptor( * Creates a RTMP connection descriptor from an URL * * @param url the server url (syntax: rtmp://host:port/app/streamKey) + * @param connectInfo configures the RTMP connect command sent during the handshake * @return RTMP connection descriptor */ - fun fromUrl(url: String) = - fromUri(url.toUri()) + internal fun fromUrl( + url: String, + connectInfo: (ConnectObjectBuilder.() -> Unit)? = null + ) = fromUri(url.toUri(), connectInfo) /** * Creates a RTMP connection descriptor from an Uri * * @param uri the server Uri + * @param connectInfo configures the RTMP connect command sent during the handshake * @return RTMP connection descriptor */ - fun fromUri(uri: Uri): RtmpMediaDescriptor { + internal fun fromUri( + uri: Uri, + connectInfo: (ConnectObjectBuilder.() -> Unit)? = null + ): RtmpMediaDescriptor = parse(uri, connectInfo) + + /** + * Creates a RTMP connection descriptor from a generic [MediaDescriptor]. + * + * If the descriptor is already a [RtmpMediaDescriptor], it is returned as is. + * If the descriptor is an [UriMediaDescriptor], it will be converted to a [RtmpMediaDescriptor]. + * Otherwise, an [InvalidParameterException] will be thrown. + * + * @param descriptor the source descriptor + * @return RTMP connection descriptor + */ + internal fun fromMediaDescriptor(descriptor: MediaDescriptor): RtmpMediaDescriptor = + when (descriptor) { + is RtmpMediaDescriptor -> descriptor + is UriMediaDescriptor -> { + parse( + descriptor.uri, + null + ) + } + + else -> throw InvalidParameterException("Invalid descriptor ${descriptor::class.java.simpleName} for RTMP") + } + + private fun parse( + uri: Uri, + connectInfo: (ConnectObjectBuilder.() -> Unit)? + ): RtmpMediaDescriptor { val scheme = uri.scheme ?: throw InvalidParameterException("Invalid scheme ${uri.scheme}") val host = uri.host ?: throw InvalidParameterException("Invalid host ${uri.host}") val port = if (uri.port > 0) { uri.port } else { - if ((scheme == RTMPS_SCHEME) || (scheme == RTMPTS_SCHEME)) { - SSL_DEFAULT_PORT - } else if ((scheme == RTMPT_SCHEME) || (scheme == RTMPTE_SCHEME)) { - HTTP_DEFAULT_PORT - } else { - DEFAULT_PORT + when (scheme) { + RTMPS_SCHEME, RTMPTS_SCHEME -> SSL_DEFAULT_PORT + RTMPT_SCHEME, RTMPTE_SCHEME -> HTTP_DEFAULT_PORT + else -> DEFAULT_PORT } } if (uri.pathSegments.isEmpty()) { @@ -123,12 +161,12 @@ class RtmpMediaDescriptor( val streamKey = uri.lastPathSegment ?: throw InvalidParameterException("Invalid streamKey ${uri.lastPathSegment}") if (uri.pathSegments.size == 1) { - return RtmpMediaDescriptor(scheme, host, port, null, streamKey) + return RtmpMediaDescriptor(scheme, host, port, null, streamKey, connectInfo) } else { val app = uri.pathSegments.minus(uri.lastPathSegment).joinToString("/") - return RtmpMediaDescriptor(scheme, host, port, app, streamKey) + return RtmpMediaDescriptor(scheme, host, port, app, streamKey, connectInfo) } } } } -} \ No newline at end of file +} diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt index b0d3d8eea..85076aa91 100644 --- a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpoint.kt @@ -16,18 +16,24 @@ package io.github.thibaultbee.streampack.ext.rtmp.elements.endpoints import android.content.Context +import io.github.komedia.komuxer.amf.AmfVersion import io.github.komedia.komuxer.flv.tags.FLVTag +import io.github.komedia.komuxer.flv.tags.audio.AudioData +import io.github.komedia.komuxer.flv.tags.video.VideoData import io.github.komedia.komuxer.rtmp.RtmpConnectionBuilder import io.github.komedia.komuxer.rtmp.client.RtmpClient +import io.github.komedia.komuxer.rtmp.client.RtmpClientSettings import io.github.komedia.komuxer.rtmp.connect import io.github.komedia.komuxer.rtmp.messages.command.StreamPublishType +import io.github.komedia.komuxer.rtmp.util.metrics.RtmpMetrics import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor -import io.github.thibaultbee.streampack.core.elements.data.FrameWithCloseable +import io.github.thibaultbee.streampack.core.elements.data.Frame import io.github.thibaultbee.streampack.core.elements.encoders.CodecConfig import io.github.thibaultbee.streampack.core.elements.endpoints.ClosedException import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.elements.endpoints.IEndpointInternal import io.github.thibaultbee.streampack.core.elements.endpoints.composites.CompositeEndpoint.EndpointInfo +import io.github.thibaultbee.streampack.core.elements.metrics.WithEndpointMetrics import io.github.thibaultbee.streampack.core.elements.utils.ChannelWithCloseableData import io.github.thibaultbee.streampack.core.elements.utils.useConsumeEach import io.github.thibaultbee.streampack.core.logger.Logger @@ -35,6 +41,7 @@ import io.github.thibaultbee.streampack.core.pipelines.IDispatcherProvider import io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.muxer.FlvMuxerInfo import io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.muxer.utils.FlvTagBuilder import io.github.thibaultbee.streampack.ext.flv.elements.endpoints.composites.muxer.utils.close +import io.github.thibaultbee.streampack.ext.rtmp.configuration.mediadescriptor.RtmpMediaDescriptor import io.ktor.network.selector.SelectorManager import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -51,18 +58,43 @@ import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import kotlinx.io.EOFException import java.io.IOException +import kotlin.concurrent.atomics.ExperimentalAtomicApi /** * An endpoint that send frame to an RTMP server. */ +@OptIn(ExperimentalAtomicApi::class) class RtmpEndpoint internal constructor( defaultDispatcher: CoroutineDispatcher, val ioDispatcher: CoroutineDispatcher -) : IEndpointInternal { +) : IEndpointInternal, WithEndpointMetrics { private val coroutineScope = CoroutineScope(SupervisorJob() + defaultDispatcher) private val mutex = Mutex() + private val metricsLock = Any() + private var frameDropped = 0L + private var audioFrameDropped = 0L + private var videoFrameDropped = 0L + + private var payloadSendDroppedSize = 0L + private var audioPayloadSendDroppedSize = 0L + private var videoPayloadSendDroppedSize = 0L + private val flvTagChannel = ChannelWithCloseableData( - 10 /* Arbitrary buffer size. TODO: add a parameter to set it */, BufferOverflow.DROP_OLDEST + 10 /* Arbitrary buffer size. TODO: add a parameter to set it */, BufferOverflow.DROP_OLDEST, + onUndeliveredElement = { flvTag -> + synchronized(metricsLock) { + val payloadSize = flvTag.data.getSize(AmfVersion.AMF0) + frameDropped++ + payloadSendDroppedSize += payloadSize + if (flvTag.data is AudioData) { + audioFrameDropped++ + audioPayloadSendDroppedSize += payloadSize + } else if (flvTag.data is VideoData) { + videoFrameDropped++ + videoPayloadSendDroppedSize += payloadSize + } + } + } ) private val flvTagBuilder = FlvTagBuilder(flvTagChannel) @@ -70,12 +102,27 @@ class RtmpEndpoint internal constructor( private val connectionBuilder = RtmpConnectionBuilder(selectorManager) private var rtmpClient: RtmpClient? = null - private var startUpTimestamp = INVALID_TIMESTAMP private val timestampMutex = Mutex() - override val metrics: Any - get() = TODO("Not yet implemented") + private val syncMetrics: RtmpMetrics + get() { + val metrics = + rtmpClient?.metrics ?: return RtmpMetrics.ZERO + return synchronized(metricsLock) { + metrics.copy( + messagesSendDropped = metrics.messagesSendDropped + frameDropped, + audioMessagesSendDropped = metrics.audioMessagesSendDropped + audioFrameDropped, + videoMessagesSendDropped = metrics.videoMessagesSendDropped + videoFrameDropped, + payloadSendDroppedSize = metrics.payloadSendDroppedSize + payloadSendDroppedSize, + audioPayloadSendDroppedSize = metrics.audioPayloadSendDroppedSize + audioPayloadSendDroppedSize, + videoPayloadSendDroppedSize = metrics.videoPayloadSendDroppedSize + videoPayloadSendDroppedSize + ) + } + } + + override val metrics: RtmpEndpointMetrics + get() = RtmpEndpointMetrics { syncMetrics } private val _isOpenFlow = MutableStateFlow(false) override val isOpenFlow = _isOpenFlow.asStateFlow() @@ -95,13 +142,28 @@ class RtmpEndpoint internal constructor( } } + private fun resetMetrics() { + synchronized(metricsLock) { + frameDropped = 0L + audioFrameDropped = 0L + videoFrameDropped = 0L + + payloadSendDroppedSize = 0L + audioPayloadSendDroppedSize = 0L + videoPayloadSendDroppedSize = 0L + } + } + private suspend fun safeClient(block: suspend (RtmpClient) -> T): T { val rtmpClient = requireNotNull(rtmpClient) { "Not opened" } require(!rtmpClient.isClosed) { "Connection closed" } return mutex.withLock { block(rtmpClient) } } - override suspend fun open(descriptor: MediaDescriptor) { + override suspend fun open(descriptor: MediaDescriptor) = + open(RtmpMediaDescriptor(descriptor)) + + private suspend fun open(descriptor: RtmpMediaDescriptor) { withContext(ioDispatcher) { mutex.withLock { if (rtmpClient?.isClosed == false) { @@ -109,7 +171,16 @@ class RtmpEndpoint internal constructor( return@withContext } - rtmpClient = connectionBuilder.connect(descriptor.uri.toString()).apply { + resetMetrics() + val client = if (descriptor.connectInfo != null) { + connectionBuilder.connect( + descriptor.uri.toString(), + RtmpClientSettings(connectInfo = descriptor.connectInfo) + ) + } else { + connectionBuilder.connect(descriptor.uri.toString()) + } + rtmpClient = client.apply { _isOpenFlow.emit(true) socketContext.invokeOnCompletion { throwable -> @@ -164,12 +235,15 @@ class RtmpEndpoint internal constructor( } override suspend fun write( - closeableFrame: FrameWithCloseable, streamPid: Int + frame: Frame, streamPid: Int ) { - val frame = closeableFrame.frame + if (!_isOpenFlow.value) { + frame.close() + return + } val startUpTimestamp = getStartUpTimestamp(frame.ptsInUs) val ts = (frame.ptsInUs - startUpTimestamp) / 1000 - flvTagBuilder.write(closeableFrame, ts.toInt(), streamPid) + flvTagBuilder.write(frame, ts.toInt(), streamPid) } override suspend fun addStreams(streamConfigs: List): Map { diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpointMetrics.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpointMetrics.kt new file mode 100644 index 000000000..93609a04b --- /dev/null +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/elements/endpoints/RtmpEndpointMetrics.kt @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.ext.rtmp.elements.endpoints + +import io.github.komedia.komuxer.rtmp.util.metrics.RtmpMetrics +import io.github.thibaultbee.streampack.core.elements.metrics.EndpointMetrics +import kotlin.time.Duration + +/** + * Creates a [RtmpEndpointMetrics] from a [metricsProvider]. + */ +fun RtmpEndpointMetrics(metricsProvider: () -> RtmpMetrics?): RtmpEndpointMetrics { + return RtmpEndpointMetrics(RtmpRawMetrics(metricsProvider)) +} + +/** + * Creates a [RtmpEndpointMetrics] from a [RtmpMetrics]. + */ +fun RtmpEndpointMetrics(rawMetrics: RtmpRawMetrics): RtmpEndpointMetrics { + val metrics = rawMetrics.rtmpMetrics + return RtmpEndpointMetrics( + uptime = metrics.uptime, + packetsWritten = metrics.messagesSent, + packetsWriteDropped = metrics.messagesSendDropped, + packetsWriteLost = 0L, + bytesWritten = metrics.totalBytesSent, + bytesWriteDropped = metrics.payloadSendDroppedSize, + rawMetrics = rawMetrics + ) +} + +/** + * Specific [EndpointMetrics] for RTMP protocol, based on [RtmpMetrics]. + */ +data class RtmpEndpointMetrics( + override val uptime: Duration, + override val packetsWritten: Long, + override val packetsWriteDropped: Long, + override val packetsWriteLost: Long, + override val bytesWritten: Long, + override val bytesWriteDropped: Long, + override val rawMetrics: RtmpRawMetrics +) : EndpointMetrics + + +/** + * Provides an access to internal RTMP metrics APIs. + */ +class RtmpRawMetrics internal constructor(private val metricsProvider: () -> RtmpMetrics?) { + /** + * Returns the [RtmpMetrics] if the client is available, otherwise null. + */ + val rtmpMetricsOrNull: RtmpMetrics? + get() = metricsProvider() +} + +/** + * Returns the [RtmpMetrics] if the client is available, otherwise [RtmpMetrics.ZERO]. + */ +val RtmpRawMetrics.rtmpMetrics: RtmpMetrics + get() = rtmpMetricsOrNull ?: RtmpMetrics.ZERO \ No newline at end of file diff --git a/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/regulator/RtmpBitrateRegulator.kt b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/regulator/RtmpBitrateRegulator.kt new file mode 100644 index 000000000..c7acf37b4 --- /dev/null +++ b/extensions/rtmp/src/main/java/io/github/thibaultbee/streampack/ext/rtmp/regulator/RtmpBitrateRegulator.kt @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.ext.rtmp.regulator + +import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig +import io.github.thibaultbee.streampack.core.elements.metrics.EndpointMetricsTracker +import io.github.thibaultbee.streampack.core.regulator.BitrateRegulator +import io.github.thibaultbee.streampack.core.regulator.IBitrateRegulator + +/** + * Base class of RTMP bitrate regulation implementation. + * + * If you want to implement your custom bitrate regulator, it must inherit from this class. + * The bitrate regulator object is created by streamers with the [IBitrateRegulator.Factory]. + * + * @param bitrateRegulatorConfig bitrate regulation configuration + * @param onVideoTargetBitrateChange call when you have to change video bitrate + * @param onAudioTargetBitrateChange call when you have to change audio bitrate + */ +abstract class RtmpBitrateRegulator( + metricsTracker: EndpointMetricsTracker, + bitrateRegulatorConfig: BitrateRegulatorConfig, + onVideoTargetBitrateChange: ((Int) -> Unit), + onAudioTargetBitrateChange: ((Int) -> Unit) +) : BitrateRegulator( + metricsTracker, + bitrateRegulatorConfig, + onVideoTargetBitrateChange, + onAudioTargetBitrateChange +) { + + + /** + * Factory interface you must use to create a [RtmpBitrateRegulator] object. + * If you want to create a custom RTMP bitrate regulation implementation, create a factory that + * implements this interface. + */ + interface Factory : IBitrateRegulator.Factory { + /** + * Creates a [RtmpBitrateRegulator] object from given parameters + * + * @param metricsTracker endpoint metrics tracker + * @param bitrateRegulatorConfig bitrate regulation configuration + * @param onVideoTargetBitrateChange call when you have to change video bitrate + * @param onAudioTargetBitrateChange call when you have to change audio bitrate + * @return a [RtmpBitrateRegulator] object + */ + override fun newBitrateRegulator( + metricsTracker: EndpointMetricsTracker, + bitrateRegulatorConfig: BitrateRegulatorConfig, + onVideoTargetBitrateChange: ((Int) -> Unit), + onAudioTargetBitrateChange: ((Int) -> Unit) + ): RtmpBitrateRegulator + } +} \ No newline at end of file diff --git a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/SrtEndpointFactory.kt b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/SrtEndpointFactory.kt index 920181103..f91dbc8f6 100644 --- a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/SrtEndpointFactory.kt +++ b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/SrtEndpointFactory.kt @@ -1,7 +1,23 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.github.thibaultbee.streampack.ext.srt.elements.endpoints import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.createDefaultTsServiceInfo import io.github.thibaultbee.streampack.core.elements.endpoints.composites.CompositeEndpointFactory +import io.github.thibaultbee.streampack.core.elements.endpoints.composites.CompositeEndpointWithMetricsFactory import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.TsMuxer import io.github.thibaultbee.streampack.core.elements.endpoints.composites.muxers.ts.data.TSServiceInfo import io.github.thibaultbee.streampack.ext.srt.elements.endpoints.composites.sinks.SrtSink @@ -19,7 +35,7 @@ fun SrtEndpointFactory( serviceInfo: TSServiceInfo = createDefaultTsServiceInfo(), coroutineDispatcher: CoroutineDispatcher ) = - CompositeEndpointFactory( + CompositeEndpointWithMetricsFactory( TsMuxer().apply { addService(serviceInfo) }, SrtSink(coroutineDispatcher) ) \ No newline at end of file diff --git a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/SrtEndpointMetrics.kt b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/SrtEndpointMetrics.kt new file mode 100644 index 000000000..9c516290d --- /dev/null +++ b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/SrtEndpointMetrics.kt @@ -0,0 +1,259 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.ext.srt.elements.endpoints + +import io.github.thibaultbee.srtdroid.core.models.Stats +import io.github.thibaultbee.srtdroid.ktx.CoroutineSrtSocket +import io.github.thibaultbee.streampack.core.elements.metrics.BasicEndpointMetrics +import io.github.thibaultbee.streampack.core.elements.metrics.EndpointMetrics +import io.github.thibaultbee.streampack.ext.srt.utils.SrtStatsHelper +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds + +/** + * Creates a [SrtEndpointMetrics] from a [SrtRawMetrics]. + */ +fun SrtEndpointMetrics(rawMetrics: SrtRawMetrics): SrtEndpointMetrics { + val stats = rawMetrics.bstats(clear = false) + return SrtEndpointMetrics( + uptime = stats.msTimeStamp.milliseconds, + packetsWritten = stats.pktSentTotal, + packetsRead = stats.pktRecvTotal, + packetsWriteLost = stats.pktSndLossTotal.toLong(), + packetsReadLost = stats.pktRcvLossTotal, + packetsRetransmitted = stats.pktRetransTotal, + packetsWriteACK = stats.pktSentACKTotal, + packetsReadACK = stats.pktRecvACKTotal, + packetsWriteNAK = stats.pktSentNAKTotal, + packetsReadNAK = stats.pktRecvNAKTotal, + usWriteDuration = stats.usSndDurationTotal, + packetsWriteDropped = stats.pktSndDropTotal.toLong(), + packetsReadDropped = stats.pktRcvDropTotal, + packetsReadUndecrypt = stats.pktRcvUndecryptTotal, + bytesWritten = stats.byteSentTotal, + bytesRead = stats.byteRecvTotal, + bytesReadLost = stats.byteRcvLossTotal, + bytesRetransmitted = stats.byteRetransTotal, + bytesWriteDropped = stats.byteSndDropTotal, + bytesReadDropped = stats.byteRcvDropTotal, + bytesReadUndecrypt = stats.byteRcvUndecryptTotal, + rawMetrics = rawMetrics + ) +} + +/** + * Basic metrics for SRT protocol, without the raw metrics helper. + */ +interface SrtBasicEndpointMetrics : BasicEndpointMetrics { + /** + * The time since the entity is started + */ + override val uptime: Duration + /** + * The total number of written data packets, including retransmissions + */ + override val packetsWritten: Long + /** + * The total number of read packets + */ + val packetsRead: Long + /** + * The total number of lost packets (writer side) + */ + override val packetsWriteLost: Long + /** + * The total number of lost packets (reader side) + */ + val packetsReadLost: Int + /** + * The total number of retransmitted packets + */ + val packetsRetransmitted: Int + /** + * The total number of written ACK packets + */ + val packetsWriteACK: Int + /** + * The total number of read ACK packets + */ + val packetsReadACK: Int + /** + * The total number of written NAK packets + */ + val packetsWriteNAK: Int + /** + * The total number of read NAK packets + */ + val packetsReadNAK: Int + /** + * The total time duration when UDT is writing data (idle time exclusive) + */ + val usWriteDuration: Long + /** + * The number of too-late-to-write dropped packets + */ + override val packetsWriteDropped: Long + /** + * The number of too-late-to-play missing packets + */ + val packetsReadDropped: Int + /** + * The number of undecrypted packets + */ + val packetsReadUndecrypt: Int + /** + * The total number of written data bytes, including retransmissions + */ + override val bytesWritten: Long + /** + * The total number of read bytes + */ + val bytesRead: Long + /** + * The total number of lost bytes + */ + val bytesReadLost: Long + /** + * The total number of retransmitted bytes + */ + val bytesRetransmitted: Long + /** + * The number of too-late-to-write dropped bytes + */ + override val bytesWriteDropped: Long + /** + * The number of too-late-to-play missing bytes (estimate based on average packet size) + */ + val bytesReadDropped: Long + /** + * The number of undecrypted bytes + */ + val bytesReadUndecrypt: Long + + override operator fun minus(other: BasicEndpointMetrics): BasicEndpointMetrics { + if (other !is SrtBasicEndpointMetrics) return super.minus(other) + + return object : SrtBasicEndpointMetrics { + override val uptime = this@SrtBasicEndpointMetrics.uptime - other.uptime + override val packetsWritten = this@SrtBasicEndpointMetrics.packetsWritten - other.packetsWritten + override val packetsRead = this@SrtBasicEndpointMetrics.packetsRead - other.packetsRead + override val packetsWriteLost = this@SrtBasicEndpointMetrics.packetsWriteLost - other.packetsWriteLost + override val packetsReadLost = this@SrtBasicEndpointMetrics.packetsReadLost - other.packetsReadLost + override val packetsRetransmitted = this@SrtBasicEndpointMetrics.packetsRetransmitted - other.packetsRetransmitted + override val packetsWriteACK = this@SrtBasicEndpointMetrics.packetsWriteACK - other.packetsWriteACK + override val packetsReadACK = this@SrtBasicEndpointMetrics.packetsReadACK - other.packetsReadACK + override val packetsWriteNAK = this@SrtBasicEndpointMetrics.packetsWriteNAK - other.packetsWriteNAK + override val packetsReadNAK = this@SrtBasicEndpointMetrics.packetsReadNAK - other.packetsReadNAK + override val usWriteDuration = this@SrtBasicEndpointMetrics.usWriteDuration - other.usWriteDuration + override val packetsWriteDropped = this@SrtBasicEndpointMetrics.packetsWriteDropped - other.packetsWriteDropped + override val packetsReadDropped = this@SrtBasicEndpointMetrics.packetsReadDropped - other.packetsReadDropped + override val packetsReadUndecrypt = this@SrtBasicEndpointMetrics.packetsReadUndecrypt - other.packetsReadUndecrypt + override val bytesWritten = this@SrtBasicEndpointMetrics.bytesWritten - other.bytesWritten + override val bytesRead = this@SrtBasicEndpointMetrics.bytesRead - other.bytesRead + override val bytesReadLost = this@SrtBasicEndpointMetrics.bytesReadLost - other.bytesReadLost + override val bytesRetransmitted = this@SrtBasicEndpointMetrics.bytesRetransmitted - other.bytesRetransmitted + override val bytesWriteDropped = this@SrtBasicEndpointMetrics.bytesWriteDropped - other.bytesWriteDropped + override val bytesReadDropped = this@SrtBasicEndpointMetrics.bytesReadDropped - other.bytesReadDropped + override val bytesReadUndecrypt = this@SrtBasicEndpointMetrics.bytesReadUndecrypt - other.bytesReadUndecrypt + } + } +} + +/** + * Specific [EndpointMetrics] for SRT protocol, based on [SrtRawMetrics]. + */ +data class SrtEndpointMetrics( + override val uptime: Duration, + override val packetsWritten: Long, + override val packetsRead: Long, + override val packetsWriteLost: Long, + override val packetsReadLost: Int, + override val packetsRetransmitted: Int, + override val packetsWriteACK: Int, + override val packetsReadACK: Int, + override val packetsWriteNAK: Int, + override val packetsReadNAK: Int, + override val usWriteDuration: Long, + override val packetsWriteDropped: Long, + override val packetsReadDropped: Int, + override val packetsReadUndecrypt: Int, + override val bytesWritten: Long, + override val bytesRead: Long, + override val bytesReadLost: Long, + override val bytesRetransmitted: Long, + override val bytesWriteDropped: Long, + override val bytesReadDropped: Long, + override val bytesReadUndecrypt: Long, + /** + * Raw SRT socket helper + */ + override val rawMetrics: SrtRawMetrics, +) : SrtBasicEndpointMetrics, EndpointMetrics + +/** + * Provides an access to internal SRT APIs. + */ +class SrtRawMetrics internal constructor(private val socketProvider: () -> CoroutineSrtSocket?) { + /** + * Reports the current statistics. + * + * **See Also:** [srt_bstats](https://github.com/Haivision/srt/blob/master/docs/API/API-functions.md#srt_bstats) + * + * @param clear true if the statistics should be cleared after retrieval + * @return the current [Stats] or null if the socket is not connected + */ + fun bstatsOrNull(clear: Boolean): Stats? { + return socketProvider()?.bstats(clear) + } + + /** + * Reports the current statistics. + * + * **See Also:** [srt_bistats](https://github.com/Haivision/srt/blob/master/docs/API/API-functions.md#srt_bistats) + * + * @param clear true if the statistics should be cleared after retrieval + * @param instantaneous true if the statistics should use instant data, not moving averages + * @return the current [Stats] if the socket is not connected + */ + fun bistatsOrNull(clear: Boolean, instantaneous: Boolean): Stats? { + return socketProvider()?.bistats(clear, instantaneous) + } +} + +/** + * Reports the current statistics. + * + * **See Also:** [srt_bstats](https://github.com/Haivision/srt/blob/master/docs/API/API-functions.md#srt_bstats) + * + * @param clear true if the statistics should be cleared after retrieval + * @return the current [Stats] or [io.github.thibaultbee.streampack.ext.srt.utils.SrtStatsHelper.ZERO] if the socket is not connected + */ +fun SrtRawMetrics.bstats(clear: Boolean): Stats { + return bstatsOrNull(clear) ?: SrtStatsHelper.ZERO +} + +/** + * Reports the current statistics. + * + * **See Also:** [srt_bistats](https://github.com/Haivision/srt/blob/master/docs/API/API-functions.md#srt_bistats) + * + * @param clear true if the statistics should be cleared after retrieval + * @param instantaneous true if the statistics should use instant data, not moving averages + * @return the current [Stats] or [SrtStatsHelper.ZERO] if the socket is not connected + */ +fun SrtRawMetrics.bistats(clear: Boolean, instantaneous: Boolean): Stats { + return bistatsOrNull(clear, instantaneous) ?: SrtStatsHelper.ZERO +} \ No newline at end of file diff --git a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/composites/sinks/SrtSink.kt b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/composites/sinks/SrtSink.kt index f5659ffee..1fdc7f5f5 100644 --- a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/composites/sinks/SrtSink.kt +++ b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/elements/endpoints/composites/sinks/SrtSink.kt @@ -20,7 +20,6 @@ import io.github.thibaultbee.srtdroid.core.enums.SockOpt import io.github.thibaultbee.srtdroid.core.enums.Transtype import io.github.thibaultbee.srtdroid.core.models.MsgCtrl import io.github.thibaultbee.srtdroid.core.models.SrtUrl.Mode -import io.github.thibaultbee.srtdroid.core.models.Stats import io.github.thibaultbee.srtdroid.ktx.CoroutineSrtSocket import io.github.thibaultbee.srtdroid.ktx.connect import io.github.thibaultbee.streampack.core.configuration.mediadescriptor.MediaDescriptor @@ -30,13 +29,17 @@ import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data. import io.github.thibaultbee.streampack.core.elements.endpoints.composites.data.SrtPacket import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.AbstractSink import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.SinkConfiguration +import io.github.thibaultbee.streampack.core.elements.endpoints.composites.sinks.ISinkWithMetricsInternal import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.ext.srt.configuration.mediadescriptor.SrtMediaDescriptor +import io.github.thibaultbee.streampack.ext.srt.elements.endpoints.SrtRawMetrics +import io.github.thibaultbee.streampack.ext.srt.elements.endpoints.SrtEndpointMetrics import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow -class SrtSink(private val coroutineDispatcher: CoroutineDispatcher) : AbstractSink() { +class SrtSink(private val coroutineDispatcher: CoroutineDispatcher) : AbstractSink(), + ISinkWithMetricsInternal { override val supportedSinkTypes: List = listOf(MediaSinkType.SRT) private var socket: CoroutineSrtSocket? = null @@ -45,12 +48,13 @@ class SrtSink(private val coroutineDispatcher: CoroutineDispatcher) : AbstractSi private var bitrate = 0L + private val srtRawMetrics = SrtRawMetrics { socket } + /** * Get SRT stats */ - override val metrics: Stats - get() = socket?.bistats(clear = true, instantaneous = true) - ?: throw IllegalStateException("Socket is not initialized") + override val metrics: SrtEndpointMetrics + get() = SrtEndpointMetrics(srtRawMetrics) private val _isOpenFlow = MutableStateFlow(false) override val isOpenFlow = _isOpenFlow.asStateFlow() diff --git a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DefaultSrtBitrateRegulator.kt b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DummySrtBitrateRegulator.kt similarity index 77% rename from extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DefaultSrtBitrateRegulator.kt rename to extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DummySrtBitrateRegulator.kt index ad7b288bb..4e5cc3e2f 100644 --- a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DefaultSrtBitrateRegulator.kt +++ b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/DummySrtBitrateRegulator.kt @@ -15,8 +15,9 @@ */ package io.github.thibaultbee.streampack.ext.srt.regulator -import io.github.thibaultbee.srtdroid.core.models.Stats import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig +import io.github.thibaultbee.streampack.core.elements.metrics.EndpointMetricsTracker +import io.github.thibaultbee.streampack.ext.srt.elements.endpoints.SrtEndpointMetrics import kotlin.math.max import kotlin.math.min @@ -27,11 +28,13 @@ import kotlin.math.min * @param onVideoTargetBitrateChange call when you have to change video bitrate * @param onAudioTargetBitrateChange not used in this implementation. */ -class DefaultSrtBitrateRegulator( +class DummySrtBitrateRegulator( + metricsTracker: EndpointMetricsTracker, bitrateRegulatorConfig: BitrateRegulatorConfig, onVideoTargetBitrateChange: ((Int) -> Unit), onAudioTargetBitrateChange: ((Int) -> Unit) ) : SrtBitrateRegulator( + metricsTracker, bitrateRegulatorConfig, onVideoTargetBitrateChange, onAudioTargetBitrateChange @@ -42,7 +45,9 @@ class DefaultSrtBitrateRegulator( const val SEND_PACKET_THRESHOLD = 50 } - override fun update(stats: Stats, currentVideoBitrate: Int, currentAudioBitrate: Int) { + override fun update(currentVideoBitrate: Int, currentAudioBitrate: Int) { + val metrics = metricsTracker.cumulative as SrtEndpointMetrics + val stats = metrics.rawMetrics.bistatsOrNull(clear = true, instantaneous = true) ?: return val estimatedBandwidth = (stats.mbpsBandwidth * 1000000).toInt() if (currentVideoBitrate > bitrateRegulatorConfig.videoBitrateRange.lower) { @@ -72,18 +77,11 @@ class DefaultSrtBitrateRegulator( } if (newVideoBitrate != 0) { - onVideoTargetBitrateChange( - max( - newVideoBitrate, - bitrateRegulatorConfig.videoBitrateRange.lower - ) - ) // Don't go under videoBitrateRange.lower + onVideoTargetBitrateChange(newVideoBitrate) return } - } - - // Can bitrate go upper? - if (currentVideoBitrate < bitrateRegulatorConfig.videoBitrateRange.upper) { + // Can bitrate go upper? + } else if (currentVideoBitrate < bitrateRegulatorConfig.videoBitrateRange.upper) { val newVideoBitrate = when { (currentVideoBitrate + currentAudioBitrate) < estimatedBandwidth -> { currentVideoBitrate + min( @@ -96,36 +94,34 @@ class DefaultSrtBitrateRegulator( } if (newVideoBitrate != 0) { - onVideoTargetBitrateChange( - max( - newVideoBitrate, - bitrateRegulatorConfig.videoBitrateRange.lower - ) - ) // Don't go under videoBitrateRange.lower + onVideoTargetBitrateChange(newVideoBitrate) return } } } /** - * Factory that creates a [DefaultSrtBitrateRegulator]. + * Factory that creates a [DummySrtBitrateRegulator]. */ class Factory : SrtBitrateRegulator.Factory { /** - * Creates a [DefaultSrtBitrateRegulator] object from given parameters + * Creates a [DummySrtBitrateRegulator] object from given parameters * + * @param metricsTracker endpoint metrics tracker * @param bitrateRegulatorConfig bitrate regulation configuration * @param onVideoTargetBitrateChange call when you have to change video bitrate * @param onAudioTargetBitrateChange call when you have to change audio bitrate - * @return a [DefaultSrtBitrateRegulator] object + * @return a [DummySrtBitrateRegulator] object */ override fun newBitrateRegulator( + metricsTracker: EndpointMetricsTracker, bitrateRegulatorConfig: BitrateRegulatorConfig, onVideoTargetBitrateChange: ((Int) -> Unit), onAudioTargetBitrateChange: ((Int) -> Unit) - ): DefaultSrtBitrateRegulator { - return DefaultSrtBitrateRegulator( + ): DummySrtBitrateRegulator { + return DummySrtBitrateRegulator( + metricsTracker, bitrateRegulatorConfig, onVideoTargetBitrateChange, onAudioTargetBitrateChange diff --git a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/SrtBitrateRegulator.kt b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/SrtBitrateRegulator.kt index dbfb53a84..79920fe30 100644 --- a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/SrtBitrateRegulator.kt +++ b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/SrtBitrateRegulator.kt @@ -15,8 +15,8 @@ */ package io.github.thibaultbee.streampack.ext.srt.regulator -import io.github.thibaultbee.srtdroid.core.models.Stats import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig +import io.github.thibaultbee.streampack.core.elements.metrics.EndpointMetricsTracker import io.github.thibaultbee.streampack.core.regulator.BitrateRegulator import io.github.thibaultbee.streampack.core.regulator.IBitrateRegulator @@ -31,25 +31,17 @@ import io.github.thibaultbee.streampack.core.regulator.IBitrateRegulator * @param onAudioTargetBitrateChange call when you have to change audio bitrate */ abstract class SrtBitrateRegulator( + metricsTracker: EndpointMetricsTracker, bitrateRegulatorConfig: BitrateRegulatorConfig, onVideoTargetBitrateChange: ((Int) -> Unit), onAudioTargetBitrateChange: ((Int) -> Unit) ) : BitrateRegulator( + metricsTracker, bitrateRegulatorConfig, onVideoTargetBitrateChange, onAudioTargetBitrateChange ) { - override fun update(stats: Any, currentVideoBitrate: Int, currentAudioBitrate: Int) = - update(stats as Stats, currentVideoBitrate, currentAudioBitrate) - /** - * Call regularly to get new SRT stats - * - * @param stats SRT transmission stats - * @param currentVideoBitrate current video bitrate target in bits/s. - * @param currentAudioBitrate current audio bitrate target in bits/s. - */ - abstract fun update(stats: Stats, currentVideoBitrate: Int, currentAudioBitrate: Int) /** * Factory interface you must use to create a [SrtBitrateRegulator] object. @@ -60,12 +52,14 @@ abstract class SrtBitrateRegulator( /** * Creates a [SrtBitrateRegulator] object from given parameters * + * @param metricsTracker endpoint metrics tracker * @param bitrateRegulatorConfig bitrate regulation configuration * @param onVideoTargetBitrateChange call when you have to change video bitrate * @param onAudioTargetBitrateChange call when you have to change audio bitrate * @return a [SrtBitrateRegulator] object */ override fun newBitrateRegulator( + metricsTracker: EndpointMetricsTracker, bitrateRegulatorConfig: BitrateRegulatorConfig, onVideoTargetBitrateChange: ((Int) -> Unit), onAudioTargetBitrateChange: ((Int) -> Unit) diff --git a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/controllers/DefaultSrtBitrateRegulatorController.kt b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/controllers/DefaultSrtBitrateRegulatorController.kt deleted file mode 100644 index ae290cdf4..000000000 --- a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/controllers/DefaultSrtBitrateRegulatorController.kt +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2024 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.ext.srt.regulator.controllers - -import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableAudioEncodingPipelineOutput -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IConfigurableVideoEncodingPipelineOutput -import io.github.thibaultbee.streampack.core.pipelines.outputs.encoding.IEncodingPipelineOutput -import io.github.thibaultbee.streampack.core.regulator.controllers.BitrateRegulatorController -import io.github.thibaultbee.streampack.core.regulator.controllers.DummyBitrateRegulatorController -import io.github.thibaultbee.streampack.ext.srt.regulator.DefaultSrtBitrateRegulator -import io.github.thibaultbee.streampack.ext.srt.regulator.SrtBitrateRegulator -import kotlinx.coroutines.CoroutineDispatcher - -/** - * A [DummyBitrateRegulatorController] implementation for a [SrtSink]. - */ -class DefaultSrtBitrateRegulatorController { - class Factory( - private val bitrateRegulatorFactory: SrtBitrateRegulator.Factory = DefaultSrtBitrateRegulator.Factory(), - private val bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig(), - private val delayTimeInMs: Long = 500 - ) : BitrateRegulatorController.Factory() { - override fun newBitrateRegulatorController( - pipelineOutput: IEncodingPipelineOutput, - coroutineDispatcher: CoroutineDispatcher - ): DummyBitrateRegulatorController { - require(pipelineOutput is IConfigurableVideoEncodingPipelineOutput) { - "Pipeline output must be an video encoding output" - } - - val videoEncoder = requireNotNull(pipelineOutput.videoEncoder) { - "Video encoder must be set" - } - - val audioEncoder = if (pipelineOutput is IConfigurableAudioEncodingPipelineOutput) { - pipelineOutput.audioEncoder - } else { - null - } - return DummyBitrateRegulatorController( - audioEncoder, - videoEncoder, - pipelineOutput.endpoint, - bitrateRegulatorFactory, - coroutineDispatcher, - bitrateRegulatorConfig, - delayTimeInMs - ) - } - } -} diff --git a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/controllers/SrtBitrateRegulatorControllerFactories.kt b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/controllers/SrtBitrateRegulatorControllerFactories.kt new file mode 100644 index 000000000..43fc3a2d5 --- /dev/null +++ b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/regulator/controllers/SrtBitrateRegulatorControllerFactories.kt @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.ext.srt.regulator.controllers + +import io.github.thibaultbee.streampack.core.configuration.BitrateRegulatorConfig +import io.github.thibaultbee.streampack.core.regulator.IBitrateRegulator +import io.github.thibaultbee.streampack.core.regulator.controllers.IntervalBitrateRegulatorController +import io.github.thibaultbee.streampack.core.regulator.controllers.IntervalBitrateRegulatorController.Companion.DEFAULT_POLLING_TIME +import io.github.thibaultbee.streampack.ext.srt.regulator.DummySrtBitrateRegulator +import io.github.thibaultbee.streampack.ext.srt.regulator.SrtBitrateRegulator +import kotlin.time.Duration + +/** + * A [IntervalBitrateRegulatorController.Factory] for [SrtBitrateRegulator]. + * + * @param bitrateRegulatorFactory the [SrtBitrateRegulator.Factory] implementation. Use it to make your own bitrate regulator. + * @param bitrateRegulatorConfig bitrate regulator configuration + * @param pollingTime delay between each call to [IBitrateRegulator.update] + * + * @see IntervalBitrateRegulatorController.Factory + * @see DummySrtBitrateRegulator.Factory + */ +fun intervalSrtBitrateRegulatorControllerFactory( + bitrateRegulatorFactory: SrtBitrateRegulator.Factory = DummySrtBitrateRegulator.Factory(), + bitrateRegulatorConfig: BitrateRegulatorConfig = BitrateRegulatorConfig(), + pollingTime: Duration = DEFAULT_POLLING_TIME +) = IntervalBitrateRegulatorController.Factory( + bitrateRegulatorFactory, + bitrateRegulatorConfig, + pollingTime +) diff --git a/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/utils/SrtStatsHelper.kt b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/utils/SrtStatsHelper.kt new file mode 100644 index 000000000..e337b3575 --- /dev/null +++ b/extensions/srt/src/main/java/io/github/thibaultbee/streampack/ext/srt/utils/SrtStatsHelper.kt @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.ext.srt.utils + +import io.github.thibaultbee.srtdroid.core.models.Stats + +object SrtStatsHelper { + val ZERO = Stats( + msTimeStamp = 0L, + pktSentTotal = 0L, + pktRecvTotal = 0L, + pktSndLossTotal = 0, + pktRcvLossTotal = 0, + pktRetransTotal = 0, + pktSentACKTotal = 0, + pktRecvACKTotal = 0, + pktSentNAKTotal = 0, + pktRecvNAKTotal = 0, + usSndDurationTotal = 0, + pktSndDropTotal = 0, + pktRcvDropTotal = 0, + pktRcvUndecryptTotal = 0, + byteSentTotal = 0, + byteRecvTotal = 0, + byteRcvLossTotal = 0, + byteRetransTotal = 0, + byteSndDropTotal = 0, + byteRcvDropTotal = 0, + byteRcvUndecryptTotal = 0, + pktSent = 0, + pktRecv = 0, + pktSndLoss = 0, + pktRcvLoss = 0, + pktRetrans = 0, + pktRcvRetrans = 0, + pktSentACK = 0, + pktRecvACK = 0, + pktSentNAK = 0, + pktRecvNAK = 0, + mbpsSendRate = 0.0, + mbpsRecvRate = 0.0, + usSndDuration = 0, + pktReorderDistance = 0, + pktRcvAvgBelatedTime = 0.0, + pktRcvBelated = 0, + pktSndDrop = 0, + pktRcvDrop = 0, + pktRcvUndecrypt = 0, + byteSent = 0, + byteRecv = 0, + byteRcvLoss = 0, + byteRetrans = 0, + byteSndDrop = 0, + byteRcvDrop = 0, + byteRcvUndecrypt = 0, + usPktSndPeriod = 0.0, + pktFlowWindow = 0, + pktCongestionWindow = 0, + pktFlightSize = 0, + msRTT = 0.0, + mbpsBandwidth = 0.0, + byteAvailSndBuf = 0, + byteAvailRcvBuf = 0, + mbpsMaxBW = 0.0, + byteMSS = 0, + pktSndBuf = 0, + byteSndBuf = 0, + msSndBuf = 0, + msSndTsbPdDelay = 0, + pktRcvBuf = 0, + byteRcvBuf = 0, + msRcvBuf = 0, + msRcvTsbPdDelay = 0, + pktSndFilterExtraTotal = 0, + pktRcvFilterExtraTotal = 0, + pktRcvFilterSupplyTotal = 0, + pktRcvFilterLossTotal = 0, + pktSndFilterExtra = 0, + pktRcvFilterExtra = 0, + pktRcvFilterSupply = 0, + pktRcvFilterLoss = 0, + pktReorderTolerance = 0, + pktSentUniqueTotal = 0, + pktRecvUniqueTotal = 0, + byteSentUniqueTotal = 0, + byteRecvUniqueTotal = 0, + pktSentUnique = 0, + pktRecvUnique = 0, + byteSentUnique = 0, + byteRecvUnique = 0 + ) +} + + diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4602a3dcf..feda42b74 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,6 +7,7 @@ videoApiClient = "1.6.7" androidxActivity = "1.10.1" androidxAppcompat = "1.7.1" androidxCamera = "1.4.0-alpha13" +androidxComposeBom = "2026.01.00" androidxConstraintlayout = "2.2.1" androidxCore = "1.17.0" androidxDatabinding = "8.13.0" @@ -26,9 +27,10 @@ kotlinxIo = "0.8.0" material = "1.13.0" mockk = "1.14.5" robolectric = "4.16" -komuxer = "0.3.4" +komuxer = "0.4.0" srtdroid = "1.9.5" junitKtx = "1.3.0" +compose = "1.10.1" [libraries] android-documentation-plugin = { module = "org.jetbrains.dokka:android-documentation-plugin", version.ref = "dokka" } @@ -42,6 +44,7 @@ android-material = { module = "com.google.android.material:material", version.re androidx-activity = { module = "androidx.activity:activity", version.ref = "androidxActivity" } androidx-appcompat = { module = "androidx.appcompat:appcompat", version.ref = "androidxAppcompat" } androidx-camera-viewfinder-view = { module = "androidx.camera.viewfinder:viewfinder-view", version.ref = "androidxCamera" } +androidx-compose-bom = { module = "androidx.compose:compose-bom", version.ref = "androidxComposeBom" } androidx-concurrent-futures = { module = "androidx.concurrent:concurrent-futures", version.ref = "concurrentFutures" } androidx-constraintlayout = { module = "androidx.constraintlayout:constraintlayout", version.ref = "androidxConstraintlayout" } androidx-core-ktx = { module = "androidx.core:core-ktx", version.ref = "androidxCore" } @@ -70,10 +73,16 @@ kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.re mockk = { module = "io.mockk:mockk", version.ref = "mockk" } robolectric = { module = "org.robolectric:robolectric", version.ref = "robolectric" } srtdroid-ktx = { module = "io.github.thibaultbee.srtdroid:srtdroid-ktx", version.ref = "srtdroid" } +androidx-compose-runtime = { group = "androidx.compose.runtime", name = "runtime", version.ref = "compose" } +androidx-compose-ui = { group = "androidx.compose.ui", name = "ui", version.ref = "compose" } +androidx-compose-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview", version.ref = "compose" } +androidx-compose-ui-tooling = { group = "androidx.compose.ui", name = "ui-tooling", version.ref = "compose" } +androidx-compose-foundation = { group = "androidx.compose.foundation", name = "foundation", version.ref = "compose" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } android-library = { id = "com.android.library", version.ref = "agp" } +compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } kotlin-kapt = { id = "org.jetbrains.kotlin.kapt", version.ref = "kotlin" } dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt index 0b740e6b5..5f5888bfe 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/MediaProjectionService.kt @@ -163,7 +163,7 @@ abstract class MediaProjectionService( } if (streamer is IWithVideoSource) { - val videoSource = streamer.videoInput?.sourceFlow?.value + val videoSource = streamer.videoInput.sourceFlow.value if (videoSource is IMediaProjectionSource) { streamer.setVideoSource(MediaProjectionVideoSourceFactory(mediaProjection)) } else if (videoSource == null) { diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt index 61f5bc703..d9866e2e4 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/utils/StreamerFactory.kt @@ -16,13 +16,17 @@ package io.github.thibaultbee.streampack.services.utils import android.content.Context +import android.view.Surface import io.github.thibaultbee.streampack.core.elements.utils.RotationValue import io.github.thibaultbee.streampack.core.elements.utils.extensions.displayRotation import io.github.thibaultbee.streampack.core.interfaces.IStreamer import io.github.thibaultbee.streampack.core.streamers.dual.DualStreamer import io.github.thibaultbee.streampack.core.streamers.dual.IDualStreamer +import io.github.thibaultbee.streampack.core.streamers.dual.VideoOnlyDualStreamer +import io.github.thibaultbee.streampack.core.streamers.single.AudioOnlySingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.ISingleStreamer import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.core.streamers.single.VideoOnlySingleStreamer import io.github.thibaultbee.streampack.services.StreamerService @@ -53,12 +57,19 @@ open class SingleStreamerFactory( ) : StreamerFactory { override fun create(context: Context): ISingleStreamer { - return SingleStreamer( - context, - withAudio, - withVideo, - defaultRotation = defaultRotation ?: context.displayRotation - ) + return if (withAudio && withVideo) { + SingleStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else if (withAudio) { + AudioOnlySingleStreamer(context) + } else { + VideoOnlySingleStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } } } @@ -76,11 +87,18 @@ open class DualStreamerFactory( ) : StreamerFactory { override fun create(context: Context): IDualStreamer { - return DualStreamer( - context, - withAudio, - withVideo, - defaultRotation = defaultRotation ?: context.displayRotation - ) + return if (withAudio && withVideo) { + DualStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else if (withVideo) { + VideoOnlyDualStreamer( + context, + defaultRotation = defaultRotation ?: context.displayRotation + ) + } else { + throw IllegalArgumentException("DualStreamer audio only is not supported yet") + } } } \ No newline at end of file diff --git a/settings.libs.gradle.kts b/settings.libs.gradle.kts index c42addcde..7a8a4328c 100644 --- a/settings.libs.gradle.kts +++ b/settings.libs.gradle.kts @@ -1,11 +1,17 @@ // StreamPack libraries include(":core") project(":core").name = "streampack-core" -include(":ui") -project(":ui").name = "streampack-ui" include(":services") project(":services").name = "streampack-services" +// UI +include(":ui") +project(":ui").projectDir = File(rootDir, "ui/ui") +project(":ui").name = "streampack-ui" +include(":compose") +project(":compose").projectDir = File(rootDir, "ui/compose") +project(":compose").name = "streampack-compose" + // Extensions include(":extension-flv") project(":extension-flv").projectDir = File(rootDir, "extensions/flv") diff --git a/ui/.gitignore b/ui/compose/.gitignore similarity index 100% rename from ui/.gitignore rename to ui/compose/.gitignore diff --git a/ui/compose/build.gradle.kts b/ui/compose/build.gradle.kts new file mode 100644 index 000000000..de8c2b318 --- /dev/null +++ b/ui/compose/build.gradle.kts @@ -0,0 +1,35 @@ +plugins { + id(libs.plugins.android.library.get().pluginId) + id(libs.plugins.kotlin.android.get().pluginId) + id("android-library-convention") + alias(libs.plugins.compose.compiler) +} + +description = "Jetpack compose components for StreamPack." + +android { + namespace = "io.github.thibaultbee.streampack.compose" + + defaultConfig { + minSdk = 23 + } + buildFeatures { + compose = true + } +} + +dependencies { + implementation(project(":streampack-core")) + implementation(project(":streampack-ui")) + + val composeBom = platform(libs.androidx.compose.bom) + implementation(composeBom) + implementation(libs.androidx.compose.runtime) + implementation(libs.androidx.compose.ui) + implementation(libs.androidx.compose.ui.tooling.preview) + implementation(libs.androidx.compose.foundation) + + androidTestImplementation(composeBom) + + debugImplementation(libs.androidx.compose.ui.tooling) +} diff --git a/ui/proguard-rules.pro b/ui/compose/proguard-rules.pro similarity index 100% rename from ui/proguard-rules.pro rename to ui/compose/proguard-rules.pro diff --git a/ui/src/main/AndroidManifest.xml b/ui/compose/src/main/AndroidManifest.xml similarity index 100% rename from ui/src/main/AndroidManifest.xml rename to ui/compose/src/main/AndroidManifest.xml diff --git a/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/SourcePreview.kt b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/SourcePreview.kt new file mode 100644 index 000000000..aa2e1035d --- /dev/null +++ b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/SourcePreview.kt @@ -0,0 +1,116 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.compose + +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.viewinterop.AndroidView +import io.github.thibaultbee.streampack.compose.utils.BitmapUtils +import io.github.thibaultbee.streampack.core.elements.sources.video.IPreviewableSource +import io.github.thibaultbee.streampack.core.elements.sources.video.bitmap.BitmapSourceFactory +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings.FocusMetering.Companion.DEFAULT_AUTO_CANCEL_DURATION_MS +import io.github.thibaultbee.streampack.core.interfaces.IWithVideoSource +import io.github.thibaultbee.streampack.core.logger.Logger +import io.github.thibaultbee.streampack.core.streamers.single.SingleStreamer +import io.github.thibaultbee.streampack.ui.views.PreviewView +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.withLock + +private const val TAG = "ComposeSourcePreview" + +/** + * Displays the preview of a [IWithVideoSource]. + * + * A [IWithVideoSource] must have a video sourc. + * + * @param videoSource the [IWithVideoSource] to preview + * @param modifier the [Modifier] to apply to the [PreviewView] + * @param enableZoomOnPinch enable zoom on pinch gesture + * @param enableTapToFocus enable tap to focus + * @param onTapToFocusTimeoutMs the duration in milliseconds after which the focus area set by tap-to-focus is cleared + */ +@Composable +fun SourcePreview( + videoSource: IWithVideoSource, + modifier: Modifier = Modifier, + onZoomChanged: ((zoomRatio: Float) -> Unit)? = null, + enableZoomOnPinch: Boolean = true, + enableTapToFocus: Boolean = true, + onTapToFocusTimeoutMs: Long = DEFAULT_AUTO_CANCEL_DURATION_MS +) { + val scope = rememberCoroutineScope() + + AndroidView( + factory = { context -> + PreviewView(context).apply { + this.enableZoomOnPinch = enableZoomOnPinch + this.enableTapToFocus = enableTapToFocus + this.onTapToFocusTimeoutMs = onTapToFocusTimeoutMs + onZoomChanged?.let { + val onZoomChangedListener = object : CameraSettings.Zoom.OnZoomChangedListener { + override fun onZoomChanged(zoomRatio: Float) { + onZoomChanged(zoomRatio) + } + } + this.setZoomListener(onZoomChangedListener) + } + + + scope.launch { + try { + setVideoSourceProvider(videoSource) + } catch (e: Exception) { + Logger.e(TAG, "Failed to start preview", e) + } + } + } + }, + modifier = modifier, + onRelease = { + scope.launch { + val source = videoSource.videoInput.sourceFlow.value as? IPreviewableSource + source?.previewMutex?.withLock { + source.stopPreview() + source.resetPreview() + } + } + }) +} + +@Preview +@Composable +fun PreviewScreenSourcePreview() { + val context = LocalContext.current + val streamer = SingleStreamer(context) + LaunchedEffect(Unit) { + streamer.setVideoSource( + BitmapSourceFactory( + BitmapUtils.createImage( + 1280, + 720 + ) + ) + ) + } + + SourcePreview(streamer, modifier = Modifier.fillMaxSize()) +} \ No newline at end of file diff --git a/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt new file mode 100644 index 000000000..c1cfec02b --- /dev/null +++ b/ui/compose/src/main/java/io/github/thibaultbee/streampack/compose/utils/BitmapUtils.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2026 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.compose.utils + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint + +object BitmapUtils { + /** + * Creates a bitmap with the given width, height and color. + * + * @param width The width of the bitmap. + * @param height The height of the bitmap. + * @param color The color of the bitmap. + * @return The created bitmap. + */ + fun createImage(width: Int, height: Int, color: Int = Color.RED): Bitmap { + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bitmap) + val paint = Paint().apply { + this.color = color + } + canvas.drawRect(0f, 0f, width.toFloat(), height.toFloat(), paint) + return bitmap + } +} diff --git a/ui/ui/.gitignore b/ui/ui/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/ui/ui/.gitignore @@ -0,0 +1 @@ +/build diff --git a/ui/build.gradle.kts b/ui/ui/build.gradle.kts similarity index 100% rename from ui/build.gradle.kts rename to ui/ui/build.gradle.kts diff --git a/ui/ui/proguard-rules.pro b/ui/ui/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/ui/ui/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/ui/ui/src/main/AndroidManifest.xml b/ui/ui/src/main/AndroidManifest.xml new file mode 100644 index 000000000..cc947c567 --- /dev/null +++ b/ui/ui/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/AutoFitSurfaceView.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt similarity index 90% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt index b9f2f4be3..19e06c5d8 100644 --- a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt @@ -35,6 +35,7 @@ import androidx.camera.viewfinder.core.ScaleType import androidx.camera.viewfinder.core.ViewfinderSurfaceRequest import androidx.camera.viewfinder.core.populateFromCharacteristics import io.github.thibaultbee.streampack.core.elements.sources.video.IPreviewableSource +import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings import io.github.thibaultbee.streampack.core.elements.sources.video.camera.CameraSettings.FocusMetering.Companion.DEFAULT_AUTO_CANCEL_DURATION_MS import io.github.thibaultbee.streampack.core.elements.sources.video.camera.ICameraSource import io.github.thibaultbee.streampack.core.elements.sources.video.camera.extensions.getCameraCharacteristics @@ -108,9 +109,11 @@ class PreviewView @JvmOverloads constructor( } /** - * The [Listener] to listen to specific view events. + * The [PreviewListener] to listen to specific view events. */ - var listener: Listener? = null + var listener: PreviewListener? = null + + private var zoomListener: CameraSettings.Zoom.OnZoomChangedListener? = null private var touchUpEvent: MotionEvent? = null @@ -160,8 +163,8 @@ class PreviewView @JvmOverloads constructor( streamer: IWithVideoSource ) { sourceJob += defaultScope.launch { - streamer.videoInput?.sourceFlow?.runningHistoryNotNull() - ?.collect { (previousVideoSource, newVideoSource) -> + streamer.videoInput.sourceFlow.runningHistoryNotNull() + .collect { (previousVideoSource, newVideoSource) -> if (previousVideoSource == newVideoSource) { Logger.w(TAG, "No change in video source") } else { @@ -176,12 +179,54 @@ class PreviewView @JvmOverloads constructor( } if (newVideoSource is IPreviewableSource) { attachToStreamerIfReady(true) + zoomListener?.let { + registerZoomListener(it) + } } } } } } + /** + * Sets the [CameraSettings.Zoom.OnZoomChangedListener] to listen to zoom changes. + * + * @param listener the [CameraSettings.Zoom.OnZoomChangedListener] to listen to zoom changes. + */ + fun setZoomListener(listener: CameraSettings.Zoom.OnZoomChangedListener?) { + if (listener == null) { + unregisterZoomListener() + zoomListener = null + } else { + val delegatedListener = object : CameraSettings.Zoom.OnZoomChangedListener { + override fun onZoomChanged(zoomRatio: Float) { + post { + listener.onZoomChanged(zoomRatio) + } + } + } + registerZoomListener(delegatedListener) + zoomListener = delegatedListener + } + } + + private fun registerZoomListener(listener: CameraSettings.Zoom.OnZoomChangedListener) { + val source = streamer?.videoInput?.sourceFlow?.value + if (source is ICameraSource) { + source.settings.zoom.addListener(listener) + } + } + + private fun unregisterZoomListener() { + zoomListener?.let { + val source = streamer?.videoInput?.sourceFlow?.value + if (source is ICameraSource) { + source.settings.zoom.removeListener(it) + } + } + } + + /** * Sets the [IWithVideoSource] to preview. * @@ -336,9 +381,7 @@ class PreviewView @JvmOverloads constructor( } /** - * Requests a [Surface] for the size and the current streamer video source. - * - * The [Surface] is emit to the [surfaceFlow]. + * Starts the preview for the given [size]. */ private fun startPreview(size: Size) { Logger.d(TAG, "Requesting surface for $size") @@ -351,9 +394,7 @@ class PreviewView @JvmOverloads constructor( } /** - * Requests a [Surface] for the size and the [videoSource]. - * - * The [Surface] is emit to the [surfaceFlow]. + * Starts the preview for the given [size] and [videoSource]. */ private fun startPreview( size: Size, @@ -418,7 +459,7 @@ class PreviewView @JvmOverloads constructor( private suspend fun stopPreview() { streamer?.let { - val videoSource = it.videoInput?.sourceFlow?.value + val videoSource = it.videoInput.sourceFlow.value if (videoSource is IPreviewableSource) { Logger.d(TAG, "Stopping preview") videoSource.previewMutex.withLock { @@ -511,10 +552,6 @@ class PreviewView @JvmOverloads constructor( mutex.withLock { val zoom = source.settings.zoom zoom.onPinch(scaleFactor) - val newZoomRatio = zoom.getZoomRatio() - post { - listener?.onZoomRationOnPinchChanged(newZoomRatio) - } } } return true @@ -528,7 +565,7 @@ class PreviewView @JvmOverloads constructor( /** * A listener for the [PreviewView]. */ - interface Listener { + interface PreviewListener { /** * Called when the preview is started. */ @@ -538,12 +575,6 @@ class PreviewView @JvmOverloads constructor( * Called when the preview failed to start. */ fun onPreviewFailed(t: Throwable) {} - - /** - * Called when the zoom ratio is changed. - * @param zoomRatio the new zoom ratio - */ - fun onZoomRationOnPinchChanged(zoomRatio: Float) {} } /** diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt similarity index 91% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt index 7134c77cd..68a4efd41 100644 --- a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt +++ b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/StreamerExtensions.kt @@ -18,7 +18,7 @@ suspend fun IWithVideoSource.setPreview( viewfinder: CameraViewfinder, previewSize: Size ): ViewfinderSurfaceRequest { - val videoSource = videoInput?.sourceFlow?.value as? IPreviewableSource + val videoSource = videoInput.sourceFlow.value as? IPreviewableSource ?: throw IllegalStateException("Video source is not previewable") return videoSource.setPreview(viewfinder, previewSize) } @@ -35,7 +35,7 @@ suspend fun IWithVideoSource.startPreview( viewfinder: CameraViewfinder, previewSize: Size ): ViewfinderSurfaceRequest { - val videoSource = videoInput?.sourceFlow?.value as? IPreviewableSource + val videoSource = videoInput.sourceFlow.value as? IPreviewableSource ?: throw IllegalStateException("Video source is not previewable") return videoSource.startPreview(viewfinder, previewSize) } diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/VideoSourceExtensions.kt diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt b/ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt similarity index 100% rename from ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt rename to ui/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/ViewExtensions.kt diff --git a/ui/src/main/res/values/attrs.xml b/ui/ui/src/main/res/values/attrs.xml similarity index 100% rename from ui/src/main/res/values/attrs.xml rename to ui/ui/src/main/res/values/attrs.xml