diff --git a/README.md b/README.md index d7e4e2aee..49d38e4fb 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Everyone is welcome to our [Discourse community](https://react-native-webrtc.dis ## WebRTC Revision -* Currently used revision: [M100](https://github.com/jitsi/webrtc/releases/tag/v100.0.0) +* Currently used revision: [M94](https://github.com/jitsi/webrtc/releases/tag/v94.0.0) * Supported architectures * Android: armeabi-v7a, arm64-v8a, x86, x86_64 * iOS: arm64, x86_64 (for bitcode support, run [this script](https://github.com/react-native-webrtc/react-native-webrtc/blob/master/tools/downloadBitcode.sh)) diff --git a/android/build.gradle b/android/build.gradle index 7180a6c2b..052cfdaa1 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -4,6 +4,15 @@ def safeExtGet(prop, fallback) { rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback } +allprojects { + repositories { + google() + jcenter() + maven { url 'https://www.jitpack.io' } + mavenCentral() + } +} + android { compileSdkVersion safeExtGet('compileSdkVersion', 23) buildToolsVersion safeExtGet('buildToolsVersion', "23.0.1") @@ -28,5 +37,6 @@ android { dependencies { implementation 'com.facebook.react:react-native:+' + implementation 'com.google.mlkit:segmentation-selfie:16.0.0-beta4' api fileTree(dir: 'libs', include: ['*.jar']) } diff --git a/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java b/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java index 5097b5561..97fcf0564 100644 --- a/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java +++ b/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java @@ -199,7 +199,7 @@ void getUserMedia( cameraEnumerator, videoConstraintsMap); - videoTrack = createVideoTrack(cameraCaptureController); + videoTrack = createVideoTrack(cameraCaptureController, videoConstraintsMap.hasKey("vb")); } if (audioTrack == null && videoTrack == null) { @@ -355,10 +355,10 @@ private VideoTrack createScreenTrack() { int height = displayMetrics.heightPixels; ScreenCaptureController screenCaptureController = new ScreenCaptureController(reactContext.getCurrentActivity(), width, height, mediaProjectionPermissionResultData); - return createVideoTrack(screenCaptureController); + return createVideoTrack(screenCaptureController, false); } - private VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureController) { + private VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureController, Boolean vb) { videoCaptureController.initializeVideoCapturer(); VideoCapturer videoCapturer = videoCaptureController.videoCapturer; @@ -379,6 +379,11 @@ private VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureC VideoSource videoSource = pcFactory.createVideoSource(videoCapturer.isScreencast()); videoCapturer.initialize(surfaceTextureHelper, reactContext, videoSource.getCapturerObserver()); + if(vb) { + VideoProcessor p = new VirtualBackgroundVideoProcessor(reactContext, surfaceTextureHelper); + videoSource.setVideoProcessor(p); + } + String id = UUID.randomUUID().toString(); VideoTrack track = pcFactory.createVideoTrack(id, videoSource); diff --git a/android/src/main/java/com/oney/WebRTCModule/VirtualBackgroundVideoProcessor.java b/android/src/main/java/com/oney/WebRTCModule/VirtualBackgroundVideoProcessor.java new file mode 100644 index 000000000..35eaf8d62 --- /dev/null +++ b/android/src/main/java/com/oney/WebRTCModule/VirtualBackgroundVideoProcessor.java @@ -0,0 +1,166 @@ +package com.oney.WebRTCModule; + +import static android.graphics.Color.argb; +import static android.graphics.PorterDuff.Mode.DST_OVER; +import static android.graphics.PorterDuff.Mode.SRC_IN; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Canvas; +import android.graphics.Matrix; +import android.graphics.Paint; +import android.graphics.PorterDuffXfermode; +import android.opengl.GLES20; +import android.opengl.GLUtils; + +import androidx.annotation.Nullable; + +import com.facebook.react.bridge.ReactApplicationContext; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.android.gms.tasks.Task; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.segmentation.Segmentation; +import com.google.mlkit.vision.segmentation.SegmentationMask; +import com.google.mlkit.vision.segmentation.Segmenter; +import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions; + +import org.webrtc.SurfaceTextureHelper; +import org.webrtc.TextureBufferImpl; +import org.webrtc.VideoFrame; +import org.webrtc.VideoProcessor; +import org.webrtc.VideoSink; +import org.webrtc.YuvConverter; + +public class VirtualBackgroundVideoProcessor implements VideoProcessor { + + private VideoSink target; + private final SurfaceTextureHelper surfaceTextureHelper; + final YuvConverter yuvConverter = new YuvConverter(); + + private YuvFrame yuvFrame; + private Bitmap inputFrameBitmap; + private int frameCounter = 0; + + final Bitmap backgroundImage; + final Bitmap scaled; + + final SelfieSegmenterOptions options = + new SelfieSegmenterOptions.Builder() + .setDetectorMode(SelfieSegmenterOptions.STREAM_MODE) + .build(); + final Segmenter segmenter = Segmentation.getClient(options); + + public VirtualBackgroundVideoProcessor(ReactApplicationContext context, SurfaceTextureHelper surfaceTextureHelper) { + super(); + + this.surfaceTextureHelper = surfaceTextureHelper; + + backgroundImage = BitmapFactory.decodeResource(context.getResources(), R.drawable.portrait_background); + scaled = Bitmap.createScaledBitmap(backgroundImage, 640, 640, false ); + } + + @Override + public void setSink(@Nullable VideoSink videoSink) { + target = videoSink; + } + + @Override + public void onCapturerStarted(boolean b) { + + } + + @Override + public void onCapturerStopped() { + + } + + @Override + public void onFrameCaptured(VideoFrame videoFrame) { + + if(frameCounter == 0) { + yuvFrame = new YuvFrame(videoFrame); + inputFrameBitmap = yuvFrame.getBitmap(); + + InputImage image = InputImage.fromBitmap(inputFrameBitmap, 0); + + Task result = + segmenter.process(image) + .addOnSuccessListener( + new OnSuccessListener() { + @Override + public void onSuccess(SegmentationMask mask) { + + mask.getBuffer().rewind(); + int[] arr = maskColorsFromByteBuffer(mask); + Bitmap segmentedBitmap = Bitmap.createBitmap( + arr, mask.getWidth(), mask.getHeight(), Bitmap.Config.ARGB_8888 + ); + arr = null; + + Bitmap segmentedBitmapMutable = segmentedBitmap.copy(Bitmap.Config.ARGB_8888, true); + segmentedBitmap.recycle(); + Canvas canvas = new Canvas(segmentedBitmapMutable); + + Paint paint = new Paint(); + paint.setXfermode(new PorterDuffXfermode(SRC_IN)); + canvas.drawBitmap(scaled, 0, 0, paint); + paint.setXfermode(new PorterDuffXfermode(DST_OVER)); + canvas.drawBitmap(inputFrameBitmap, 0, 0, paint); + + surfaceTextureHelper.getHandler().post(new Runnable() { + @Override + public void run() { + + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + TextureBufferImpl buffer = new TextureBufferImpl(segmentedBitmapMutable.getWidth(), + segmentedBitmapMutable.getHeight(), VideoFrame.TextureBuffer.Type.RGB, + GLES20.GL_TEXTURE0, new Matrix(), surfaceTextureHelper.getHandler(), yuvConverter, null); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE0); + + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST); + GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, segmentedBitmapMutable, 0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); + + VideoFrame.I420Buffer i420Buf = yuvConverter.convert(buffer); + VideoFrame out = new VideoFrame(i420Buf, 180, videoFrame.getTimestampNs()); + + buffer.release(); + //yuvFrame.dispose(); + + target.onFrame(out); + out.release(); + } + }); + + } + }); + } + updateFrameCounter(); + } + + private void updateFrameCounter() { + frameCounter++; + if(frameCounter == 3) { + frameCounter = 0; + } + } + + private int[] maskColorsFromByteBuffer(SegmentationMask mask) { + int[] colors = new int[mask.getHeight() * mask.getWidth()]; + for (int i = 0; i < mask.getHeight() * mask.getWidth(); i++) { + float backgroundLikelihood = 1 - mask.getBuffer().getFloat(); + if (backgroundLikelihood > 0.9) { + colors[i] = argb(255, 255, 0, 255); + } else if (backgroundLikelihood > 0.2) { + // Linear interpolation to make sure when backgroundLikelihood is 0.2, the alpha is 0 and + // when backgroundLikelihood is 0.9, the alpha is 128. + // +0.5 to round the float value to the nearest int. + double d = 182.9 * backgroundLikelihood - 36.6 + 0.5; + int alpha = (int) d; + colors[i] = argb(alpha, 255, 0, 255); + } + } + return colors; + } +} diff --git a/android/src/main/java/com/oney/WebRTCModule/YuvFrame.java b/android/src/main/java/com/oney/WebRTCModule/YuvFrame.java new file mode 100644 index 000000000..12fc29e96 --- /dev/null +++ b/android/src/main/java/com/oney/WebRTCModule/YuvFrame.java @@ -0,0 +1,196 @@ +package com.oney.WebRTCModule; + +import android.graphics.Bitmap; +import android.graphics.Matrix; + +import org.webrtc.VideoFrame; + +import java.nio.ByteBuffer; + +public class YuvFrame { + public int width; + public int height; + public byte[] nv21Buffer; + public int rotationDegree; + public long timestamp; + + private final Object planeLock = new Object(); + + public YuvFrame(final VideoFrame videoFrame) { + fromVideoFrame(videoFrame, System.nanoTime()); + } + + public void fromVideoFrame(final VideoFrame videoFrame, final long timestamp) { + if (videoFrame == null) { + return; + } + + synchronized (planeLock) { + try { + // Save timestamp + this.timestamp = timestamp; + + // Copy rotation information + rotationDegree = videoFrame.getRotation(); // Just save rotation info for now, doing actual rotation can wait until per-pixel processing. + + // Copy the pixel data, processing as requested. + copyPlanes(videoFrame.getBuffer()); + } catch (Throwable t) { + dispose(); + } + } + } + + public void dispose() { + nv21Buffer = null; + } + + private void copyPlanes( final VideoFrame.Buffer videoFrameBuffer ) + { + VideoFrame.I420Buffer i420Buffer = null; + + if ( videoFrameBuffer != null ) + { + i420Buffer = videoFrameBuffer.toI420(); + } + + if ( i420Buffer == null ) + { + return; + } + + synchronized ( planeLock ) + { + // Set the width and height of the frame. + width = i420Buffer.getWidth(); + height = i420Buffer.getHeight(); + + // Calculate sizes needed to convert to NV21 buffer format + final int size = width * height; + final int chromaStride = width; + final int chromaWidth = ( width + 1 ) / 2; + final int chromaHeight = ( height + 1 ) / 2; + final int nv21Size = size + chromaStride * chromaHeight; + + if ( nv21Buffer == null || nv21Buffer.length != nv21Size ) + { + nv21Buffer = new byte[nv21Size]; + } + + final ByteBuffer yPlane = i420Buffer.getDataY(); + final ByteBuffer uPlane = i420Buffer.getDataU(); + final ByteBuffer vPlane = i420Buffer.getDataV(); + final int yStride = i420Buffer.getStrideY(); + final int uStride = i420Buffer.getStrideU(); + final int vStride = i420Buffer.getStrideV(); + + // Populate a buffer in NV21 format because that's what the converter wants + for ( int y = 0; y < height; y++ ) + { + for ( int x = 0; x < width; x++ ) + { + nv21Buffer[y * width + x] = yPlane.get( y * yStride + x ); + } + } + + for ( int y = 0; y < chromaHeight; y++ ) + { + for ( int x = 0; x < chromaWidth; x++ ) + { + // Swapping U and V values here because it makes the image the right color + + // Store V + nv21Buffer[size + y * chromaStride + 2 * x + 1] = uPlane.get( y * uStride + x ); + + // Store U + nv21Buffer[size + y * chromaStride + 2 * x] = vPlane.get( y * vStride + x ); + } + } + } + i420Buffer.release(); + } + + public Bitmap getBitmap() + { + if ( nv21Buffer == null ) + { + return null; + } + + // Calculate the size of the frame + final int size = width * height; + + // Allocate an array to hold the ARGB pixel data + int[] argb = new int[size]; + + // Use the converter (based on WebRTC source) to change to ARGB format + YUV_NV21_TO_RGB(argb, nv21Buffer, width, height); + + // Construct a Bitmap based on the new pixel data + Bitmap bitmap = Bitmap.createBitmap( argb, width, height, Bitmap.Config.ARGB_8888 ); + argb = null; + + // If necessary, generate a rotated version of the Bitmap + if ( rotationDegree == 90 || rotationDegree == -270 ) + { + final Matrix m = new Matrix(); + m.preScale(-1, 1); + m.postRotate( 90 ); + + return Bitmap.createBitmap( bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), m, true ); + } + else if ( rotationDegree == 180 || rotationDegree == -180 ) + { + final Matrix m = new Matrix(); + m.preScale(-1, 1); + m.postRotate( 180 ); + + return Bitmap.createBitmap( bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), m, true ); + } + else if ( rotationDegree == 270 || rotationDegree == -90 ) + { + final Matrix m = new Matrix(); + m.preScale(1, -1); + m.postRotate( 270 ); + + return Bitmap.createBitmap( bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), m, true ); + } + else + { + final Matrix m = new Matrix(); + m.preScale(-1, 1); + + return Bitmap.createBitmap( bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), m, true ); + } + } + + public static void YUV_NV21_TO_RGB(int[] argb, byte[] yuv, int width, int height) { + final int frameSize = width * height; + + final int ii = 0; + final int ij = 0; + final int di = +1; + final int dj = +1; + + int a = 0; + for (int i = 0, ci = ii; i < height; ++i, ci += di) { + for (int j = 0, cj = ij; j < width; ++j, cj += dj) { + int y = (0xff & ((int) yuv[ci * width + cj])); + int v = (0xff & ((int) yuv[frameSize + (ci >> 1) * width + (cj & ~1) + 0])); + int u = (0xff & ((int) yuv[frameSize + (ci >> 1) * width + (cj & ~1) + 1])); + y = y < 16 ? 16 : y; + + int r = (int) (1.164f * (y - 16) + 1.596f * (v - 128)); + int g = (int) (1.164f * (y - 16) - 0.813f * (v - 128) - 0.391f * (u - 128)); + int b = (int) (1.164f * (y - 16) + 2.018f * (u - 128)); + + r = r < 0 ? 0 : (r > 255 ? 255 : r); + g = g < 0 ? 0 : (g > 255 ? 255 : g); + b = b < 0 ? 0 : (b > 255 ? 255 : b); + + argb[a++] = 0xff000000 | (r << 16) | (g << 8) | b; + } + } + } +} + diff --git a/android/src/main/res/drawable/portrait_background.jpg b/android/src/main/res/drawable/portrait_background.jpg new file mode 100644 index 000000000..fc9233db3 Binary files /dev/null and b/android/src/main/res/drawable/portrait_background.jpg differ diff --git a/ios/RCTWebRTC.xcodeproj/project.pbxproj b/ios/RCTWebRTC.xcodeproj/project.pbxproj index 0a4611097..87ebfbe73 100644 --- a/ios/RCTWebRTC.xcodeproj/project.pbxproj +++ b/ios/RCTWebRTC.xcodeproj/project.pbxproj @@ -307,7 +307,15 @@ 35A222291CB493700015FD5C /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { - FRAMEWORK_SEARCH_PATHS = "$(PROJECT_DIR)/../apple/"; + FRAMEWORK_SEARCH_PATHS = ( + "$(PROJECT_DIR)/../apple/", + "\"${PODS_ROOT}/MLKitSegmentationSelfie/Frameworks\"", + "\"${PODS_ROOT}/MLKitSegmentationCommon/Frameworks\"", + "\"${PODS_ROOT}/MLKitCommon/Frameworks\"", + "\"${PODS_ROOT}/MLImage/Frameworks\"", + "\"${PODS_ROOT}/MLKitVision/Frameworks\"", + "\"${PODS_ROOT}/MLKitXenoCommon/Frameworks\"", + ); LIBRARY_SEARCH_PATHS = "$(inherited)"; OTHER_LDFLAGS = "-ObjC"; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -318,7 +326,15 @@ 35A2222A1CB493700015FD5C /* Release */ = { isa = XCBuildConfiguration; buildSettings = { - FRAMEWORK_SEARCH_PATHS = "$(PROJECT_DIR)/../apple/"; + FRAMEWORK_SEARCH_PATHS = ( + "$(PROJECT_DIR)/../apple/", + "\"${PODS_ROOT}/MLKitSegmentationSelfie/Frameworks\"", + "\"${PODS_ROOT}/MLKitSegmentationCommon/Frameworks\"", + "\"${PODS_ROOT}/MLKitCommon/Frameworks\"", + "\"${PODS_ROOT}/MLImage/Frameworks\"", + "\"${PODS_ROOT}/MLKitVision/Frameworks\"", + "\"${PODS_ROOT}/MLKitXenoCommon/Frameworks\"", + ); LIBRARY_SEARCH_PATHS = "$(inherited)"; OTHER_LDFLAGS = "-ObjC"; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/ios/RCTWebRTC/ScreenCaptureController.m b/ios/RCTWebRTC/ScreenCaptureController.m index 33443e9ad..b94da4c9a 100644 --- a/ios/RCTWebRTC/ScreenCaptureController.m +++ b/ios/RCTWebRTC/ScreenCaptureController.m @@ -35,10 +35,6 @@ - (instancetype)initWithCapturer:(nonnull ScreenCapturer *)capturer { return self; } -- (void)dealloc { - [self.capturer stopCapture]; -} - - (void)startCapture { if (!self.appGroupIdentifier) { return; diff --git a/ios/RCTWebRTC/ScreenCapturer.m b/ios/RCTWebRTC/ScreenCapturer.m index 572f54d64..f9a962b1d 100644 --- a/ios/RCTWebRTC/ScreenCapturer.m +++ b/ios/RCTWebRTC/ScreenCapturer.m @@ -141,13 +141,6 @@ - (instancetype)initWithDelegate:(__weak id)delegate { return self; } -- (void)setConnection:(SocketConnection *)connection { - if (_connection != connection) { - [_connection close]; - _connection = connection; - } -} - - (void)startCaptureWithConnection:(SocketConnection *)connection { _startTimeStampNs = -1; diff --git a/ios/RCTWebRTC/VideoCaptureController.m b/ios/RCTWebRTC/VideoCaptureController.m index 8ecf374f1..a53483186 100644 --- a/ios/RCTWebRTC/VideoCaptureController.m +++ b/ios/RCTWebRTC/VideoCaptureController.m @@ -15,6 +15,7 @@ @interface VideoCaptureController () @property (nonatomic, assign) int width; @property (nonatomic, assign) int height; @property (nonatomic, assign) int frameRate; +@property (nonatomic, assign) BOOL vb; @end @@ -34,6 +35,9 @@ - (instancetype)initWithCapturer:(RTCCameraVideoCapturer *)capturer self.width = [constraints[@"width"] intValue]; self.height = [constraints[@"height"] intValue]; self.frameRate = [constraints[@"frameRate"] intValue]; + if(constraints[@"vb"]) { + self.vb = YES; + } id facingMode = constraints[@"facingMode"]; @@ -96,6 +100,17 @@ - (void)startCapture { dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); __weak VideoCaptureController *weakSelf = self; + + if (self.vb) { + // ML Kit library requires kCVPixelFormatType_32BGRA format + for (AVCaptureOutput *output in _capturer.captureSession.outputs) { + RCTLog(@"Changing capturer output to %@", ((AVCaptureVideoDataOutput*)output).videoSettings); + if([output isKindOfClass:AVCaptureVideoDataOutput.class]) { + ((AVCaptureVideoDataOutput*)output).videoSettings = @{(NSString *)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA]}; + } + } + } + [self.capturer startCaptureWithDevice:self.device format:format fps:self.frameRate completionHandler:^(NSError *err) { if (err) { RCTLogError(@"[VideoCaptureController] Error starting capture: %@", err); @@ -227,8 +242,13 @@ - (void)throttleFrameRateForDevice:(AVCaptureDevice *)device { return; } - device.activeVideoMinFrameDuration = CMTimeMake(1, 20); - device.activeVideoMaxFrameDuration = CMTimeMake(1, 15); + if (self.vb) { + device.activeVideoMinFrameDuration = CMTimeMake(1, 15); + device.activeVideoMaxFrameDuration = CMTimeMake(1, 12); + } else { + device.activeVideoMinFrameDuration = CMTimeMake(1, 20); + device.activeVideoMaxFrameDuration = CMTimeMake(1, 15); + } [device unlockForConfiguration]; } diff --git a/ios/RCTWebRTC/VideoSourceInterceptor.h b/ios/RCTWebRTC/VideoSourceInterceptor.h new file mode 100644 index 000000000..76cd1acc1 --- /dev/null +++ b/ios/RCTWebRTC/VideoSourceInterceptor.h @@ -0,0 +1,21 @@ +// +// VideoSourceInterceptor.h +// react-native-webrtc +// +// Created by YAVUZ SELIM CAKIR on 18.06.2022. +// + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface VideoSourceInterceptor : NSObject + +@property(nonatomic, strong) RTCVideoSource *videoSource; + +- (instancetype)initWithVideoSource: (RTCVideoSource*) videoSource; + +@end + +NS_ASSUME_NONNULL_END diff --git a/ios/RCTWebRTC/VideoSourceInterceptor.m b/ios/RCTWebRTC/VideoSourceInterceptor.m new file mode 100644 index 000000000..2a7f80832 --- /dev/null +++ b/ios/RCTWebRTC/VideoSourceInterceptor.m @@ -0,0 +1,324 @@ +// +// VideoSourceInterceptor.m +// react-native-webrtc +// +// Created by YAVUZ SELIM CAKIR on 18.06.2022. +// + +#import "VideoSourceInterceptor.h" +#import +#import +#import +#import + +#import +#import +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface VideoSourceInterceptor () + +@property (nonatomic) RTCVideoCapturer *capturer; +@property (nonatomic, strong) MLKSegmenter *segmenter; +@property (nonatomic) RTCVideoRotation rotation; +@property (nonatomic) int64_t timeStampNs; +@property (nonatomic) CVPixelBufferRef backgroundBuffer; +@property (nonatomic) CVPixelBufferRef rightRotatedBackgroundBuffer; +@property (nonatomic) CVPixelBufferRef leftRotatedBackgroundBuffer; +@property (nonatomic) CVPixelBufferRef upsideRotatedBackgroundBuffer; + +@property (nonatomic) UIImage *backgroundImage; +@property (nonatomic) UIImage *rightRotatedBackgroundImage; +@property (nonatomic) UIImage *leftRotatedBackgroundImage; +@property (nonatomic) UIImage *upsideDownBackgroundImage; + +@end + +@implementation VideoSourceInterceptor + +- (instancetype)initWithVideoSource: (RTCVideoSource*) videoSource { + if (self = [super init]) { + _videoSource = videoSource; + + MLKSelfieSegmenterOptions *options = [[MLKSelfieSegmenterOptions alloc] init]; + options.segmenterMode = MLKSegmenterModeStream; + options.shouldEnableRawSizeMask = NO; + + self.segmenter = [MLKSegmenter segmenterWithOptions:options]; + + _backgroundImage = [UIImage imageNamed:@"portraitBackground"]; + _rightRotatedBackgroundImage = [UIImage imageNamed:@"rightRotatedBackground"]; + _leftRotatedBackgroundImage = [UIImage imageNamed:@"leftRotatedBackground"]; + _upsideDownBackgroundImage = [UIImage imageNamed:@"upsideDownBackground"]; + + _backgroundBuffer = [self pixelBufferFromCGImage:_backgroundImage.CGImage]; + _rightRotatedBackgroundBuffer = [self pixelBufferFromCGImage:_rightRotatedBackgroundImage.CGImage]; + _leftRotatedBackgroundBuffer = [self pixelBufferFromCGImage:_leftRotatedBackgroundImage.CGImage]; + _upsideRotatedBackgroundBuffer = [self pixelBufferFromCGImage:_upsideDownBackgroundImage.CGImage]; + } + return self; +} + +- (CVPixelBufferRef) pixelBufferFromCGImage: (CGImageRef) image +{ + NSDictionary *options = @{ + (NSString*)kCVPixelBufferCGImageCompatibilityKey : @YES, + (NSString*)kCVPixelBufferCGBitmapContextCompatibilityKey : @YES, + }; + + CVPixelBufferRef pxbuffer = NULL; + + CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault, CGImageGetWidth(image), + CGImageGetHeight(image), kCVPixelFormatType_32BGRA, (__bridge CFDictionaryRef) options, + &pxbuffer); + + NSParameterAssert(status == kCVReturnSuccess && pxbuffer != NULL); + + CVPixelBufferLockBaseAddress(pxbuffer, 0); + void *pxdata = CVPixelBufferGetBaseAddress(pxbuffer); + + CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate(pxdata, CGImageGetWidth(image), + CGImageGetHeight(image), 8, 4*CGImageGetWidth(image), rgbColorSpace, + kCGImageAlphaPremultipliedLast); + NSParameterAssert(context); + + CGContextConcatCTM(context, CGAffineTransformMakeRotation(-90 * M_PI / 180.0)); + CGContextTranslateCTM(context, -640, 0); + CGAffineTransform flipVertical = CGAffineTransformMake( 1, 0, 0, -1, 0, CGImageGetHeight(image) ); + CGContextConcatCTM(context, flipVertical); + + CGContextDrawImage(context, CGRectMake(0, 0, CGImageGetWidth(image), + CGImageGetHeight(image)), image); + CGColorSpaceRelease(rgbColorSpace); + CGContextRelease(context); + + CVPixelBufferUnlockBaseAddress(pxbuffer, 0); + return pxbuffer; +} + +- (void)capturer:(nonnull RTCVideoCapturer *)capturer didCaptureVideoFrame:(nonnull RTCVideoFrame *)frame { + + RTCCVPixelBuffer* pixelBufferr = (RTCCVPixelBuffer *)frame.buffer; + CVPixelBufferRef pixelBufferRef = pixelBufferr.pixelBuffer; + + self.rotation = frame.rotation; + self.timeStampNs = frame.timeStampNs; + self.capturer = capturer; + + CMSampleBufferRef sampleBuffer = [self getCMSampleBuffer:pixelBufferRef timeStamp:self.timeStampNs]; + + MLKVisionImage *image = [[MLKVisionImage alloc] initWithBuffer:sampleBuffer]; + image.orientation = [self imageOrientation]; + + NSError *error; + MLKSegmentationMask *mask = + [self.segmenter resultsInImage:image error:&error]; + if (error != nil) { + // Error. + return; + } + + [self applySegmentationMask:mask + toPixelBuffer:pixelBufferRef + rotation:self.rotation]; + + RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer = + [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; + + RTCI420Buffer *i420buffer = [rtcPixelBuffer toI420]; + + RTC_OBJC_TYPE(RTCVideoFrame) *processedFrame = + [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:i420buffer + rotation:self.rotation + timeStampNs:self.timeStampNs]; + + [_videoSource capturer:self.capturer didCaptureVideoFrame:processedFrame]; + + CMSampleBufferInvalidate(sampleBuffer); + CFRelease(sampleBuffer); + sampleBuffer = NULL; +} + +- (CMSampleBufferRef)getCMSampleBuffer: (CVPixelBufferRef)pixelBuffer timeStamp: (int64_t) timeStampNs { + + CMSampleTimingInfo info = kCMTimingInfoInvalid; + info.presentationTimeStamp = CMTimeMake(timeStampNs, 1000000000);; + info.duration = kCMTimeInvalid; + info.decodeTimeStamp = kCMTimeInvalid; + + CMFormatDescriptionRef formatDesc = nil; + CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &formatDesc); + + CMSampleBufferRef sampleBuffer = nil; + + CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, + pixelBuffer, + formatDesc, + &info, + &sampleBuffer); + + return sampleBuffer; +} + +- (UIImageOrientation)imageOrientation { + return [self imageOrientationFromDevicePosition:AVCaptureDevicePositionFront]; +} + +- (UIImageOrientation)imageOrientationFromDevicePosition:(AVCaptureDevicePosition)devicePosition { + + UIDeviceOrientation deviceOrientation = UIDevice.currentDevice.orientation; + + if (deviceOrientation == UIDeviceOrientationFaceDown || + deviceOrientation == UIDeviceOrientationFaceUp || + deviceOrientation == UIDeviceOrientationUnknown) { + deviceOrientation = [self currentUIOrientation]; + } + + switch (deviceOrientation) { + case UIDeviceOrientationPortrait: + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationLeftMirrored + : UIImageOrientationRight; + case UIDeviceOrientationLandscapeLeft: + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationDownMirrored + : UIImageOrientationUp; + case UIDeviceOrientationPortraitUpsideDown: + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationRightMirrored + : UIImageOrientationLeft; + case UIDeviceOrientationLandscapeRight: + return devicePosition == AVCaptureDevicePositionFront ? UIImageOrientationUpMirrored + : UIImageOrientationDown; + case UIDeviceOrientationFaceDown: + case UIDeviceOrientationFaceUp: + case UIDeviceOrientationUnknown: + return UIImageOrientationUp; + } +} + +- (UIDeviceOrientation)currentUIOrientation { + UIDeviceOrientation (^deviceOrientation)(void) = ^UIDeviceOrientation(void) { + switch (UIApplication.sharedApplication.statusBarOrientation) { + case UIInterfaceOrientationLandscapeLeft: + return UIDeviceOrientationLandscapeRight; + case UIInterfaceOrientationLandscapeRight: + return UIDeviceOrientationLandscapeLeft; + case UIInterfaceOrientationPortraitUpsideDown: + return UIDeviceOrientationPortraitUpsideDown; + case UIInterfaceOrientationPortrait: + case UIInterfaceOrientationUnknown: + return UIDeviceOrientationPortrait; + } + }; + + if (NSThread.isMainThread) { + return deviceOrientation(); + } else { + __block UIDeviceOrientation currentOrientation = UIDeviceOrientationPortrait; + dispatch_sync(dispatch_get_main_queue(), ^{ + currentOrientation = deviceOrientation(); + }); + return currentOrientation; + } +} + +- (void)applySegmentationMask:(MLKSegmentationMask *)mask + toPixelBuffer:(CVPixelBufferRef)imageBuffer + rotation:(RTCVideoRotation)rotation{ + + CVPixelBufferRef currentBackground = NULL; + + switch (rotation) { + case RTCVideoRotation_90: + currentBackground = _backgroundBuffer; + break; + case RTCVideoRotation_0: //Right rotated screen + currentBackground = _leftRotatedBackgroundBuffer; + break; + case RTCVideoRotation_270: //Upside down screen + currentBackground = _upsideRotatedBackgroundBuffer; + break; + case RTCVideoRotation_180: //Left rotated screen + currentBackground = _rightRotatedBackgroundBuffer; + break; + } + + size_t width = CVPixelBufferGetWidth(mask.buffer); + size_t height = CVPixelBufferGetHeight(mask.buffer); + + CVPixelBufferLockBaseAddress(imageBuffer, 0); + CVPixelBufferLockBaseAddress(currentBackground, 0); + CVPixelBufferLockBaseAddress(mask.buffer, kCVPixelBufferLock_ReadOnly); + + float *maskAddress = (float *)CVPixelBufferGetBaseAddress(mask.buffer); + size_t maskBytesPerRow = CVPixelBufferGetBytesPerRow(mask.buffer); + + unsigned char *imageAddress = (unsigned char *)CVPixelBufferGetBaseAddress(imageBuffer); + size_t bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer); + static const int kBGRABytesPerPixel = 4; + + unsigned char *backgroundImageAddress = (unsigned char *)CVPixelBufferGetBaseAddress(currentBackground); + size_t backgroundImageBytesPerRow = CVPixelBufferGetBytesPerRow(currentBackground); + + static const float kMaxColorComponentValue = 255.0f; + + for (int row = 0; row < height; ++row) { + for (int col = 0; col < width; ++col) { + int pixelOffset = col * kBGRABytesPerPixel; + int blueOffset = pixelOffset; + int greenOffset = pixelOffset + 1; + int redOffset = pixelOffset + 2; + int alphaOffset = pixelOffset + 3; + + float maskValue = maskAddress[col]; + float backgroundRegionRatio = 1.0f - maskValue; + + float originalPixelRed = imageAddress[redOffset] / kMaxColorComponentValue; + float originalPixelGreen = imageAddress[greenOffset] / kMaxColorComponentValue; + float originalPixelBlue = imageAddress[blueOffset] / kMaxColorComponentValue; + float originalPixelAlpha = imageAddress[alphaOffset] / kMaxColorComponentValue; + + float redOverlay = backgroundImageAddress[blueOffset] / kMaxColorComponentValue; + float greenOverlay = backgroundImageAddress[greenOffset] / kMaxColorComponentValue; + float blueOverlay = backgroundImageAddress[redOffset] / kMaxColorComponentValue; + float alphaOverlay = backgroundRegionRatio; + + // Calculate composite color component values. + // Derived from https://en.wikipedia.org/wiki/Alpha_compositing#Alpha_blending + float compositeAlpha = ((1.0f - alphaOverlay) * originalPixelAlpha) + alphaOverlay; + float compositeRed = 0.0f; + float compositeGreen = 0.0f; + float compositeBlue = 0.0f; + // Only perform rgb blending calculations if the output alpha is > 0. A zero-value alpha + // means none of the color channels actually matter, and would introduce division by 0. + if (fabs(compositeAlpha) > FLT_EPSILON) { + compositeRed = (((1.0f - alphaOverlay) * originalPixelAlpha * originalPixelRed) + + (alphaOverlay * redOverlay)) / + compositeAlpha; + compositeGreen = (((1.0f - alphaOverlay) * originalPixelAlpha * originalPixelGreen) + + (alphaOverlay * greenOverlay)) / + compositeAlpha; + compositeBlue = (((1.0f - alphaOverlay) * originalPixelAlpha * originalPixelBlue) + + (alphaOverlay * blueOverlay)) / + compositeAlpha; + } + + imageAddress[blueOffset] = compositeBlue * kMaxColorComponentValue; + imageAddress[greenOffset] = compositeGreen * kMaxColorComponentValue; + imageAddress[redOffset] = compositeRed * kMaxColorComponentValue; + imageAddress[alphaOffset] = compositeAlpha * kMaxColorComponentValue; + } + imageAddress += bytesPerRow / sizeof(unsigned char); + backgroundImageAddress += backgroundImageBytesPerRow / sizeof(unsigned char); + maskAddress += maskBytesPerRow / sizeof(float); + } + + CVPixelBufferUnlockBaseAddress(imageBuffer, 0); + CVPixelBufferUnlockBaseAddress(currentBackground, 0); + CVPixelBufferUnlockBaseAddress(mask.buffer, kCVPixelBufferLock_ReadOnly); +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m index 654bbaa44..5e2bcc567 100644 --- a/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m +++ b/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m @@ -45,10 +45,26 @@ - (RTCVideoTrack *)createVideoTrack:(NSDictionary *)constraints { RTCVideoTrack *videoTrack = [self.peerConnectionFactory videoTrackWithSource:videoSource trackId:trackUUID]; #if !TARGET_IPHONE_SIMULATOR - RTCCameraVideoCapturer *videoCapturer = [[RTCCameraVideoCapturer alloc] initWithDelegate:videoSource]; + NSDictionary *videoContraints = constraints[@"video"]; + RTCCameraVideoCapturer *videoCapturer; + + NSLog(@"Video constraint in create video track: %@", videoContraints); + + // If virtual backround is enabled, use video source interceptor before video source + if (videoContraints[@"vb"]) { + self.videoSourceInterceptor = [[VideoSourceInterceptor alloc]initWithVideoSource:videoSource]; + videoCapturer = [[RTCCameraVideoCapturer alloc] initWithDelegate:self.videoSourceInterceptor]; + } + else { + videoCapturer = [[RTCCameraVideoCapturer alloc] initWithDelegate:videoSource]; + } + +// self.videoSourceInterceptor = [[VideoSourceInterceptor alloc]initWithVideoSource:videoSource]; +// videoCapturer = [[RTCCameraVideoCapturer alloc] initWithDelegate:self.videoSourceInterceptor]; + VideoCaptureController *videoCaptureController = [[VideoCaptureController alloc] initWithCapturer:videoCapturer - andConstraints:constraints[@"video"]]; + andConstraints:videoContraints]; videoTrack.captureController = videoCaptureController; [videoCaptureController startCapture]; #endif @@ -114,6 +130,9 @@ - (RTCVideoTrack *)createScreenCaptureVideoTrack { RCT_EXPORT_METHOD(getUserMedia:(NSDictionary *)constraints successCallback:(RCTResponseSenderBlock)successCallback errorCallback:(RCTResponseSenderBlock)errorCallback) { + + NSLog(@"Video constraint in RTCMediaStream get user media %@", constraints); + RTCAudioTrack *audioTrack = nil; RTCVideoTrack *videoTrack = nil; @@ -278,6 +297,9 @@ - (RTCVideoTrack *)createScreenCaptureVideoTrack { track.isEnabled = NO; [track.captureController stopCapture]; [self.localTracks removeObjectForKey:trackID]; + if([track.kind isEqualToString:kRTCMediaStreamTrackKindVideo]) { + self.videoSourceInterceptor = nil; + } } } diff --git a/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m b/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m index 822f58b59..8ac7c63da 100644 --- a/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m +++ b/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m @@ -444,8 +444,6 @@ - (void)appendValue:(NSObject *)statisticsValue toString:(NSMutableString *)s { NSString *jsonString = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding]; [s appendString:jsonString]; } - } else if ([statisticsValue isKindOfClass:[NSNumber class]]) { - [s appendString:[NSString stringWithFormat:@"%@", statisticsValue]]; } else { [s appendString:@"\""]; [s appendString:[NSString stringWithFormat:@"%@", statisticsValue]]; diff --git a/ios/RCTWebRTC/WebRTCModule.h b/ios/RCTWebRTC/WebRTCModule.h index dee0d71c6..797d8c8e0 100644 --- a/ios/RCTWebRTC/WebRTCModule.h +++ b/ios/RCTWebRTC/WebRTCModule.h @@ -19,6 +19,7 @@ #import #import #import +#import "VideoSourceInterceptor.h" static NSString *const kEventPeerConnectionSignalingStateChanged = @"peerConnectionSignalingStateChanged"; static NSString *const kEventPeerConnectionStateChanged = @"peerConnectionStateChanged"; @@ -42,6 +43,7 @@ static NSString *const kEventMediaStreamTrackMuteChanged = @"mediaStreamTrackMut @property (nonatomic, strong) NSMutableDictionary *peerConnections; @property (nonatomic, strong) NSMutableDictionary *localStreams; @property (nonatomic, strong) NSMutableDictionary *localTracks; +@property (nonatomic, strong) VideoSourceInterceptor *videoSourceInterceptor; - (instancetype)initWithEncoderFactory:(id)encoderFactory decoderFactory:(id)decoderFactory; diff --git a/package.json b/package.json index a2ca75919..3c1d5a2f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-native-webrtc", - "version": "1.100.1", + "version": "1.100.3", "repository": { "type": "git", "url": "git+https://github.com/react-native-webrtc/react-native-webrtc.git" diff --git a/src/RTCUtil.ts b/src/RTCUtil.ts index 1d1c6d7eb..0568340fe 100644 --- a/src/RTCUtil.ts +++ b/src/RTCUtil.ts @@ -200,5 +200,11 @@ export function normalizeConstraints(constraints) { } } + if(constraints['video'] && constraints['video'].hasOwnProperty('vb')) { + if(c['video']) { + c['video'].vb = true; + } + } + return c; } diff --git a/src/getUserMedia.ts b/src/getUserMedia.ts index 37a647407..dd9511aa4 100644 --- a/src/getUserMedia.ts +++ b/src/getUserMedia.ts @@ -30,6 +30,7 @@ export default function getUserMedia(constraints: Constraints = {}) { // Normalize constraints. constraints = RTCUtil.normalizeConstraints(constraints); + console.log("Constraints in getUserMedia: ", constraints); // Request required permissions const reqPermissions: Array> = [];