From 093d9f0d9e106d71a2177c8b4055995aa3762459 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 5 Sep 2023 13:48:30 +0200 Subject: [PATCH 1/2] detect when user is talking and send data channel message Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index c86b080c1b..ecfe0d43cc 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -39,7 +39,10 @@ import android.graphics.Color import android.graphics.Typeface import android.graphics.drawable.Icon import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioRecord import android.media.MediaPlayer +import android.media.MediaRecorder import android.net.Uri import android.os.Build import android.os.Bundle @@ -181,6 +184,7 @@ import java.util.Objects import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger import javax.inject.Inject +import kotlin.math.abs import kotlin.math.roundToInt @AutoInjector(NextcloudTalkApplication::class) @@ -363,6 +367,15 @@ class CallActivity : CallBaseActivity() { private var reactionAnimator: ReactionAnimator? = null private var othersInCall = false + private lateinit var micInputAudioRecorder: AudioRecord + private var micInputAudioRecordThread: Thread? = null + private var isMicInputAudioThreadRunning: Boolean = false + private val bufferSize = AudioRecord.getMinBufferSize( + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT + ) + @SuppressLint("ClickableViewAccessibility") override fun onCreate(savedInstanceState: Bundle?) { Log.d(TAG, "onCreate") @@ -523,6 +536,19 @@ class CallActivity : CallBaseActivity() { override fun onStop() { super.onStop() active = false + + if (isMicInputAudioThreadRunning) { + stopMicInputDetection() + } + } + + private fun stopMicInputDetection() { + if (micInputAudioRecordThread != null) { + micInputAudioRecorder.stop() + micInputAudioRecorder.release() + isMicInputAudioThreadRunning = false + micInputAudioRecordThread = null + } } private fun enableBluetoothManager() { @@ -999,6 +1025,8 @@ class CallActivity : CallBaseActivity() { } private fun microphoneInitialization() { + startMicInputDetection() + // create an AudioSource instance audioSource = peerConnectionFactory!!.createAudioSource(audioConstraints) localAudioTrack = peerConnectionFactory!!.createAudioTrack("NCa0", audioSource) @@ -1006,6 +1034,61 @@ class CallActivity : CallBaseActivity() { localStream!!.addTrack(localAudioTrack) } + @SuppressLint("MissingPermission") + private fun startMicInputDetection() { + if (permissionUtil!!.isMicrophonePermissionGranted() && micInputAudioRecordThread == null) { + var isSpeakingLongTerm = false + micInputAudioRecorder = AudioRecord( + MediaRecorder.AudioSource.MIC, + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + bufferSize + ) + isMicInputAudioThreadRunning = true + micInputAudioRecorder.startRecording() + micInputAudioRecordThread = Thread( + Runnable { + while (isMicInputAudioThreadRunning) { + val byteArr = ByteArray(bufferSize / 2) + micInputAudioRecorder.read(byteArr, 0, byteArr.size) + val isCurrentlySpeaking = abs(byteArr[0].toDouble()) > MICROPHONE_VALUE_THRESHOLD + + if (isCurrentlySpeaking && !isSpeakingLongTerm) { + isSpeakingLongTerm = true + sendIsSpeakingMessage(true) + } else if (!isCurrentlySpeaking && isSpeakingLongTerm) { + isSpeakingLongTerm = false + sendIsSpeakingMessage(false) + } + Thread.sleep(MICROPHONE_VALUE_SLEEP) + } + } + ) + micInputAudioRecordThread!!.start() + } + } + + private fun sendIsSpeakingMessage(isSpeaking: Boolean) { + val isSpeakingMessage: String = + if (isSpeaking) SIGNALING_MESSAGE_SPEAKING_STARTED else SIGNALING_MESSAGE_SPEAKING_STOPPED + + if (isConnectionEstablished && othersInCall) { + if (!hasMCU) { + for (peerConnectionWrapper in peerConnectionWrapperList) { + peerConnectionWrapper.sendChannelData(DataChannelMessage(isSpeakingMessage)) + } + } else { + for (peerConnectionWrapper in peerConnectionWrapperList) { + if (peerConnectionWrapper.sessionId == webSocketClient!!.sessionId) { + peerConnectionWrapper.sendChannelData(DataChannelMessage(isSpeakingMessage)) + break + } + } + } + } + } + private fun createCameraCapturer(enumerator: CameraEnumerator?): VideoCapturer? { val deviceNames = enumerator!!.deviceNames @@ -3000,5 +3083,12 @@ class CallActivity : CallBaseActivity() { const val CALL_TIME_ONE_HOUR = 3600 const val CALL_DURATION_EMPTY = "--:--" const val API_RETRIES: Long = 3 + + private const val SAMPLE_RATE = 8000 + private const val MICROPHONE_VALUE_THRESHOLD = 20 + private const val MICROPHONE_VALUE_SLEEP: Long = 1000 + + private const val SIGNALING_MESSAGE_SPEAKING_STARTED = "speaking" + private const val SIGNALING_MESSAGE_SPEAKING_STOPPED = "stoppedSpeaking" } } From 3db829c644c8a9379da43f8faf2c8b2a4fb34b43 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Tue, 5 Sep 2023 15:54:25 +0200 Subject: [PATCH 2/2] solve ktlint + detekt warnings Signed-off-by: Marcel Hibbe --- .../nextcloud/talk/activities/CallActivity.kt | 17 ++++++++++++----- app/src/main/res/layout/item_system_message.xml | 3 ++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index ecfe0d43cc..268e42eae2 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -1069,6 +1069,7 @@ class CallActivity : CallBaseActivity() { } } + @Suppress("Detekt.NestedBlockDepth") private fun sendIsSpeakingMessage(isSpeaking: Boolean) { val isSpeakingMessage: String = if (isSpeaking) SIGNALING_MESSAGE_SPEAKING_STARTED else SIGNALING_MESSAGE_SPEAKING_STOPPED @@ -1234,10 +1235,10 @@ class CallActivity : CallBaseActivity() { private fun toggleMedia(enable: Boolean, video: Boolean) { var message: String if (video) { - message = "videoOff" + message = SIGNALING_MESSAGE_VIDEO_OFF if (enable) { binding!!.cameraButton.alpha = OPACITY_ENABLED - message = "videoOn" + message = SIGNALING_MESSAGE_VIDEO_ON startVideoCapture() } else { binding!!.cameraButton.alpha = OPACITY_DISABLED @@ -1258,9 +1259,9 @@ class CallActivity : CallBaseActivity() { binding!!.selfVideoRenderer.visibility = View.INVISIBLE } } else { - message = "audioOff" + message = SIGNALING_MESSAGE_AUDIO_OFF if (enable) { - message = "audioOn" + message = SIGNALING_MESSAGE_AUDIO_ON binding!!.microphoneButton.alpha = OPACITY_ENABLED } else { binding!!.microphoneButton.alpha = OPACITY_DISABLED @@ -3050,7 +3051,7 @@ class CallActivity : CallBaseActivity() { val newX = event.rawX - binding!!.selfVideoViewWrapper.width / 2f binding!!.selfVideoViewWrapper.y = newY binding!!.selfVideoViewWrapper.x = newX - } else if (event.actionMasked == MotionEvent.ACTION_UP && duration < 100) { + } else if (event.actionMasked == MotionEvent.ACTION_UP && duration < SWITCH_CAMERA_THRESHOLD_DURATION) { switchCamera() } return true @@ -3084,11 +3085,17 @@ class CallActivity : CallBaseActivity() { const val CALL_DURATION_EMPTY = "--:--" const val API_RETRIES: Long = 3 + const val SWITCH_CAMERA_THRESHOLD_DURATION = 100 + private const val SAMPLE_RATE = 8000 private const val MICROPHONE_VALUE_THRESHOLD = 20 private const val MICROPHONE_VALUE_SLEEP: Long = 1000 private const val SIGNALING_MESSAGE_SPEAKING_STARTED = "speaking" private const val SIGNALING_MESSAGE_SPEAKING_STOPPED = "stoppedSpeaking" + private const val SIGNALING_MESSAGE_VIDEO_ON = "videoOn" + private const val SIGNALING_MESSAGE_VIDEO_OFF = "videoOff" + private const val SIGNALING_MESSAGE_AUDIO_ON = "audioOn" + private const val SIGNALING_MESSAGE_AUDIO_OFF = "audioOff" } } diff --git a/app/src/main/res/layout/item_system_message.xml b/app/src/main/res/layout/item_system_message.xml index 4448611b1a..abdc531862 100644 --- a/app/src/main/res/layout/item_system_message.xml +++ b/app/src/main/res/layout/item_system_message.xml @@ -33,7 +33,8 @@ android:layout_marginStart="@dimen/standard_margin" android:layout_marginTop="@dimen/standard_eighth_margin" android:layout_marginEnd="@dimen/standard_margin" - android:layout_marginBottom="@dimen/standard_eighth_margin"> + android:layout_marginBottom="@dimen/standard_eighth_margin" + tools:ignore="UselessParent">