Skip to content

Commit

Permalink
Merge pull request #3301 from nextcloud/backport/3298/detectTalkingAn…
Browse files Browse the repository at this point in the history 
…dSendDataChannelMessage_stable17.1

Backport/3298/detect talking and send data channel message stable17.1
  • Loading branch information
@mahibi
mahibi authored Sep 6, 2023
2 parents f4a7393 + 3db829c commit 0f5fed1
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 6 deletions.
107 changes: 102 additions & 5 deletions app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,10 @@ import android.graphics.Color
import android.graphics.Typeface
import android.graphics.drawable.Icon
import android.media.AudioAttributes
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaPlayer
import android.media.MediaRecorder
import android.net.Uri
import android.os.Build
import android.os.Bundle
Expand Down Expand Up @@ -181,6 +184,7 @@ import java.util.Objects
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import javax.inject.Inject
import kotlin.math.abs
import kotlin.math.roundToInt

@AutoInjector(NextcloudTalkApplication::class)
Expand Down Expand Up @@ -363,6 +367,15 @@ class CallActivity : CallBaseActivity() {
private var reactionAnimator: ReactionAnimator? = null
private var othersInCall = false

private lateinit var micInputAudioRecorder: AudioRecord
private var micInputAudioRecordThread: Thread? = null
private var isMicInputAudioThreadRunning: Boolean = false
private val bufferSize = AudioRecord.getMinBufferSize(
SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
)

@SuppressLint("ClickableViewAccessibility")
override fun onCreate(savedInstanceState: Bundle?) {
Log.d(TAG, "onCreate")
Expand Down Expand Up @@ -523,6 +536,19 @@ class CallActivity : CallBaseActivity() {
override fun onStop() {
super.onStop()
active = false

if (isMicInputAudioThreadRunning) {
stopMicInputDetection()
}
}

private fun stopMicInputDetection() {
if (micInputAudioRecordThread != null) {
micInputAudioRecorder.stop()
micInputAudioRecorder.release()
isMicInputAudioThreadRunning = false
micInputAudioRecordThread = null
}
}

private fun enableBluetoothManager() {
Expand Down Expand Up @@ -999,13 +1025,71 @@ class CallActivity : CallBaseActivity() {
}

private fun microphoneInitialization() {
startMicInputDetection()

// create an AudioSource instance
audioSource = peerConnectionFactory!!.createAudioSource(audioConstraints)
localAudioTrack = peerConnectionFactory!!.createAudioTrack("NCa0", audioSource)
localAudioTrack!!.setEnabled(false)
localStream!!.addTrack(localAudioTrack)
}

@SuppressLint("MissingPermission")
private fun startMicInputDetection() {
if (permissionUtil!!.isMicrophonePermissionGranted() && micInputAudioRecordThread == null) {
var isSpeakingLongTerm = false
micInputAudioRecorder = AudioRecord(
MediaRecorder.AudioSource.MIC,
SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
bufferSize
)
isMicInputAudioThreadRunning = true
micInputAudioRecorder.startRecording()
micInputAudioRecordThread = Thread(
Runnable {
while (isMicInputAudioThreadRunning) {
val byteArr = ByteArray(bufferSize / 2)
micInputAudioRecorder.read(byteArr, 0, byteArr.size)
val isCurrentlySpeaking = abs(byteArr[0].toDouble()) > MICROPHONE_VALUE_THRESHOLD

if (isCurrentlySpeaking && !isSpeakingLongTerm) {
isSpeakingLongTerm = true
sendIsSpeakingMessage(true)
} else if (!isCurrentlySpeaking && isSpeakingLongTerm) {
isSpeakingLongTerm = false
sendIsSpeakingMessage(false)
}
Thread.sleep(MICROPHONE_VALUE_SLEEP)
}
}
)
micInputAudioRecordThread!!.start()
}
}

@Suppress("Detekt.NestedBlockDepth")
private fun sendIsSpeakingMessage(isSpeaking: Boolean) {
val isSpeakingMessage: String =
if (isSpeaking) SIGNALING_MESSAGE_SPEAKING_STARTED else SIGNALING_MESSAGE_SPEAKING_STOPPED

if (isConnectionEstablished && othersInCall) {
if (!hasMCU) {
for (peerConnectionWrapper in peerConnectionWrapperList) {
peerConnectionWrapper.sendChannelData(DataChannelMessage(isSpeakingMessage))
}
} else {
for (peerConnectionWrapper in peerConnectionWrapperList) {
if (peerConnectionWrapper.sessionId == webSocketClient!!.sessionId) {
peerConnectionWrapper.sendChannelData(DataChannelMessage(isSpeakingMessage))
break
}
}
}
}
}

private fun createCameraCapturer(enumerator: CameraEnumerator?): VideoCapturer? {
val deviceNames = enumerator!!.deviceNames

Expand Down Expand Up @@ -1151,10 +1235,10 @@ class CallActivity : CallBaseActivity() {
private fun toggleMedia(enable: Boolean, video: Boolean) {
var message: String
if (video) {
message = "videoOff"
message = SIGNALING_MESSAGE_VIDEO_OFF
if (enable) {
binding!!.cameraButton.alpha = OPACITY_ENABLED
message = "videoOn"
message = SIGNALING_MESSAGE_VIDEO_ON
startVideoCapture()
} else {
binding!!.cameraButton.alpha = OPACITY_DISABLED
Expand All @@ -1175,9 +1259,9 @@ class CallActivity : CallBaseActivity() {
binding!!.selfVideoRenderer.visibility = View.INVISIBLE
}
} else {
message = "audioOff"
message = SIGNALING_MESSAGE_AUDIO_OFF
if (enable) {
message = "audioOn"
message = SIGNALING_MESSAGE_AUDIO_ON
binding!!.microphoneButton.alpha = OPACITY_ENABLED
} else {
binding!!.microphoneButton.alpha = OPACITY_DISABLED
Expand Down Expand Up @@ -2967,7 +3051,7 @@ class CallActivity : CallBaseActivity() {
val newX = event.rawX - binding!!.selfVideoViewWrapper.width / 2f
binding!!.selfVideoViewWrapper.y = newY
binding!!.selfVideoViewWrapper.x = newX
} else if (event.actionMasked == MotionEvent.ACTION_UP && duration < 100) {
} else if (event.actionMasked == MotionEvent.ACTION_UP && duration < SWITCH_CAMERA_THRESHOLD_DURATION) {
switchCamera()
}
return true
Expand Down Expand Up @@ -3000,5 +3084,18 @@ class CallActivity : CallBaseActivity() {
const val CALL_TIME_ONE_HOUR = 3600
const val CALL_DURATION_EMPTY = "--:--"
const val API_RETRIES: Long = 3

const val SWITCH_CAMERA_THRESHOLD_DURATION = 100

private const val SAMPLE_RATE = 8000
private const val MICROPHONE_VALUE_THRESHOLD = 20
private const val MICROPHONE_VALUE_SLEEP: Long = 1000

private const val SIGNALING_MESSAGE_SPEAKING_STARTED = "speaking"
private const val SIGNALING_MESSAGE_SPEAKING_STOPPED = "stoppedSpeaking"
private const val SIGNALING_MESSAGE_VIDEO_ON = "videoOn"
private const val SIGNALING_MESSAGE_VIDEO_OFF = "videoOff"
private const val SIGNALING_MESSAGE_AUDIO_ON = "audioOn"
private const val SIGNALING_MESSAGE_AUDIO_OFF = "audioOff"
}
}
3 changes: 2 additions & 1 deletion app/src/main/res/layout/item_system_message.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@
android:layout_marginStart="@dimen/standard_margin"
android:layout_marginTop="@dimen/standard_eighth_margin"
android:layout_marginEnd="@dimen/standard_margin"
android:layout_marginBottom="@dimen/standard_eighth_margin">
android:layout_marginBottom="@dimen/standard_eighth_margin"
tools:ignore="UselessParent">

<ImageView
android:id="@+id/expandCollapseIcon"
Expand Down

0 comments on commit 0f5fed1

Please sign in to comment.