diff --git a/app/src/main/kotlin/com/wire/android/ui/home/conversations/model/AssetBundle.kt b/app/src/main/kotlin/com/wire/android/ui/home/conversations/model/AssetBundle.kt index b6ee61b96d7..c25a395b769 100644 --- a/app/src/main/kotlin/com/wire/android/ui/home/conversations/model/AssetBundle.kt +++ b/app/src/main/kotlin/com/wire/android/ui/home/conversations/model/AssetBundle.kt @@ -69,7 +69,8 @@ data class AssetBundle( */ data class UriAsset( val uri: Uri, - val saveToDeviceIfInvalid: Boolean = false + val saveToDeviceIfInvalid: Boolean = false, + val mimeType: String? = null ) private object PathAsStringSerializer : KSerializer { diff --git a/app/src/main/kotlin/com/wire/android/ui/home/conversations/sendmessage/SendMessageViewModel.kt b/app/src/main/kotlin/com/wire/android/ui/home/conversations/sendmessage/SendMessageViewModel.kt index ebf8afaffb1..ff6a7a24fea 100644 --- a/app/src/main/kotlin/com/wire/android/ui/home/conversations/sendmessage/SendMessageViewModel.kt +++ b/app/src/main/kotlin/com/wire/android/ui/home/conversations/sendmessage/SendMessageViewModel.kt @@ -40,7 +40,6 @@ import com.wire.android.ui.home.messagecomposer.model.MessageBundle import com.wire.android.ui.home.messagecomposer.model.Ping import com.wire.android.ui.navArgs import com.wire.android.ui.sharing.SendMessagesSnackbarMessages -import com.wire.android.util.SUPPORTED_AUDIO_MIME_TYPE import com.wire.android.util.ImageUtil import com.wire.android.util.dispatchers.DispatcherProvider import com.wire.android.util.getAudioLengthInMs @@ -208,8 +207,7 @@ class SendMessageViewModel @Inject constructor( is ComposableMessageBundle.AudioMessageBundle -> { handleAssetMessageBundle( attachmentUri = messageBundle.attachmentUri, - conversationId = messageBundle.conversationId, - specifiedMimeType = SUPPORTED_AUDIO_MIME_TYPE, + conversationId = messageBundle.conversationId ) } @@ -244,13 +242,12 @@ class SendMessageViewModel @Inject constructor( private suspend fun handleAssetMessageBundle( conversationId: ConversationId, - attachmentUri: UriAsset, - specifiedMimeType: String? = null, // specify a particular mimetype, otherwise it will be taken from the uri / file extension + attachmentUri: UriAsset ) { when (val result = handleUriAsset.invoke( uri = attachmentUri.uri, saveToDeviceIfInvalid = attachmentUri.saveToDeviceIfInvalid, - specifiedMimeType = specifiedMimeType + specifiedMimeType = attachmentUri.mimeType )) { is HandleUriAssetUseCase.Result.Failure.AssetTooLarge -> { assetTooLargeDialogState = AssetTooLargeDialogState.Visible( diff --git a/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/AudioMediaRecorder.kt b/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/AudioMediaRecorder.kt index dc327c31fdc..ec0d9ed2750 100644 --- a/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/AudioMediaRecorder.kt +++ b/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/AudioMediaRecorder.kt @@ -20,6 +20,11 @@ package com.wire.android.ui.home.messagecomposer.recordaudio import android.annotation.SuppressLint import android.media.AudioFormat import android.media.AudioRecord +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaExtractor +import android.media.MediaFormat +import android.media.MediaMuxer import android.media.MediaRecorder import com.wire.android.appLogger import com.wire.android.util.dispatchers.DispatcherProvider @@ -29,13 +34,17 @@ import com.wire.kalium.logic.feature.asset.GetAssetSizeLimitUseCaseImpl.Companio import com.wire.kalium.util.DateTimeUtil import dagger.hilt.android.scopes.ViewModelScoped import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import okio.Path import okio.buffer +import java.io.File +import java.io.FileInputStream import java.io.IOException import java.nio.ByteBuffer import java.nio.ByteOrder @@ -58,6 +67,7 @@ class AudioMediaRecorder @Inject constructor( var originalOutputPath: Path? = null var effectsOutputPath: Path? = null + var mp4OutputPath: Path? = null private val _maxFileSizeReached = MutableSharedFlow() fun getMaxFileSizeReached(): Flow = @@ -86,6 +96,9 @@ class AudioMediaRecorder @Inject constructor( effectsOutputPath = kaliumFileSystem .tempFilePath(getRecordingAudioEffectsFileName()) + + mp4OutputPath = kaliumFileSystem + .tempFilePath(getRecordingMP4AudioFileName()) } } @@ -209,13 +222,128 @@ class AudioMediaRecorder @Inject constructor( } } + @Suppress("LongMethod", "CyclomaticComplexMethod") + suspend fun convertWavToMp4(shouldApplyEffects: Boolean): Boolean = withContext(Dispatchers.IO) { + var codec: MediaCodec? = null + var muxer: MediaMuxer? = null + var fileInputStream: FileInputStream? = null + + try { + val inputFilePath = if (shouldApplyEffects) { + effectsOutputPath!!.toString() + } else { + originalOutputPath!!.toString() + } + + val inputFile = File(inputFilePath) + fileInputStream = FileInputStream(inputFile) + + val mediaExtractor = MediaExtractor() + mediaExtractor.setDataSource(inputFilePath) + + val format = MediaFormat.createAudioFormat( + MediaFormat.MIMETYPE_AUDIO_AAC, + SAMPLING_RATE, + AUDIO_CHANNELS + ) + format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE) + format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC) + + codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC) + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + codec.start() + + val bufferInfo = MediaCodec.BufferInfo() + muxer = MediaMuxer(mp4OutputPath.toString(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + var trackIndex = -1 + var sawInputEOS = false + var sawOutputEOS = false + + while (!sawOutputEOS) { + if (!sawInputEOS) { + val inputBufferIndex = codec.dequeueInputBuffer(TIMEOUT_US) + if (inputBufferIndex >= 0) { + val inputBuffer = codec.getInputBuffer(inputBufferIndex) + inputBuffer?.clear() + + val sampleSize = fileInputStream.channel.read(inputBuffer!!) + if (sampleSize < 0) { + codec.queueInputBuffer(inputBufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + sawInputEOS = true + } else { + val presentationTimeUs = System.nanoTime() / NANOSECONDS_IN_MICROSECOND + codec.queueInputBuffer(inputBufferIndex, 0, sampleSize, presentationTimeUs, 0) + } + } + } + + val outputBufferIndex = codec.dequeueOutputBuffer(bufferInfo, TIMEOUT_US) + if (outputBufferIndex >= 0) { + val outputBuffer = codec.getOutputBuffer(outputBufferIndex) + + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { + codec.releaseOutputBuffer(outputBufferIndex, false) + continue + } + + if (bufferInfo.size != 0) { + outputBuffer?.position(bufferInfo.offset) + outputBuffer?.limit(bufferInfo.offset + bufferInfo.size) + + if (trackIndex == -1) { + val newFormat = codec.outputFormat + trackIndex = muxer.addTrack(newFormat) + muxer.start() + } + + muxer.writeSampleData(trackIndex, outputBuffer!!, bufferInfo) + } + + codec.releaseOutputBuffer(outputBufferIndex, false) + + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { + sawOutputEOS = true + } + } + } + true + } catch (e: IllegalStateException) { + appLogger.e("Could not convert wav to mp4: ${e.message}", throwable = e) + false + } catch (e: IOException) { + appLogger.e("Could not convert wav to mp4: ${e.message}", throwable = e) + false + } finally { + try { + fileInputStream?.close() + } catch (e: IOException) { + appLogger.e("Could not close FileInputStream: ${e.message}", throwable = e) + } + + try { + muxer?.stop() + muxer?.release() + } catch (e: IllegalStateException) { + appLogger.e("Could not stop or release MediaMuxer: ${e.message}", throwable = e) + } + + try { + codec?.stop() + codec?.release() + } catch (e: IllegalStateException) { + appLogger.e("Could not stop or release MediaCodec: ${e.message}", throwable = e) + } + } + } + companion object { fun getRecordingAudioFileName(): String = "wire-audio-${DateTimeUtil.currentInstant().fileDateTime()}.wav" + fun getRecordingMP4AudioFileName(): String = "wire-audio-${DateTimeUtil.currentInstant().fileDateTime()}.mp4" fun getRecordingAudioEffectsFileName(): String = "wire-audio-${DateTimeUtil.currentInstant().fileDateTime()}-filter.wav" const val SIZE_OF_1MB = 1024 * 1024 const val AUDIO_CHANNELS = 1 // Mono - const val SAMPLING_RATE = 44100 + const val SAMPLING_RATE = 16000 const val BUFFER_SIZE = 1024 const val AUDIO_ENCODING = AudioFormat.ENCODING_PCM_16BIT const val BITS_PER_SAMPLE = 16 @@ -233,5 +361,9 @@ class AudioMediaRecorder @Inject constructor( const val FORMAT_WAVE = "WAVE" const val SUBCHUNK1_ID_FMT = "fmt " const val SUBCHUNK2_ID_DATA = "data" + + private const val BIT_RATE = 64000 + private const val TIMEOUT_US: Long = 10000 + const val NANOSECONDS_IN_MICROSECOND = 1000 } } diff --git a/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/GenerateAudioFileWithEffectsUseCase.kt b/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/GenerateAudioFileWithEffectsUseCase.kt index 432b65ff2ba..454c0e5c52a 100644 --- a/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/GenerateAudioFileWithEffectsUseCase.kt +++ b/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/GenerateAudioFileWithEffectsUseCase.kt @@ -38,7 +38,7 @@ class GenerateAudioFileWithEffectsUseCase @Inject constructor( suspend operator fun invoke( context: Context, originalFilePath: String, - effectsFilePath: String, + effectsFilePath: String ) = withContext(dispatchers.io()) { appLogger.i("[$TAG] -> Start generating audio file with effects") diff --git a/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/RecordAudioViewModel.kt b/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/RecordAudioViewModel.kt index 024c6e9a0eb..ea5c2d05a87 100644 --- a/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/RecordAudioViewModel.kt +++ b/app/src/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/RecordAudioViewModel.kt @@ -267,26 +267,49 @@ class RecordAudioViewModel @Inject constructor( viewModelScope.launch { recordAudioMessagePlayer.stop() recordAudioMessagePlayer.close() + state = state.copy( + buttonState = RecordAudioButtonState.ENCODING, audioState = AudioState.DEFAULT, + originalOutputFile = null, + effectsOutputFile = null + ) - val resultFile = if (state.shouldApplyEffects) { - try { - state.originalOutputFile?.toPath()?.deleteIfExists() - } catch (exception: IOException) { - appLogger.e("[$tag] -> Couldn't delete original audio file before sending audio file with effects.") - } - state.effectsOutputFile!!.toUri() - } else { - try { - state.effectsOutputFile?.toPath()?.deleteIfExists() - } catch (exception: IOException) { - appLogger.e("[$tag] -> Couldn't delete audio file with effects before sending original audio file.") + val didSucceed = audioMediaRecorder.convertWavToMp4(state.shouldApplyEffects) + + try { + when { + didSucceed -> { + state.originalOutputFile?.toPath()?.deleteIfExists() + state.effectsOutputFile?.toPath()?.deleteIfExists() + } + + state.shouldApplyEffects -> { + state.originalOutputFile?.toPath()?.deleteIfExists() + } + + !state.shouldApplyEffects -> { + state.effectsOutputFile?.toPath()?.deleteIfExists() + } } - state.originalOutputFile!!.toUri() + } catch (exception: IOException) { + appLogger.e("[$tag] -> Couldn't delete audio files") } onAudioRecorded( UriAsset( - uri = resultFile, + uri = if (didSucceed) { + audioMediaRecorder.mp4OutputPath!!.toFile().toUri() + } else { + if (state.shouldApplyEffects) { + audioMediaRecorder.effectsOutputPath!!.toFile().toUri() + } else { + audioMediaRecorder.originalOutputPath!!.toFile().toUri() + } + }, + mimeType = if (didSucceed) { + "audio/mp4" + } else { + "audio/wav" + }, saveToDeviceIfInvalid = false ) )