Skip to content

Commit

Permalink
fix: improve audio file size [WPB-10001] (#3261)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Garzas @github-actions
Garzas authored and github-actions[bot] committed Aug 6, 2024
1 parent 55117ba commit 8a32e3d
Show file tree
Hide file tree
Showing 5 changed files with 176 additions and 23 deletions.
3 changes: 2 additions & 1 deletion app/src/main/kotlin/com/wire/android/ui/home/conversations/model/AssetBundle.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ data class AssetBundle(
*/
data class UriAsset(
val uri: Uri,
val saveToDeviceIfInvalid: Boolean = false
val saveToDeviceIfInvalid: Boolean = false,
val mimeType: String? = null
)

private object PathAsStringSerializer : KSerializer<Path> {
Expand Down
9 changes: 3 additions & 6 deletions ...rc/main/kotlin/com/wire/android/ui/home/conversations/sendmessage/SendMessageViewModel.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ import com.wire.android.ui.home.messagecomposer.model.MessageBundle
import com.wire.android.ui.home.messagecomposer.model.Ping
import com.wire.android.ui.navArgs
import com.wire.android.ui.sharing.SendMessagesSnackbarMessages
import com.wire.android.util.SUPPORTED_AUDIO_MIME_TYPE
import com.wire.android.util.ImageUtil
import com.wire.android.util.dispatchers.DispatcherProvider
import com.wire.android.util.getAudioLengthInMs
Expand Down Expand Up @@ -208,8 +207,7 @@ class SendMessageViewModel @Inject constructor(
is ComposableMessageBundle.AudioMessageBundle -> {
handleAssetMessageBundle(
attachmentUri = messageBundle.attachmentUri,
conversationId = messageBundle.conversationId,
specifiedMimeType = SUPPORTED_AUDIO_MIME_TYPE,
conversationId = messageBundle.conversationId
)
}

Expand Down Expand Up @@ -244,13 +242,12 @@ class SendMessageViewModel @Inject constructor(

private suspend fun handleAssetMessageBundle(
conversationId: ConversationId,
attachmentUri: UriAsset,
specifiedMimeType: String? = null, // specify a particular mimetype, otherwise it will be taken from the uri / file extension
attachmentUri: UriAsset
) {
when (val result = handleUriAsset.invoke(
uri = attachmentUri.uri,
saveToDeviceIfInvalid = attachmentUri.saveToDeviceIfInvalid,
specifiedMimeType = specifiedMimeType
specifiedMimeType = attachmentUri.mimeType
)) {
is HandleUriAssetUseCase.Result.Failure.AssetTooLarge -> {
assetTooLargeDialogState = AssetTooLargeDialogState.Visible(
Expand Down
134 changes: 133 additions & 1 deletion ...rc/main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/AudioMediaRecorder.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ package com.wire.android.ui.home.messagecomposer.recordaudio
import android.annotation.SuppressLint
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaCodec
import android.media.MediaCodecInfo
import android.media.MediaExtractor
import android.media.MediaFormat
import android.media.MediaMuxer
import android.media.MediaRecorder
import com.wire.android.appLogger
import com.wire.android.util.dispatchers.DispatcherProvider
Expand All @@ -29,13 +34,17 @@ import com.wire.kalium.logic.feature.asset.GetAssetSizeLimitUseCaseImpl.Companio
import com.wire.kalium.util.DateTimeUtil
import dagger.hilt.android.scopes.ViewModelScoped
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import okio.Path
import okio.buffer
import java.io.File
import java.io.FileInputStream
import java.io.IOException
import java.nio.ByteBuffer
import java.nio.ByteOrder
Expand All @@ -58,6 +67,7 @@ class AudioMediaRecorder @Inject constructor(

var originalOutputPath: Path? = null
var effectsOutputPath: Path? = null
var mp4OutputPath: Path? = null

private val _maxFileSizeReached = MutableSharedFlow<RecordAudioDialogState>()
fun getMaxFileSizeReached(): Flow<RecordAudioDialogState> =
Expand Down Expand Up @@ -86,6 +96,9 @@ class AudioMediaRecorder @Inject constructor(

effectsOutputPath = kaliumFileSystem
.tempFilePath(getRecordingAudioEffectsFileName())

mp4OutputPath = kaliumFileSystem
.tempFilePath(getRecordingMP4AudioFileName())
}
}

Expand Down Expand Up @@ -209,13 +222,128 @@ class AudioMediaRecorder @Inject constructor(
}
}

@Suppress("LongMethod", "CyclomaticComplexMethod")
suspend fun convertWavToMp4(shouldApplyEffects: Boolean): Boolean = withContext(Dispatchers.IO) {
var codec: MediaCodec? = null
var muxer: MediaMuxer? = null
var fileInputStream: FileInputStream? = null

try {
val inputFilePath = if (shouldApplyEffects) {
effectsOutputPath!!.toString()
} else {
originalOutputPath!!.toString()
}

val inputFile = File(inputFilePath)
fileInputStream = FileInputStream(inputFile)

val mediaExtractor = MediaExtractor()
mediaExtractor.setDataSource(inputFilePath)

val format = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_AAC,
SAMPLING_RATE,
AUDIO_CHANNELS
)
format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE)
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC)

codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
codec.start()

val bufferInfo = MediaCodec.BufferInfo()
muxer = MediaMuxer(mp4OutputPath.toString(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
var trackIndex = -1
var sawInputEOS = false
var sawOutputEOS = false

while (!sawOutputEOS) {
if (!sawInputEOS) {
val inputBufferIndex = codec.dequeueInputBuffer(TIMEOUT_US)
if (inputBufferIndex >= 0) {
val inputBuffer = codec.getInputBuffer(inputBufferIndex)
inputBuffer?.clear()

val sampleSize = fileInputStream.channel.read(inputBuffer!!)
if (sampleSize < 0) {
codec.queueInputBuffer(inputBufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
sawInputEOS = true
} else {
val presentationTimeUs = System.nanoTime() / NANOSECONDS_IN_MICROSECOND
codec.queueInputBuffer(inputBufferIndex, 0, sampleSize, presentationTimeUs, 0)
}
}
}

val outputBufferIndex = codec.dequeueOutputBuffer(bufferInfo, TIMEOUT_US)
if (outputBufferIndex >= 0) {
val outputBuffer = codec.getOutputBuffer(outputBufferIndex)

if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) {
codec.releaseOutputBuffer(outputBufferIndex, false)
continue
}

if (bufferInfo.size != 0) {
outputBuffer?.position(bufferInfo.offset)
outputBuffer?.limit(bufferInfo.offset + bufferInfo.size)

if (trackIndex == -1) {
val newFormat = codec.outputFormat
trackIndex = muxer.addTrack(newFormat)
muxer.start()
}

muxer.writeSampleData(trackIndex, outputBuffer!!, bufferInfo)
}

codec.releaseOutputBuffer(outputBufferIndex, false)

if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
sawOutputEOS = true
}
}
}
true
} catch (e: IllegalStateException) {
appLogger.e("Could not convert wav to mp4: ${e.message}", throwable = e)
false
} catch (e: IOException) {
appLogger.e("Could not convert wav to mp4: ${e.message}", throwable = e)
false
} finally {
try {
fileInputStream?.close()
} catch (e: IOException) {
appLogger.e("Could not close FileInputStream: ${e.message}", throwable = e)
}

try {
muxer?.stop()
muxer?.release()
} catch (e: IllegalStateException) {
appLogger.e("Could not stop or release MediaMuxer: ${e.message}", throwable = e)
}

try {
codec?.stop()
codec?.release()
} catch (e: IllegalStateException) {
appLogger.e("Could not stop or release MediaCodec: ${e.message}", throwable = e)
}
}
}

companion object {
fun getRecordingAudioFileName(): String = "wire-audio-${DateTimeUtil.currentInstant().fileDateTime()}.wav"
fun getRecordingMP4AudioFileName(): String = "wire-audio-${DateTimeUtil.currentInstant().fileDateTime()}.mp4"
fun getRecordingAudioEffectsFileName(): String = "wire-audio-${DateTimeUtil.currentInstant().fileDateTime()}-filter.wav"

const val SIZE_OF_1MB = 1024 * 1024
const val AUDIO_CHANNELS = 1 // Mono
const val SAMPLING_RATE = 44100
const val SAMPLING_RATE = 16000
const val BUFFER_SIZE = 1024
const val AUDIO_ENCODING = AudioFormat.ENCODING_PCM_16BIT
const val BITS_PER_SAMPLE = 16
Expand All @@ -233,5 +361,9 @@ class AudioMediaRecorder @Inject constructor(
const val FORMAT_WAVE = "WAVE"
const val SUBCHUNK1_ID_FMT = "fmt "
const val SUBCHUNK2_ID_DATA = "data"

private const val BIT_RATE = 64000
private const val TIMEOUT_US: Long = 10000
const val NANOSECONDS_IN_MICROSECOND = 1000
}
}
2 changes: 1 addition & 1 deletion ...m/wire/android/ui/home/messagecomposer/recordaudio/GenerateAudioFileWithEffectsUseCase.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class GenerateAudioFileWithEffectsUseCase @Inject constructor(
suspend operator fun invoke(
context: Context,
originalFilePath: String,
effectsFilePath: String,
effectsFilePath: String
) = withContext(dispatchers.io()) {
appLogger.i("[$TAG] -> Start generating audio file with effects")

Expand Down
51 changes: 37 additions & 14 deletions .../main/kotlin/com/wire/android/ui/home/messagecomposer/recordaudio/RecordAudioViewModel.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,26 +267,49 @@ class RecordAudioViewModel @Inject constructor(
viewModelScope.launch {
recordAudioMessagePlayer.stop()
recordAudioMessagePlayer.close()
state = state.copy(
buttonState = RecordAudioButtonState.ENCODING, audioState = AudioState.DEFAULT,
originalOutputFile = null,
effectsOutputFile = null
)

val resultFile = if (state.shouldApplyEffects) {
try {
state.originalOutputFile?.toPath()?.deleteIfExists()
} catch (exception: IOException) {
appLogger.e("[$tag] -> Couldn't delete original audio file before sending audio file with effects.")
}
state.effectsOutputFile!!.toUri()
} else {
try {
state.effectsOutputFile?.toPath()?.deleteIfExists()
} catch (exception: IOException) {
appLogger.e("[$tag] -> Couldn't delete audio file with effects before sending original audio file.")
val didSucceed = audioMediaRecorder.convertWavToMp4(state.shouldApplyEffects)

try {
when {
didSucceed -> {
state.originalOutputFile?.toPath()?.deleteIfExists()
state.effectsOutputFile?.toPath()?.deleteIfExists()
}

state.shouldApplyEffects -> {
state.originalOutputFile?.toPath()?.deleteIfExists()
}

!state.shouldApplyEffects -> {
state.effectsOutputFile?.toPath()?.deleteIfExists()
}
}
state.originalOutputFile!!.toUri()
} catch (exception: IOException) {
appLogger.e("[$tag] -> Couldn't delete audio files")
}

onAudioRecorded(
UriAsset(
uri = resultFile,
uri = if (didSucceed) {
audioMediaRecorder.mp4OutputPath!!.toFile().toUri()
} else {
if (state.shouldApplyEffects) {
audioMediaRecorder.effectsOutputPath!!.toFile().toUri()
} else {
audioMediaRecorder.originalOutputPath!!.toFile().toUri()
}
},
mimeType = if (didSucceed) {
"audio/mp4"
} else {
"audio/wav"
},
saveToDeviceIfInvalid = false
)
)
Expand Down

0 comments on commit 8a32e3d

Please sign in to comment.