From 36981da06e1d005a3d7d6f377a086668bd2d1b02 Mon Sep 17 00:00:00 2001 From: kl3jvi Date: Sat, 23 Nov 2024 23:48:10 +0100 Subject: [PATCH 1/3] fix: backup and restore progress updates in a more meaningful way. --- .../logic/data/asset/KaliumFileSystemImpl.kt | 12 ++++ .../logic/data/asset/KaliumFileSystemImpl.kt | 3 + .../com/wire/kalium/logic/util/BackupUtils.kt | 27 +++++++- .../kalium/logic/util/ProgressTrackingSink.kt | 51 +++++++++++++++ .../logic/data/asset/KaliumFileSystemImpl.kt | 7 +++ .../feature/backup/CreateBackupUseCase.kt | 63 ++++++++++--------- .../com/wire/kalium/logic/util/BackupUtils.kt | 8 ++- .../logic/data/asset/KaliumFileSystemImpl.kt | 12 ++++ 8 files changed, 151 insertions(+), 32 deletions(-) create mode 100644 logic/src/commonJvmAndroid/kotlin/com/wire/kalium/logic/util/ProgressTrackingSink.kt diff --git a/logic/src/androidMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt b/logic/src/androidMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt index c7b1cc1f29b..5392b47ed22 100644 --- a/logic/src/androidMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt +++ b/logic/src/androidMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt @@ -27,6 +27,7 @@ import okio.Sink import okio.Source import okio.buffer import okio.use +import java.io.File @Suppress("TooManyFunctions") actual class KaliumFileSystemImpl actual constructor( @@ -154,4 +155,15 @@ actual class KaliumFileSystemImpl actual constructor( * @return the list of paths found. */ override suspend fun listDirectories(dir: Path): List = SYSTEM.list(dir) + + /** + * Returns the size of the file at the specified path. + * @param path The path to the file whose size is being determined. + * @return The size of the file in bytes. + */ + override fun fileSize(path: Path): Long { +// val file = File(path.toString()) +// return if (file.exists() && file.isFile) file.length() else 0 + return SYSTEM.metadata(path).size ?: 0L + } } diff --git a/logic/src/appleMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt b/logic/src/appleMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt index fe00b14df91..524fd341806 100644 --- a/logic/src/appleMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt +++ b/logic/src/appleMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt @@ -151,4 +151,7 @@ actual class KaliumFileSystemImpl actual constructor( * @return the list of paths found. */ override suspend fun listDirectories(dir: Path): List = SYSTEM.list(dir) + override fun fileSize(path: Path): Long { + return SYSTEM.metadata(path).size ?: 0L + } } diff --git a/logic/src/commonJvmAndroid/kotlin/com/wire/kalium/logic/util/BackupUtils.kt b/logic/src/commonJvmAndroid/kotlin/com/wire/kalium/logic/util/BackupUtils.kt index cd800a71cbd..cdfd6821a40 100644 --- a/logic/src/commonJvmAndroid/kotlin/com/wire/kalium/logic/util/BackupUtils.kt +++ b/logic/src/commonJvmAndroid/kotlin/com/wire/kalium/logic/util/BackupUtils.kt @@ -18,6 +18,7 @@ package com.wire.kalium.logic.util +import co.touchlab.kermit.Logger import com.wire.kalium.logic.CoreFailure import com.wire.kalium.logic.StorageFailure import com.wire.kalium.logic.data.asset.KaliumFileSystem @@ -38,13 +39,35 @@ import java.util.zip.ZipInputStream import java.util.zip.ZipOutputStream @Suppress("TooGenericExceptionCaught") -actual fun createCompressedFile(files: List>, outputSink: Sink): Either = try { +actual fun createCompressedFile( + files: List>, + outputSink: Sink, + totalBytes: Long, + onProgress: (Float) -> Unit +): Either = try { var compressedFileSize = 0L - ZipOutputStream(outputSink.buffer().outputStream()).use { zipOutputStream -> + // The progress calculation is done by weighing both the uncompressed progress and the compressed progress. The + // final progress is a weighted average of these two values, where the uncompressed progress is given less weight + // (10%) and the compressed progress is given more weight (90%). This reflects the fact that compression is a process + // that involves both reading the input data and writing the compressed output, with more focus on the output size being + // written to the zip file. + val progressSink = ProgressTrackingSink(outputSink, totalBytes) { bytesWritten, total -> + val uncompressedProgress = bytesWritten.toFloat() / total.toFloat() + val compressedProgress = compressedFileSize.toFloat() / total.toFloat() + val progress = (uncompressedProgress * 0.1f) + (compressedProgress * 0.9f) + onProgress(progress.coerceAtMost(1.0f)) + Logger.d { "Progress: $progress" } + Logger.e("Bytes Written: $bytesWritten, Uncompressed Progress: $uncompressedProgress, Compressed Progress: $compressedProgress\"") + }.buffer() + + ZipOutputStream(progressSink.outputStream()).use { zipOutputStream -> files.forEach { (fileSource, fileName) -> compressedFileSize += addToCompressedFile(zipOutputStream, fileSource, fileName) + zipOutputStream.flush() } } + // Ensure that the progress is at 100% when the file is fully compressed + onProgress(1.0f) Either.Right(compressedFileSize) } catch (e: Exception) { Either.Left(StorageFailure.Generic(RuntimeException("There was an error trying to compress the provided files", e))) diff --git a/logic/src/commonJvmAndroid/kotlin/com/wire/kalium/logic/util/ProgressTrackingSink.kt b/logic/src/commonJvmAndroid/kotlin/com/wire/kalium/logic/util/ProgressTrackingSink.kt new file mode 100644 index 00000000000..b6be64278a0 --- /dev/null +++ b/logic/src/commonJvmAndroid/kotlin/com/wire/kalium/logic/util/ProgressTrackingSink.kt @@ -0,0 +1,51 @@ +/* + * Wire + * Copyright (C) 2024 Wire Swiss GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + */ +package com.wire.kalium.logic.util + +import co.touchlab.kermit.Logger +import okio.ForwardingSink +import okio.Sink + +/** + * A sink that tracks the progress of writing data to a delegate sink. + * + * @param delegate The sink to which data is written. + * @param totalBytes The total number of bytes that will be written. + * @param onProgress A callback that is invoked with the number of bytes written and the total number of bytes. + */ +class ProgressTrackingSink( + delegate: Sink, + private val totalBytes: Long, + private val onProgress: (bytesWritten: Long, totalBytes: Long) -> Unit +) : ForwardingSink(delegate) { + private var bytesWritten: Long = 0 + + override fun write(source: okio.Buffer, byteCount: Long) { + super.write(source, byteCount) + bytesWritten += byteCount + onProgress(bytesWritten, totalBytes) + } + + override fun close() { + delegate.close() + } + + override fun flush() { + delegate.flush() + } +} diff --git a/logic/src/commonMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt b/logic/src/commonMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt index 2671f0f5a48..134860f3003 100644 --- a/logic/src/commonMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt +++ b/logic/src/commonMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt @@ -129,4 +129,11 @@ interface KaliumFileSystem { * @return the list of paths found. */ suspend fun listDirectories(dir: Path): List + + /** + * Returns the size of the file at the specified path. + * @param path The path to the file whose size is being determined. + * @return The size of the file in bytes. + */ + fun fileSize(path: Path): Long } diff --git a/logic/src/commonMain/kotlin/com/wire/kalium/logic/feature/backup/CreateBackupUseCase.kt b/logic/src/commonMain/kotlin/com/wire/kalium/logic/feature/backup/CreateBackupUseCase.kt index c32f1e3fcf7..a6be056c5eb 100644 --- a/logic/src/commonMain/kotlin/com/wire/kalium/logic/feature/backup/CreateBackupUseCase.kt +++ b/logic/src/commonMain/kotlin/com/wire/kalium/logic/feature/backup/CreateBackupUseCase.kt @@ -26,11 +26,11 @@ import com.wire.kalium.logic.CoreFailure import com.wire.kalium.logic.StorageFailure import com.wire.kalium.logic.clientPlatform import com.wire.kalium.logic.data.asset.KaliumFileSystem +import com.wire.kalium.logic.data.id.CurrentClientIdProvider import com.wire.kalium.logic.data.id.IdMapper import com.wire.kalium.logic.data.user.UserId import com.wire.kalium.logic.data.user.UserRepository import com.wire.kalium.logic.di.MapperProvider -import com.wire.kalium.logic.data.id.CurrentClientIdProvider import com.wire.kalium.logic.feature.backup.BackupConstants.BACKUP_ENCRYPTED_FILE_NAME import com.wire.kalium.logic.feature.backup.BackupConstants.BACKUP_METADATA_FILE_NAME import com.wire.kalium.logic.feature.backup.BackupConstants.BACKUP_USER_DB_NAME @@ -63,7 +63,7 @@ interface CreateBackupUseCase { * with the provided password if it is not empty. Otherwise, the file will be unencrypted. * @param password The password to encrypt the backup file with. If empty, the file will be unencrypted. */ - suspend operator fun invoke(password: String): CreateBackupResult + suspend operator fun invoke(password: String, onProgress: (Float) -> Unit): CreateBackupResult } @Suppress("LongParameterList") @@ -78,32 +78,33 @@ internal class CreateBackupUseCaseImpl( private val idMapper: IdMapper = MapperProvider.idMapper(), ) : CreateBackupUseCase { - override suspend operator fun invoke(password: String): CreateBackupResult = withContext(dispatchers.default) { - val userHandle = userRepository.getSelfUser()?.handle?.replace(".", "-") - val timeStamp = DateTimeUtil.currentSimpleDateTimeString() - val backupName = createBackupFileName(userHandle, timeStamp) - val backupFilePath = kaliumFileSystem.tempFilePath(backupName) - deletePreviousBackupFiles(backupFilePath) - - val plainDBPath = - databaseExporter.exportToPlainDB(securityHelper.userDBOrSecretNull(userId))?.toPath() - ?: return@withContext CreateBackupResult.Failure(StorageFailure.DataNotFound) - - try { - createBackupFile(userId, plainDBPath, backupFilePath).fold( - { error -> CreateBackupResult.Failure(error) }, - { (backupFilePath, backupSize) -> - val isBackupEncrypted = password.isNotEmpty() - if (isBackupEncrypted) { - encryptAndCompressFile(backupFilePath, password) - } else CreateBackupResult.Success(backupFilePath, backupSize, backupFilePath.name) - }) - } finally { - databaseExporter.deleteBackupDBFile() + override suspend operator fun invoke(password: String, onProgress: (Float) -> Unit): CreateBackupResult = + withContext(dispatchers.default) { + val userHandle = userRepository.getSelfUser()?.handle?.replace(".", "-") + val timeStamp = DateTimeUtil.currentSimpleDateTimeString() + val backupName = createBackupFileName(userHandle, timeStamp) + val backupFilePath = kaliumFileSystem.tempFilePath(backupName) + deletePreviousBackupFiles(backupFilePath) + + val plainDBPath = + databaseExporter.exportToPlainDB(securityHelper.userDBOrSecretNull(userId))?.toPath() + ?: return@withContext CreateBackupResult.Failure(StorageFailure.DataNotFound) + + try { + createBackupFile(userId, plainDBPath, backupFilePath, onProgress).fold( + { error -> CreateBackupResult.Failure(error) }, + { (backupFilePath, backupSize) -> + val isBackupEncrypted = password.isNotEmpty() + if (isBackupEncrypted) { + encryptAndCompressFile(backupFilePath, password, onProgress) + } else CreateBackupResult.Success(backupFilePath, backupSize, backupFilePath.name) + }) + } finally { + databaseExporter.deleteBackupDBFile() + } } - } - private suspend fun encryptAndCompressFile(backupFilePath: Path, password: String): CreateBackupResult { + private suspend fun encryptAndCompressFile(backupFilePath: Path, password: String, onProgress: (Float) -> Unit): CreateBackupResult { val encryptedBackupFilePath = kaliumFileSystem.tempFilePath(BACKUP_ENCRYPTED_FILE_NAME) val backupEncryptedDataSize = encryptBackup( kaliumFileSystem.source(backupFilePath), @@ -117,7 +118,9 @@ internal class CreateBackupUseCaseImpl( return createCompressedFile( listOf(kaliumFileSystem.source(encryptedBackupFilePath) to encryptedBackupFilePath.name), - kaliumFileSystem.sink(finalBackupFilePath) + kaliumFileSystem.sink(finalBackupFilePath), + kaliumFileSystem.fileSize(encryptedBackupFilePath), + onProgress ).fold({ CreateBackupResult.Failure(StorageFailure.Generic(RuntimeException("Failed to compress encrypted backup file"))) }, { backupEncryptedCompressedDataSize -> @@ -167,7 +170,8 @@ internal class CreateBackupUseCaseImpl( private suspend fun createBackupFile( userId: UserId, plainDBPath: Path, - backupZipFilePath: Path + backupZipFilePath: Path, + onProgress: (Float) -> Unit ): Either> { return try { val backupSink = kaliumFileSystem.sink(backupZipFilePath) @@ -176,8 +180,9 @@ internal class CreateBackupUseCaseImpl( kaliumFileSystem.source(backupMetadataPath) to BACKUP_METADATA_FILE_NAME, kaliumFileSystem.source(plainDBPath) to BACKUP_USER_DB_NAME ) + val totalBytes = listOf(backupZipFilePath, plainDBPath).sumOf { kaliumFileSystem.fileSize(it) } - createCompressedFile(filesList, backupSink).flatMap { compressedFileSize -> + createCompressedFile(filesList, backupSink, totalBytes, onProgress).flatMap { compressedFileSize -> Either.Right(backupZipFilePath to compressedFileSize) } } catch (e: FileNotFoundException) { diff --git a/logic/src/commonMain/kotlin/com/wire/kalium/logic/util/BackupUtils.kt b/logic/src/commonMain/kotlin/com/wire/kalium/logic/util/BackupUtils.kt index 1635264ae95..eecf881cb37 100644 --- a/logic/src/commonMain/kotlin/com/wire/kalium/logic/util/BackupUtils.kt +++ b/logic/src/commonMain/kotlin/com/wire/kalium/logic/util/BackupUtils.kt @@ -26,7 +26,13 @@ import okio.Path import okio.Sink import okio.Source -expect fun createCompressedFile(files: List>, outputSink: Sink): Either +expect fun createCompressedFile( + files: List>, + outputSink: Sink, + totalBytes: Long, + onProgress: (Float) -> Unit +): Either + expect fun extractCompressedFile( inputSource: Source, outputRootPath: Path, diff --git a/logic/src/jvmMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt b/logic/src/jvmMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt index b5dfb1508e0..e2f4213d009 100644 --- a/logic/src/jvmMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt +++ b/logic/src/jvmMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt @@ -27,6 +27,7 @@ import okio.Sink import okio.Source import okio.buffer import okio.use +import java.io.File @Suppress("TooManyFunctions") actual class KaliumFileSystemImpl actual constructor( @@ -151,4 +152,15 @@ actual class KaliumFileSystemImpl actual constructor( * @return the list of paths found. */ override suspend fun listDirectories(dir: Path): List = SYSTEM.list(dir) + + /** + * Returns the size of the file at the specified path. + * @param path The path to the file whose size is being determined. + * @return The size of the file in bytes. + */ + override fun fileSize(path: Path): Long { +// val file = File(path.toString()) +// return if (file.exists() && file.isFile) file.length() else 0 + return SYSTEM.metadata(path).size ?: 0L + } } From 7772ac4544612d5ee29f1fc93a9076c6dc589456 Mon Sep 17 00:00:00 2001 From: Klejvi Kapaj <40796367+kl3jvi@users.noreply.github.com> Date: Wed, 27 Nov 2024 23:43:55 +0000 Subject: [PATCH 2/3] small fix --- .../wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/logic/src/androidMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt b/logic/src/androidMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt index 5392b47ed22..3ef65567339 100644 --- a/logic/src/androidMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt +++ b/logic/src/androidMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt @@ -161,9 +161,5 @@ actual class KaliumFileSystemImpl actual constructor( * @param path The path to the file whose size is being determined. * @return The size of the file in bytes. */ - override fun fileSize(path: Path): Long { -// val file = File(path.toString()) -// return if (file.exists() && file.isFile) file.length() else 0 - return SYSTEM.metadata(path).size ?: 0L - } + override fun fileSize(path: Path): Long = SYSTEM.metadata(path).size ?: 0L } From 38420f60eda14b11e5be4fa976ebcc767cad08a0 Mon Sep 17 00:00:00 2001 From: Klejvi Kapaj <40796367+kl3jvi@users.noreply.github.com> Date: Wed, 27 Nov 2024 23:46:10 +0000 Subject: [PATCH 3/3] KaliumFileSystemImpl.kt small fix --- .../wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/logic/src/jvmMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt b/logic/src/jvmMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt index e2f4213d009..21db3c5720e 100644 --- a/logic/src/jvmMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt +++ b/logic/src/jvmMain/kotlin/com/wire/kalium/logic/data/asset/KaliumFileSystemImpl.kt @@ -158,9 +158,6 @@ actual class KaliumFileSystemImpl actual constructor( * @param path The path to the file whose size is being determined. * @return The size of the file in bytes. */ - override fun fileSize(path: Path): Long { -// val file = File(path.toString()) -// return if (file.exists() && file.isFile) file.length() else 0 - return SYSTEM.metadata(path).size ?: 0L - } + override fun fileSize(path: Path): Long = SYSTEM.metadata(path).size ?: 0L } +