Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add initial crossplatform backup serialization [WPB-10575] #3121

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions backup/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -77,36 +77,42 @@ kotlin {
val androidMain by getting {
dependsOn(nonJsMain)
}
val androidInstrumentedTest by getting {
dependsOn(nonJsTest)
}
val jvmMain by getting {
dependsOn(nonJsMain)
}

val iosX64Main by getting {
val jvmTest by getting {
dependsOn(nonJsTest)
}
val appleMain by getting {
dependsOn(nonJsMain)
}
val appleTest by getting {
dependsOn(nonJsTest)
}
val iosX64Main by getting {
dependencies {
implementation(libs.pbandk.runtime.iosX64)
}
}
val iosArm64Main by getting {
dependsOn(nonJsMain)
dependencies {
implementation(libs.pbandk.runtime.iosArm64)
}
}
val iosSimulatorArm64Main by getting {
dependsOn(nonJsMain)
dependencies {
implementation(libs.pbandk.runtime.iosSimulatorArm64)
}
}
val macosX64Main by getting {
dependsOn(nonJsMain)
dependencies {
implementation(libs.pbandk.runtime.macX64)
}
}
val macosArm64Main by getting {
dependsOn(nonJsMain)
dependencies {
implementation(libs.pbandk.runtime.macArm64)
}
Expand Down
25 changes: 25 additions & 0 deletions backup/src/commonMain/kotlin/com/wire/backup/MPBackup.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Wire
* Copyright (C) 2024 Wire Swiss GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see http://www.gnu.org/licenses/.
*/
package com.wire.backup

import kotlin.js.JsExport

@JsExport
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JsExport is needed so this class can be called from JS later.

object MPBackup {
const val ZIP_ENTRY_DATA = "data.wmbu"
}
105 changes: 105 additions & 0 deletions backup/src/commonMain/kotlin/com/wire/backup/data/BackupData.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/*
* Wire
* Copyright (C) 2024 Wire Swiss GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see http://www.gnu.org/licenses/.
*/
@file:OptIn(ExperimentalObjCRefinement::class, ExperimentalObjCName::class)

package com.wire.backup.data

import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlin.experimental.ExperimentalObjCName
import kotlin.experimental.ExperimentalObjCRefinement
import kotlin.js.JsExport
import kotlin.native.ObjCName
import kotlin.native.ShouldRefineInSwift

@JsExport
class BackupData(
val metadata: BackupMetadata,
@ShouldRefineInSwift
val users: Array<BackupUser>,
@ShouldRefineInSwift
val conversations: Array<BackupConversation>,
@ShouldRefineInSwift
val messages: Array<BackupMessage>
) {
@ObjCName("users")
val userList: List<BackupUser> get() = users.toList()

@ObjCName("conversations")
val conversationList: List<BackupConversation> get() = conversations.toList()

@ObjCName("messages")
val messageList: List<BackupMessage> get() = messages.toList()
}

@JsExport
@Serializable
data class BackupQualifiedId(
@SerialName("id")
val id: String,
@SerialName("domain")
val domain: String,
) {
override fun toString() = "$id@$domain"

companion object {
private const val QUALIFIED_ID_COMPONENT_COUNT = 2

fun fromEncodedString(id: String): BackupQualifiedId? {
val components = id.split("@")
if (components.size != QUALIFIED_ID_COMPONENT_COUNT) return null
return BackupQualifiedId(components[0], components[1])
}
}
}

@JsExport
data class BackupUser(
val id: BackupQualifiedId,
val name: String,
val handle: String,
)

@JsExport
data class BackupConversation(
val id: BackupQualifiedId,
val name: String,
)

@JsExport
data class BackupMessage(
val id: String,
val conversationId: BackupQualifiedId,
val senderUserId: BackupQualifiedId,
val senderClientId: String,
val creationDate: BackupDateTime,
val content: BackupMessageContent
)

expect class BackupDateTime

expect fun BackupDateTime(timestampMillis: Long): BackupDateTime
expect fun BackupDateTime.toLongMilliseconds(): Long
Comment on lines +94 to +97
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is because JS doesn't have 64-bit Integers (Long).
So we can use Long for iOS/Android, and a custom Date wrapper for JS.


@JsExport
sealed class BackupMessageContent {
data class Text(val text: String) : BackupMessageContent()

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Text messages also contain link previews, mentions, and are possibly a reply to another message. Are we planning to include these in the first version?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can expand later, like what's done in:


// TODO: Not _yet_ implemented
data class Asset(val todo: String) : BackupMessageContent()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Wire
* Copyright (C) 2024 Wire Swiss GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see http://www.gnu.org/licenses/.
*/

package com.wire.backup.data

import kotlin.js.JsExport

@JsExport
data class BackupMetadata(
val version: String,
val userId: BackupQualifiedId,
val creationTime: BackupDateTime,
val clientId: String?
)
23 changes: 23 additions & 0 deletions backup/src/commonMain/kotlin/com/wire/backup/data/Mapping.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* Wire
* Copyright (C) 2024 Wire Swiss GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see http://www.gnu.org/licenses/.
*/
package com.wire.backup.data

import com.wire.kalium.protobuf.backup.ExportedQualifiedId

internal fun BackupQualifiedId.toProtoModel() = ExportedQualifiedId(id, domain)
internal fun ExportedQualifiedId.toModel() = BackupQualifiedId(value, domain)
113 changes: 113 additions & 0 deletions backup/src/commonMain/kotlin/com/wire/backup/dump/MPBackupExporter.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
/*
* Wire
* Copyright (C) 2024 Wire Swiss GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see http://www.gnu.org/licenses/.
*/
package com.wire.backup.dump

import com.wire.backup.data.BackupConversation
import com.wire.backup.data.BackupMessage
import com.wire.backup.data.BackupMessageContent
import com.wire.backup.data.BackupQualifiedId
import com.wire.backup.data.BackupUser
import com.wire.backup.data.toLongMilliseconds
import com.wire.backup.data.toProtoModel
import com.wire.kalium.protobuf.backup.BackupData
import com.wire.kalium.protobuf.backup.BackupInfo
import com.wire.kalium.protobuf.backup.ExportUser
import com.wire.kalium.protobuf.backup.ExportedConversation
import com.wire.kalium.protobuf.backup.ExportedMessage
import com.wire.kalium.protobuf.backup.ExportedText
import kotlinx.datetime.Clock
import pbandk.encodeToByteArray
import kotlin.experimental.ExperimentalObjCName
import kotlin.experimental.ExperimentalObjCRefinement
import kotlin.js.JsExport
import kotlin.native.ObjCName
import kotlin.native.ShouldRefineInSwift

/**
* Entity able to serialize [BackupData] entities, like [BackupMessage], [BackupConversation], [BackupUser]
* into a cross-platform [BackupData] format.
*/
@OptIn(ExperimentalObjCName::class, ExperimentalObjCRefinement::class)
@JsExport
abstract class CommonMPBackupExporter(
private val selfUserId: BackupQualifiedId
) {
private val allUsers = mutableListOf<BackupUser>()
private val allConversations = mutableListOf<BackupConversation>()
private val allMessages = mutableListOf<BackupMessage>()

// TODO: Replace `ObjCName` with `JsName` in the future and flip it around.
// Unfortunately the IDE doesn't understand this right now and
// keeps complaining if making the other way around
@ObjCName("add")
fun addUser(user: BackupUser) {
allUsers.add(user)
}

@ObjCName("add")
fun addConversation(conversation: BackupConversation) {
allConversations.add(conversation)
}

@ObjCName("add")
fun addMessage(message: BackupMessage) {
allMessages.add(message)
}
Comment on lines +57 to +70
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was planning on having add for all three methods, only changing the parameter.

Unfortunately it doesn't work on JS, as it is not a typed language...

So at least when calling from Swift it will look like this:

backupExporter.add(user: BackupUser(...))
backupExporter.add(conversation: BackupConversation(...))
backupExporter.add(message: BackupMesage(...))


@OptIn(ExperimentalStdlibApi::class)
@ShouldRefineInSwift // Hidden in Swift
fun serialize(): ByteArray {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this the method we call to produce the backup? The comment suggests it's hidden but it the POC it is visible but returns a KotlinByteArray which I'm guessing I would create a file with and present it to the user. Though I'm concerned about the size of this data and loading it all into memory at once.

I'm wondering if it would be possible for the client to pass a URL to which the library could write the backup file to. This could allow for more memory efficienct handling of the backup data.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed :)
The goal is to implement the byte-reading part in the commonMain code, which will be used internally for all clients.

JS will only have this bytearray alternative.

Android and iOS will be able to pass paths that will stream and take paginated chunks of data in, without having to load the whole file into memory

This PR only handles initial serialization. There are more PRs incoming with:

  • Expanded serialization
  • Encryption/Decryption
  • Creating APIs specific for dealing with files on iOS and Android
  • Solidifying/tweaking the API

val backupData = BackupData(
BackupInfo(
platform = "Common",
version = "1.0",
userId = selfUserId.toProtoModel(),
creationTime = Clock.System.now().toEpochMilliseconds(),
clientId = "lol"
),
allConversations.map { ExportedConversation(it.id.toProtoModel(), it.name) },
allMessages.map {
ExportedMessage(
id = it.id,
timeIso = it.creationDate.toLongMilliseconds(),
senderUserId = it.senderUserId.toProtoModel(),
senderClientId = it.senderClientId,
conversationId = it.conversationId.toProtoModel(),
content = when (val content = it.content) {
is BackupMessageContent.Asset ->
ExportedMessage.Content.Text(ExportedText("FAKE ASSET")) // TODO: Support assets
is BackupMessageContent.Text ->
ExportedMessage.Content.Text(ExportedText(content.text))
}
)
},
allUsers.map {
ExportUser(
id = it.id.toProtoModel(),
name = it.name,
handle = it.handle
)
},
)
return backupData.encodeToByteArray().also {
println("XPlatform Backup POC. Exported data bytes: ${it.toHexString()}")
}
}
}

expect class MPBackupExporter : CommonMPBackupExporter
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* Wire
* Copyright (C) 2024 Wire Swiss GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see http://www.gnu.org/licenses/.
*/
package com.wire.backup.ingest

import com.wire.backup.data.BackupData
import kotlin.js.JsExport

@JsExport
sealed class BackupImportResult {
data object ParsingFailure : BackupImportResult()
data class Success(val backupData: BackupData) : BackupImportResult()
}
Loading
Loading