Performance improvements and bug fixes (#869)

* refactor: fail on testSnode instead of recursively using up snode list. add call timeout on http client

* refactor: refactoring batch message receives and pollers

* refactor: reduce thread utils pool count to a 2 thread fixed pool. Do a check against pubkey instead of room names for oxenHostedOpenGroup

* refactor: caching lib with potential loader fixes and no-cache for giphy

* refactor: remove store and instead use ConcurrentHashMap with a backing update coroutine

* refactor: queue trim thread jobs instead of add every message processed

* fix: wrapping auth token and initial sync for open groups in a threadutils queued runnable, getting initial sync times down

* fix: fixing the user contacts cache in ConversationAdapter.kt

* refactor: improve polling and initial sync, move group joins from config messages into a background job fetching image.

* refactor: improving the job queuing for open groups, replacing placeholder avatar generation with a custom glide loader and archiving initial sync of open groups

* feat: add OpenGroupDeleteJob.kt

* feat: add open group delete job to process deletions after batch adding

* feat: add vacuum and fix job queue re-adding jobs forever, only try to set message hash values in DB if they have changed

* refactor: remove redundant inflation for profile image views throughout app

* refactor(wip): reducing layout inflation and starting to refactor the open group deletion issues taking a long time

* refactor(wip): refactoring group deletion to not iterate through and delete messages individually

* refactor(wip): refactoring group deletion to not iterate through and delete messages individually

* fix: group deletion optimisation

* build: bump build number

* build: bump build number and fix batch message receive retry logic

* fix: clear out open group deletes

* fix: update visible ConversationAdapter.kt binding for initial contact fetching and better traces for debugging background jobs

* fix: add in check for / force sync latest encryption key pair from linked devices if we already have that closed group

* Rename .java to .kt

* refactor: change MmsDatabase to kotlin to make list operations easier

* fix: nullable type

* fix: compilation issues and constants in .kt instead of .java

* fix: bug fix expiration timer on closed group recipient

* feat: use the job queue properly across executors

* feat: start on open group dispatcher-specific logic, probably a queue factory based on openGroupId if that is the same across new message and deletion jobs to ensure consistent entry and removal

* refactor: removing redundant code and fixing jobqueue per opengroup

* fix: allow attachments in note to self

* fix: make the minWidth in quote view bind max of text / title and body, wrapped ?

* fix: fixing up layouts and code view layouts

* fix: remove TODO, remove timestamp binding

* feat: fix view logic, avatars and padding, downloading attachments lazily (on bind), fixing potential crash, add WindowDebouncer.kt

* fix: NPE on viewModel recipient from removed thread while tearing down the Recipient observer in ConversationActivityV2.kt

* refactor: replace conversation notification debouncer handler with handlerthread, same as conversation list debouncer

* refactor: UI for groups and poller improvements

* fix: revert some changes in poller

* feat: add header back in for message requests

* refactor: remove Trace calls, add more conditions to the HomeDiffUtil for updating more efficiently

* feat: try update the home adapter if we get a profile picture modified event

* feat: bump build numbers

* fix: try to start with list in homeViewModel if we don't have already, render quotes to be width of attachment slide view instead of fixed

* fix: set channel to be conflated instead of no buffer

* fix: set unreads based off last local user message vs incrementing unreads to be all amount

* feat: add profile update flag, update build number

* fix: link preview thumbnails download on bind

* fix: centercrop placeholder in glide request

* feat: recycle the contact selection list and profile image in unbind

* fix: try to prevent user KP crash at weird times

* fix: remove additional log, improve attachment download success rate, fix share logs dialog issue

app/build.gradle

@@ -110,6 +110,7 @@ dependencies {
     implementation "com.google.protobuf:protobuf-java:$protobufVersion"
     implementation "com.fasterxml.jackson.core:jackson-databind:$jacksonDatabindVersion"
     implementation "com.squareup.okhttp3:okhttp:$okhttpVersion"
+    implementation 'app.cash.copper:copper-flow:1.0.0'
     implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$coroutinesVersion"
     implementation "nl.komponents.kovenant:kovenant:$kovenantVersion"
@@ -158,8 +159,8 @@ dependencies {
     testImplementation 'org.robolectric:shadows-multidex:4.4'
 }
 
-def canonicalVersionCode = 279
-def canonicalVersionName = "1.13.1"
+def canonicalVersionCode = 282
+def canonicalVersionName = "1.13.4"
 
 def postFixSize = 10
 def abiPostFix = ['armeabi-v7a' : 1,

app/src/main/java/org/thoughtcrime/securesms/ApplicationContext.java

@@ -24,7 +24,7 @@
 import android.os.AsyncTask;
 import android.os.Build;
 import android.os.Handler;
-import android.os.Looper;
+import android.os.HandlerThread;
 
 import androidx.annotation.NonNull;
 import androidx.lifecycle.DefaultLifecycleObserver;
@@ -44,6 +44,7 @@
 import org.session.libsession.utilities.SSKEnvironment;
 import org.session.libsession.utilities.TextSecurePreferences;
 import org.session.libsession.utilities.Util;
+import org.session.libsession.utilities.WindowDebouncer;
 import org.session.libsession.utilities.dynamiclanguage.DynamicLanguageContextWrapper;
 import org.session.libsession.utilities.dynamiclanguage.LocaleParser;
 import org.session.libsignal.utilities.Log;
@@ -93,6 +94,7 @@
 import java.util.Date;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.Timer;
 
 import javax.inject.Inject;
 
@@ -127,7 +129,9 @@ public class ApplicationContext extends Application implements DefaultLifecycleO
     public Poller poller = null;
     public Broadcaster broadcaster = null;
     private Job firebaseInstanceIdJob;
-    private Handler conversationListNotificationHandler;
+    private WindowDebouncer conversationListDebouncer;
+    private HandlerThread conversationListHandlerThread;
+    private Handler conversationListHandler;
     private PersistentLogger persistentLogger;
 
     @Inject LokiAPIDatabase lokiAPIDatabase;
@@ -136,9 +140,18 @@ public class ApplicationContext extends Application implements DefaultLifecycleO
     @Inject JobDatabase jobDatabase;
     @Inject TextSecurePreferences textSecurePreferences;
     CallMessageProcessor callMessageProcessor;
+    MessagingModuleConfiguration messagingModuleConfiguration;
 
     private volatile boolean isAppVisible;
 
+    @Override
+    public Object getSystemService(String name) {
+        if (MessagingModuleConfiguration.MESSAGING_MODULE_SERVICE.equals(name)) {
+            return messagingModuleConfiguration;
+        }
+        return super.getSystemService(name);
+    }
+
     public static ApplicationContext getInstance(Context context) {
         return (ApplicationContext) context.getApplicationContext();
     }
@@ -148,10 +161,21 @@ public DatabaseComponent getDatabaseComponent() {
     }
 
     public Handler getConversationListNotificationHandler() {
-        if (this.conversationListNotificationHandler == null) {
-            conversationListNotificationHandler = new Handler(Looper.getMainLooper());
+        if (this.conversationListHandlerThread == null) {
+            conversationListHandlerThread = new HandlerThread("ConversationListHandler");
+            conversationListHandlerThread.start();
+        }
+        if (this.conversationListHandler == null) {
+            conversationListHandler = new Handler(conversationListHandlerThread.getLooper());
+        }
+        return conversationListHandler;
+    }
+
+    public WindowDebouncer getConversationListDebouncer() {
+        if (conversationListDebouncer == null) {
+            conversationListDebouncer = new WindowDebouncer(1000, new Timer());
         }
-        return this.conversationListNotificationHandler;
+        return conversationListDebouncer;
     }
 
     public PersistentLogger getPersistentLogger() {
@@ -161,7 +185,12 @@ public PersistentLogger getPersistentLogger() {
     @Override
     public void onCreate() {
         DatabaseModule.init(this);
+        MessagingModuleConfiguration.configure(this);
         super.onCreate();
+        messagingModuleConfiguration = new MessagingModuleConfiguration(this,
+                storage,
+                messageDataProvider,
+                ()-> KeyPairUtilities.INSTANCE.getUserED25519KeyPair(this));
         callMessageProcessor = new CallMessageProcessor(this, textSecurePreferences, ProcessLifecycleOwner.get().getLifecycle(), storage);
         Log.i(TAG, "onCreate()");
         startKovenant();
@@ -174,11 +203,6 @@ public void onCreate() {
         messageNotifier = new OptimizedMessageNotifier(new DefaultMessageNotifier());
         broadcaster = new Broadcaster(this);
         LokiAPIDatabase apiDB = getDatabaseComponent().lokiAPIDatabase();
-        MessagingModuleConfiguration.Companion.configure(this,
-                storage,
-                messageDataProvider,
-                ()-> KeyPairUtilities.INSTANCE.getUserED25519KeyPair(this)
-        );
         SnodeModule.Companion.configure(apiDB, broadcaster);
         String userPublicKey = TextSecurePreferences.getLocalNumber(this);
         if (userPublicKey != null) {

app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachmentProvider.kt

@@ -5,7 +5,14 @@ import android.text.TextUtils
 import com.google.protobuf.ByteString
 import org.greenrobot.eventbus.EventBus
 import org.session.libsession.database.MessageDataProvider
-import org.session.libsession.messaging.sending_receiving.attachments.*
+import org.session.libsession.messaging.sending_receiving.attachments.Attachment
+import org.session.libsession.messaging.sending_receiving.attachments.AttachmentId
+import org.session.libsession.messaging.sending_receiving.attachments.AttachmentState
+import org.session.libsession.messaging.sending_receiving.attachments.DatabaseAttachment
+import org.session.libsession.messaging.sending_receiving.attachments.DatabaseAttachmentAudioExtras
+import org.session.libsession.messaging.sending_receiving.attachments.PointerAttachment
+import org.session.libsession.messaging.sending_receiving.attachments.SessionServiceAttachmentPointer
+import org.session.libsession.messaging.sending_receiving.attachments.SessionServiceAttachmentStream
 import org.session.libsession.utilities.Address
 import org.session.libsession.utilities.UploadResult
 import org.session.libsession.utilities.Util
@@ -126,7 +133,7 @@ class DatabaseAttachmentProvider(context: Context, helper: SQLCipherOpenHelper)
         val mmsDb = DatabaseComponent.get(context).mmsDatabase()
         return mmsDb.getMessage(mmsMessageId).use { cursor ->
             mmsDb.readerFor(cursor).next
-        }.isOutgoing
+        }?.isOutgoing ?: false
     }
 
     override fun isOutgoingMessage(timestamp: Long): Boolean {

app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupExporter.kt

@@ -10,29 +10,51 @@ import com.annimon.stream.function.Predicate
 import com.google.protobuf.ByteString
 import net.sqlcipher.database.SQLiteDatabase
 import org.greenrobot.eventbus.EventBus
-
-import org.session.libsession.messaging.sending_receiving.attachments.AttachmentId
 import org.session.libsession.avatars.AvatarHelper
+import org.session.libsession.messaging.sending_receiving.attachments.AttachmentId
 import org.session.libsession.utilities.Conversions
-
-import org.thoughtcrime.securesms.backup.BackupProtos.*
+import org.session.libsession.utilities.Util
+import org.session.libsignal.crypto.kdf.HKDFv3
+import org.session.libsignal.utilities.ByteUtil
+import org.session.libsignal.utilities.Log
+import org.thoughtcrime.securesms.backup.BackupProtos.Attachment
+import org.thoughtcrime.securesms.backup.BackupProtos.Avatar
+import org.thoughtcrime.securesms.backup.BackupProtos.BackupFrame
+import org.thoughtcrime.securesms.backup.BackupProtos.DatabaseVersion
+import org.thoughtcrime.securesms.backup.BackupProtos.Header
+import org.thoughtcrime.securesms.backup.BackupProtos.SharedPreference
+import org.thoughtcrime.securesms.backup.BackupProtos.SqlStatement
+import org.thoughtcrime.securesms.backup.BackupProtos.Sticker
 import org.thoughtcrime.securesms.crypto.AttachmentSecret
 import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream
 import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream
-import org.thoughtcrime.securesms.database.*
-import org.session.libsignal.utilities.Log
+import org.thoughtcrime.securesms.database.AttachmentDatabase
+import org.thoughtcrime.securesms.database.GroupReceiptDatabase
+import org.thoughtcrime.securesms.database.JobDatabase
+import org.thoughtcrime.securesms.database.LokiAPIDatabase
 import org.thoughtcrime.securesms.database.LokiBackupFilesDatabase
+import org.thoughtcrime.securesms.database.MmsDatabase
+import org.thoughtcrime.securesms.database.MmsSmsColumns
+import org.thoughtcrime.securesms.database.PushDatabase
+import org.thoughtcrime.securesms.database.SearchDatabase
+import org.thoughtcrime.securesms.database.SmsDatabase
 import org.thoughtcrime.securesms.util.BackupUtil
-import org.session.libsession.utilities.Util
-import org.session.libsignal.crypto.kdf.HKDFv3
-import org.session.libsignal.utilities.ByteUtil
-import java.io.*
-import java.lang.Exception
+import java.io.Closeable
+import java.io.File
+import java.io.FileInputStream
+import java.io.Flushable
+import java.io.IOException
+import java.io.InputStream
+import java.io.OutputStream
 import java.security.InvalidAlgorithmParameterException
 import java.security.InvalidKeyException
 import java.security.NoSuchAlgorithmException
-import java.util.*
-import javax.crypto.*
+import java.util.LinkedList
+import javax.crypto.BadPaddingException
+import javax.crypto.Cipher
+import javax.crypto.IllegalBlockSizeException
+import javax.crypto.Mac
+import javax.crypto.NoSuchPaddingException
 import javax.crypto.spec.IvParameterSpec
 import javax.crypto.spec.SecretKeySpec
 
@@ -245,8 +267,8 @@ object FullBackupExporter {
     }
 
     private fun isForNonExpiringMessage(db: SQLiteDatabase, mmsId: Long): Boolean {
-        val columns = arrayOf(MmsDatabase.EXPIRES_IN)
-        val where = MmsDatabase.ID + " = ?"
+        val columns = arrayOf(MmsSmsColumns.EXPIRES_IN)
+        val where = MmsSmsColumns.ID + " = ?"
         val args = arrayOf(mmsId.toString())
         db.query(MmsDatabase.TABLE_NAME, columns, where, args, null, null, null).use { mmsCursor ->
             if (mmsCursor != null && mmsCursor.moveToFirst()) {

app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupImporter.kt

@@ -15,19 +15,39 @@ import org.session.libsession.utilities.Util
 import org.session.libsignal.crypto.kdf.HKDFv3
 import org.session.libsignal.utilities.ByteUtil
 import org.session.libsignal.utilities.Log
-import org.thoughtcrime.securesms.backup.BackupProtos.*
+import org.thoughtcrime.securesms.backup.BackupProtos.Attachment
+import org.thoughtcrime.securesms.backup.BackupProtos.Avatar
+import org.thoughtcrime.securesms.backup.BackupProtos.BackupFrame
+import org.thoughtcrime.securesms.backup.BackupProtos.DatabaseVersion
+import org.thoughtcrime.securesms.backup.BackupProtos.SharedPreference
+import org.thoughtcrime.securesms.backup.BackupProtos.SqlStatement
 import org.thoughtcrime.securesms.crypto.AttachmentSecret
 import org.thoughtcrime.securesms.crypto.ModernEncryptingPartOutputStream
-import org.thoughtcrime.securesms.database.*
+import org.thoughtcrime.securesms.database.AttachmentDatabase
+import org.thoughtcrime.securesms.database.GroupReceiptDatabase
+import org.thoughtcrime.securesms.database.MmsDatabase
+import org.thoughtcrime.securesms.database.MmsSmsColumns
+import org.thoughtcrime.securesms.database.SearchDatabase
+import org.thoughtcrime.securesms.database.ThreadDatabase
 import org.thoughtcrime.securesms.dependencies.DatabaseComponent
 import org.thoughtcrime.securesms.util.BackupUtil
-import java.io.*
+import java.io.Closeable
+import java.io.File
+import java.io.FileOutputStream
+import java.io.IOException
+import java.io.InputStream
+import java.io.OutputStream
 import java.security.InvalidAlgorithmParameterException
 import java.security.InvalidKeyException
 import java.security.MessageDigest
 import java.security.NoSuchAlgorithmException
-import java.util.*
-import javax.crypto.*
+import java.util.LinkedList
+import java.util.Locale
+import javax.crypto.BadPaddingException
+import javax.crypto.Cipher
+import javax.crypto.IllegalBlockSizeException
+import javax.crypto.Mac
+import javax.crypto.NoSuchPaddingException
 import javax.crypto.spec.IvParameterSpec
 import javax.crypto.spec.SecretKeySpec
 
@@ -172,7 +192,7 @@ object FullBackupImporter {
     }
 
     private fun trimEntriesForExpiredMessages(context: Context, db: SQLiteDatabase) {
-        val trimmedCondition = " NOT IN (SELECT ${MmsDatabase.ID} FROM ${MmsDatabase.TABLE_NAME})"
+        val trimmedCondition = " NOT IN (SELECT ${MmsSmsColumns.ID} FROM ${MmsDatabase.TABLE_NAME})"
         db.delete(GroupReceiptDatabase.TABLE_NAME, GroupReceiptDatabase.MMS_ID + trimmedCondition, null)
         val columns = arrayOf(AttachmentDatabase.ROW_ID, AttachmentDatabase.UNIQUE_ID)
         val where = AttachmentDatabase.MMS_ID + trimmedCondition

app/src/main/java/org/thoughtcrime/securesms/components/AvatarImageView.java

@@ -8,28 +8,26 @@
 import android.graphics.Paint;
 import android.graphics.drawable.Drawable;
 import android.provider.ContactsContract;
-import androidx.annotation.NonNull;
-import androidx.annotation.Nullable;
-import androidx.appcompat.widget.AppCompatImageView;
 import android.util.AttributeSet;
 import android.view.View;
 import android.view.ViewOutlineProvider;
 
-import com.bumptech.glide.load.engine.DiskCacheStrategy;
-
-
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import androidx.appcompat.widget.AppCompatImageView;
 
-import org.thoughtcrime.securesms.util.AvatarPlaceholderGenerator;
-import org.thoughtcrime.securesms.mms.GlideApp;
-import org.thoughtcrime.securesms.mms.GlideRequests;
+import com.bumptech.glide.load.engine.DiskCacheStrategy;
 
 import org.session.libsession.avatars.ContactColors;
 import org.session.libsession.avatars.ContactPhoto;
 import org.session.libsession.avatars.ResourceContactPhoto;
 import org.session.libsession.utilities.Address;
+import org.session.libsession.utilities.ThemeUtil;
 import org.session.libsession.utilities.recipients.Recipient;
 import org.session.libsession.utilities.recipients.RecipientExporter;
-import org.session.libsession.utilities.ThemeUtil;
+import org.thoughtcrime.securesms.mms.GlideApp;
+import org.thoughtcrime.securesms.mms.GlideRequests;
+import org.thoughtcrime.securesms.util.AvatarPlaceholderGenerator;
 
 import java.util.Objects;
 
@@ -139,7 +137,7 @@ public void setAvatar(@NonNull GlideRequests requestManager, @Nullable Recipient
             requestManager.load(photo.contactPhoto)
                           .fallback(photoPlaceholderDrawable)
                           .error(photoPlaceholderDrawable)
-                          .diskCacheStrategy(DiskCacheStrategy.ALL)
+                          .diskCacheStrategy(DiskCacheStrategy.NONE)
                           .circleCrop()
                           .into(this);
           } else {