Skip to content

Commit

Permalink
sync: less compression is more (fixes #5082) (#5083)
Browse files Browse the repository at this point in the history 
Co-authored-by: dogi <[email protected]>
  • Loading branch information
@Okuro3499 @dogi
Okuro3499 and dogi authored Jan 21, 2025
1 parent 4281258 commit fd4376e
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 172 deletions.
4 changes: 2 additions & 2 deletions app/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ android {
applicationId "org.ole.planet.myplanet"
minSdkVersion 26
targetSdkVersion 34
versionCode 2224
versionName "0.22.24"
versionCode 2225
versionName "0.22.25"
ndkVersion '21.3.6528147'
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
vectorDrawables.useSupportLibrary = true
Expand Down
69 changes: 7 additions & 62 deletions app/src/main/java/org/ole/planet/myplanet/datamanager/ApiClient.kt
View file
Original file line number Diff line number Diff line change
@@ -1,85 +1,30 @@
package org.ole.planet.myplanet.datamanager

import com.google.gson.GsonBuilder
import okhttp3.Interceptor
import okhttp3.OkHttpClient
import okhttp3.Response
import retrofit2.Retrofit
import retrofit2.converter.gson.GsonConverterFactory
import java.io.IOException
import java.lang.reflect.Modifier
import java.util.concurrent.TimeUnit
import kotlin.math.pow

object ApiClient {
private const val BASE_URL = "https://vi.media.mit.edu/"
private var retrofit: Retrofit? = null
@JvmStatic
val client: Retrofit?
get() {
val client = OkHttpClient.Builder().connectTimeout(1, TimeUnit.MINUTES).readTimeout(30, TimeUnit.SECONDS)
.writeTimeout(30, TimeUnit.SECONDS).addInterceptor { chain ->
val request = chain.request().newBuilder()
.addHeader("Accept-Encoding", "gzip").build()
chain.proceed(request)
}
.retryOnConnectionFailure(true).addInterceptor(RetryInterceptor()).build()
val client = OkHttpClient.Builder().connectTimeout(1, TimeUnit.MINUTES)
.readTimeout(30, TimeUnit.SECONDS).writeTimeout(30, TimeUnit.SECONDS).build()
if (retrofit == null) {
retrofit = Retrofit.Builder()
.baseUrl(BASE_URL).client(client)
.addConverterFactory(GsonConverterFactory.create(
GsonBuilder()
.excludeFieldsWithModifiers(Modifier.FINAL, Modifier.TRANSIENT, Modifier.STATIC)
.serializeNulls()
.create()
.baseUrl(BASE_URL).client(client).addConverterFactory(
GsonConverterFactory.create(
GsonBuilder()
.excludeFieldsWithModifiers(Modifier.FINAL, Modifier.TRANSIENT, Modifier.STATIC)
.serializeNulls().create()
)
).build()
}
return retrofit
}

class RetryInterceptor : Interceptor {
val maxRetryCount = 3
val retryDelayMillis = 1000L

override fun intercept(chain: Interceptor.Chain): Response {
var attempt = 0
var response: Response? = null
val request = chain.request()
val url = request.url().toString()

while (true) {
try {
response?.close()
response = chain.proceed(request)

when (response.code()) {
in 200..299 -> {
return response
}
404 -> {
return response
}
401, 403 -> {
response.close()
throw IOException("Authentication failed: ${response.code()}")
}
else -> {
response.close()
throw IOException("Response unsuccessful: ${response.code()}")
}
}
} catch (e: IOException) {
attempt++

if (attempt >= maxRetryCount) {
throw IOException("Failed after $maxRetryCount attempts: $url", e)
}

val delayMillis = retryDelayMillis * 2.0.pow((attempt - 1).toDouble()).toLong()
Thread.sleep(delayMillis)
}
}
}
}
}
138 changes: 30 additions & 108 deletions app/src/main/java/org/ole/planet/myplanet/service/SyncManager.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,111 +144,50 @@ class SyncManager private constructor(private val context: Context) {
}
}

@Throws(IOException::class)
private fun syncResource(dbClient: ApiInterface?) {
val newIds: MutableList<String?> = ArrayList()
try {
val allDocs = dbClient?.getJsonObject(Utilities.header, "${Utilities.getUrl()}/resources/_all_docs?include_doc=false")
val all = allDocs?.execute()
if (all?.isSuccessful != true) {
return
}

val rows = getJsonArray("rows", all.body())
val keys: MutableList<String> = ArrayList()
val failedIds: MutableList<String> = ArrayList()

for (i in 0 until rows.size()) {
val `object` = rows[i].asJsonObject
if (!TextUtils.isEmpty(getString("id", `object`))) {
keys.add(getString("key", `object`))
}

if (i == rows.size() - 1 || keys.size == 1000) {
val obj = JsonObject()
obj.add("keys", Gson().fromJson(Gson().toJson(keys), JsonArray::class.java))
val response = dbClient.findDocs(Utilities.header, "application/json", "${Utilities.getUrl()}/resources/_all_docs?include_docs=true", obj).execute()

when {
response.isSuccessful == true -> {
response.body()?.let { body ->
val ids: List<String?> = save(getJsonArray("rows", body), mRealm)
newIds.addAll(ids)
}
}
response.code() == 404 -> {
failedIds.addAll(keys)
}
else -> {
val errorMessage = "Failed to sync resources: ${response.code()}"
handleException(errorMessage)

when (response.code()) {
in 500..599 -> {
addToRetryQueue(keys)
}
401, 403 -> {
handleAuthenticationError()
}
else -> {
failedIds.addAll(keys)
}
}
}
}
keys.clear()
val allDocs = dbClient?.getJsonObject(Utilities.header, Utilities.getUrl() + "/resources/_all_docs?include_doc=false")
val all = allDocs?.execute()
val rows = getJsonArray("rows", all?.body())
val keys: MutableList<String> = ArrayList()
for (i in 0 until rows.size()) {
val `object` = rows[i].asJsonObject
if (!TextUtils.isEmpty(getString("id", `object`))) keys.add(getString("key", `object`))
if (i == rows.size() - 1 || keys.size == 1000) {
val obj = JsonObject()
obj.add("keys", Gson().fromJson(Gson().toJson(keys), JsonArray::class.java))
val response = dbClient?.findDocs(Utilities.header, "application/json", Utilities.getUrl() + "/resources/_all_docs?include_docs=true", obj)?.execute()
if (response?.body() != null) {
val ids: List<String?> = save(getJsonArray("rows", response.body()), mRealm)
newIds.addAll(ids)
}
}
} catch (e: Exception) {
e.printStackTrace()
} finally {
try {
removeDeletedResource(newIds, mRealm)
} catch (e: Exception) {
e.printStackTrace()
keys.clear()
}
}
}

private fun addToRetryQueue(keys: List<String>) {
settings.edit().apply {
val existingQueue = settings.getStringSet("retry_queue", setOf()) ?: setOf()
putStringSet("retry_queue", existingQueue + keys)
apply()
}
}

private fun handleAuthenticationError() {
settings.edit().remove("credentials").apply()
handleException("Authentication failed.")
removeDeletedResource(newIds, mRealm)
}

private fun myLibraryTransactionSync() {
val apiInterface = client?.create(ApiInterface::class.java)
try {
val response = apiInterface?.getDocuments(Utilities.header, "${Utilities.getUrl()}/shelf/_all_docs")?.execute()

val res = response?.body()
res?.rows?.let { rows ->
for (i in rows.indices) {
shelfDoc = rows[i]
populateShelfItems(apiInterface)
}
val res = apiInterface?.getDocuments(Utilities.header, "${Utilities.getUrl()}/shelf/_all_docs")?.execute()?.body()
for (i in res?.rows!!.indices) {
shelfDoc = res.rows!![i]
populateShelfItems(apiInterface)
}
} catch (e: IOException) {
e.printStackTrace()
}
}

private fun populateShelfItems(apiInterface: ApiInterface?) {
private fun populateShelfItems(apiInterface: ApiInterface) {
try {
val response = apiInterface?.getJsonObject(Utilities.header, "${Utilities.getUrl()}/shelf/${shelfDoc?.id}")?.execute()

response?.body()?.let { jsonDoc ->
for (i in Constants.shelfDataList.indices) {
val shelfData = Constants.shelfDataList[i]
val array = getJsonArray(shelfData.key, jsonDoc)
memberShelfData(array, shelfData)
}
val jsonDoc = apiInterface.getJsonObject(Utilities.header, "${Utilities.getUrl()}/shelf/${shelfDoc?.id}").execute().body()
for (i in Constants.shelfDataList.indices) {
val shelfData = Constants.shelfDataList[i]
val array = getJsonArray(shelfData.key, jsonDoc)
memberShelfData(array, shelfData)
}
} catch (err: Exception) {
err.printStackTrace()
Expand Down Expand Up @@ -278,27 +217,10 @@ class SyncManager private constructor(private val context: Context) {
}

private fun validateDocument(arrayCategoryIds: JsonArray, x: Int) {
val apiInterface = client?.create(ApiInterface::class.java)
val apiInterface = client!!.create(ApiInterface::class.java)
try {
val response = apiInterface?.getJsonObject(Utilities.header, "${Utilities.getUrl()}/${stringArray[2]}/${arrayCategoryIds[x].asString}")?.execute()

when {
response?.isSuccessful == true -> {
response.body()?.let { resourceDoc ->
triggerInsert(stringArray, resourceDoc)
}
}
response?.code() == 404 -> {
return
}
else -> {
val errorMessage = "Failed to validate document: ${response?.code()}"
handleException(errorMessage)
if (response?.code() in 500..599) {
throw IOException(errorMessage)
}
}
}
val resourceDoc = apiInterface.getJsonObject(Utilities.header, "${Utilities.getUrl()}/${stringArray[2]}/${arrayCategoryIds[x].asString}").execute().body()
resourceDoc?.let { triggerInsert(stringArray, it) }
} catch (e: IOException) {
e.printStackTrace()
}
Expand Down

0 comments on commit fd4376e

Please sign in to comment.