Support Custom Repos (#803)

* Support custom repos * Fix migration * Make extension after update optional
Suwayomi · Jan 6, 2024 · 230427e · 230427e
1 parent abf1af4
commit 230427e
Show file tree

Hide file tree

Showing 19 changed files with 149 additions and 22 deletions.
diff --git a/AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigModule.kt b/AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigModule.kt
@@ -8,6 +8,8 @@ package xyz.nulldev.ts.config
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import com.typesafe.config.Config
+import com.typesafe.config.ConfigFactory
+import com.typesafe.config.ConfigValueFactory
 import io.github.config4k.getValue
 import kotlin.reflect.KProperty
 
@@ -30,18 +32,24 @@ class SystemPropertyOverrideDelegate(val getConfig: () -> Config, val moduleName
         thisRef: R,
         property: KProperty<*>,
     ): T {
-        val configValue: T = getConfig().getValue(thisRef, property)
+        val config = getConfig()
+        val configValue: T = config.getValue(thisRef, property)
 
         val combined =
             System.getProperty(
                 "$CONFIG_PREFIX.$moduleName.${property.name}",
-                configValue.toString(),
+                if (T::class.simpleName == "List") {
+                    ConfigValueFactory.fromAnyRef(configValue).render()
+                } else {
+                    configValue.toString()
+                },
             )
 
         return when (T::class.simpleName) {
             "Int" -> combined.toInt()
             "Boolean" -> combined.toBoolean()
             "Double" -> combined.toDouble()
+            "List" -> ConfigFactory.parseString("internal=" + combined).getStringList("internal").orEmpty()
             // add more types as needed
             else -> combined // covers String
         } as T

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ExtensionMutation.kt
@@ -20,7 +20,7 @@ class ExtensionMutation {
 
     data class UpdateExtensionPayload(
         val clientMutationId: String?,
-        val extension: ExtensionType,
+        val extension: ExtensionType?,
     )
 
     data class UpdateExtensionInput(
@@ -77,7 +77,8 @@ class ExtensionMutation {
         }.thenApply {
             val extension =
                 transaction {
-                    ExtensionType(ExtensionTable.select { ExtensionTable.pkgName eq id }.first())
+                    ExtensionTable.select { ExtensionTable.pkgName eq id }.firstOrNull()
+                        ?.let { ExtensionType(it) }
                 }
 
             UpdateExtensionPayload(

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt
@@ -55,6 +55,9 @@ class SettingsMutation {
         updateSetting(settings.excludeEntryWithUnreadChapters, serverConfig.excludeEntryWithUnreadChapters)
         updateSetting(settings.autoDownloadAheadLimit, serverConfig.autoDownloadAheadLimit)
 
+        // extension
+        updateSetting(settings.extensionRepos, serverConfig.extensionRepos)
+
         // requests
         updateSetting(settings.maxSourcesInParallel, serverConfig.maxSourcesInParallel)
 

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ExtensionQuery.kt
@@ -82,6 +82,7 @@ class ExtensionQuery {
     }
 
     data class ExtensionCondition(
+        val repo: String? = null,
         val apkName: String? = null,
         val iconUrl: String? = null,
         val name: String? = null,
@@ -96,6 +97,7 @@ class ExtensionQuery {
     ) : HasGetOp {
         override fun getOp(): Op<Boolean>? {
             val opAnd = OpAnd()
+            opAnd.eq(repo, ExtensionTable.repo)
             opAnd.eq(apkName, ExtensionTable.apkName)
             opAnd.eq(iconUrl, ExtensionTable.iconUrl)
             opAnd.eq(name, ExtensionTable.name)
@@ -112,6 +114,7 @@ class ExtensionQuery {
     }
 
     data class ExtensionFilter(
+        val repo: StringFilter? = null,
         val apkName: StringFilter? = null,
         val iconUrl: StringFilter? = null,
         val name: StringFilter? = null,
@@ -129,6 +132,7 @@ class ExtensionQuery {
     ) : Filter<ExtensionFilter> {
         override fun getOpList(): List<Op<Boolean>> {
             return listOfNotNull(
+                andFilterWithCompareString(ExtensionTable.repo, repo),
                 andFilterWithCompareString(ExtensionTable.apkName, apkName),
                 andFilterWithCompareString(ExtensionTable.iconUrl, iconUrl),
                 andFilterWithCompareString(ExtensionTable.name, name),

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/ExtensionType.kt
@@ -20,6 +20,7 @@ import suwayomi.tachidesk.manga.model.table.ExtensionTable
 import java.util.concurrent.CompletableFuture
 
 class ExtensionType(
+    val repo: String?,
     val apkName: String,
     val iconUrl: String,
     val name: String,
@@ -33,6 +34,7 @@ class ExtensionType(
     val isObsolete: Boolean,
 ) : Node {
     constructor(row: ResultRow) : this(
+        repo = row[ExtensionTable.repo],
         apkName = row[ExtensionTable.apkName],
         iconUrl = Extension.getExtensionIconUrl(row[ExtensionTable.apkName]),
         name = row[ExtensionTable.name],

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/MangaType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/MangaType.kt
@@ -8,6 +8,7 @@
 package suwayomi.tachidesk.graphql.types
 
 import com.expediagroup.graphql.server.extensions.getValueFromDataLoader
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import graphql.schema.DataFetchingEnvironment
 import org.jetbrains.exposed.sql.ResultRow
 import suwayomi.tachidesk.graphql.server.primitives.Cursor
@@ -37,6 +38,7 @@ class MangaType(
     val status: MangaStatus,
     val inLibrary: Boolean,
     val inLibraryAt: Long,
+    val updateStrategy: UpdateStrategy,
     val realUrl: String?,
     var lastFetchedAt: Long?, // todo
     var chaptersLastFetchedAt: Long?, // todo
@@ -73,6 +75,7 @@ class MangaType(
         MangaStatus.valueOf(row[MangaTable.status]),
         row[MangaTable.inLibrary],
         row[MangaTable.inLibraryAt],
+        UpdateStrategy.valueOf(row[MangaTable.updateStrategy]),
         row[MangaTable.realUrl],
         row[MangaTable.lastFetchedAt],
         row[MangaTable.chaptersLastFetchedAt],
@@ -92,6 +95,7 @@ class MangaType(
         MangaStatus.valueOf(dataClass.status),
         dataClass.inLibrary,
         dataClass.inLibraryAt,
+        dataClass.updateStrategy,
         dataClass.realUrl,
         dataClass.lastFetchedAt,
         dataClass.chaptersLastFetchedAt,

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt
@@ -40,6 +40,9 @@ interface Settings : Node {
     val excludeEntryWithUnreadChapters: Boolean?
     val autoDownloadAheadLimit: Int?
 
+    // extension
+    val extensionRepos: List<String>?
+
     // requests
     val maxSourcesInParallel: Int?
 
@@ -90,6 +93,8 @@ data class PartialSettingsType(
     override val autoDownloadNewChapters: Boolean?,
     override val excludeEntryWithUnreadChapters: Boolean?,
     override val autoDownloadAheadLimit: Int?,
+    // extension
+    override val extensionRepos: List<String>?,
     // requests
     override val maxSourcesInParallel: Int?,
     // updater
@@ -135,6 +140,8 @@ class SettingsType(
     override val autoDownloadNewChapters: Boolean,
     override val excludeEntryWithUnreadChapters: Boolean,
     override val autoDownloadAheadLimit: Int,
+    // extension
+    override val extensionRepos: List<String>,
     // requests
     override val maxSourcesInParallel: Int,
     // updater
@@ -179,6 +186,8 @@ class SettingsType(
         config.autoDownloadNewChapters.value,
         config.excludeEntryWithUnreadChapters.value,
         config.autoDownloadAheadLimit.value,
+        // extension
+        config.extensionRepos.value,
         // requests
         config.maxSourcesInParallel.value,
         // updater

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt
@@ -143,7 +143,7 @@ object Chapter {
                     val chapterNumber = ChapterRecognition.parseChapterNumber(manga.title, chapter.name, chapter.chapter_number.toDouble())
                     chapter.chapter_number = chapterNumber.toFloat()
                     chapter.name = chapter.name.sanitize(manga.title)
-                    chapter.scanlator = chapter.scanlator?.ifBlank { null }
+                    chapter.scanlator = chapter.scanlator?.ifBlank { null }?.trim()
                 }
 
                 val now = Instant.now().epochSecond

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt
@@ -23,6 +23,7 @@ import suwayomi.tachidesk.manga.impl.extension.github.ExtensionGithubApi
 import suwayomi.tachidesk.manga.impl.extension.github.OnlineExtension
 import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass
 import suwayomi.tachidesk.manga.model.table.ExtensionTable
+import suwayomi.tachidesk.server.serverConfig
 import java.util.concurrent.ConcurrentHashMap
 import kotlin.time.Duration.Companion.seconds
 
@@ -38,7 +39,17 @@ object ExtensionsList {
             logger.debug("Getting extensions list from the internet")
             lastUpdateCheck = System.currentTimeMillis()
 
-            val foundExtensions = ExtensionGithubApi.findExtensions()
+            val extensions =
+                (listOf(ExtensionGithubApi.REPO_URL_PREFIX) + serverConfig.extensionRepos.value).map { repo ->
+                    kotlin.runCatching {
+                        ExtensionGithubApi.findExtensions(repo)
+                    }.onFailure {
+                        logger.warn(it) {
+                            "Failed to fetch extensions for repo: $repo"
+                        }
+                    }
+                }
+            val foundExtensions = extensions.mapNotNull { it.getOrNull() }.flatten()
             updateExtensionDatabase(foundExtensions)
         } else {
             logger.debug("used cached extension list")
@@ -54,6 +65,7 @@ object ExtensionsList {
         transaction {
             ExtensionTable.selectAll().filter { it[ExtensionTable.name] != LocalSource.EXTENSION_NAME }.map {
                 ExtensionDataClass(
+                    it[ExtensionTable.repo],
                     it[ExtensionTable.apkName],
                     getExtensionIconUrl(it[ExtensionTable.apkName]),
                     it[ExtensionTable.name],
@@ -77,7 +89,7 @@ object ExtensionsList {
             val extensionsToUpdate = mutableListOf<Pair<OnlineExtension, ResultRow>>()
             val extensionsToInsert = mutableListOf<OnlineExtension>()
             val extensionsToDelete =
-                installedExtensions.mapNotNull { (pkgName, extension) ->
+                installedExtensions.filter { it.value[ExtensionTable.repo] != null }.mapNotNull { (pkgName, extension) ->
                     extension.takeUnless { foundExtensions.any { it.pkgName == pkgName } }
                 }
             foundExtensions.forEach {
@@ -124,6 +136,7 @@ object ExtensionsList {
                         extensionsToFullyUpdate.forEach { (foundExtension, extensionRecord) ->
                             addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable))
                             // extension is not installed, so we can overwrite the data without a care
+                            this[ExtensionTable.repo] = foundExtension.repo
                             this[ExtensionTable.name] = foundExtension.name
                             this[ExtensionTable.versionName] = foundExtension.versionName
                             this[ExtensionTable.versionCode] = foundExtension.versionCode
@@ -138,6 +151,7 @@ object ExtensionsList {
             }
             if (extensionsToInsert.isNotEmpty()) {
                 ExtensionTable.batchInsert(extensionsToInsert) { foundExtension ->
+                    this[ExtensionTable.repo] = foundExtension.repo
                     this[ExtensionTable.name] = foundExtension.name
                     this[ExtensionTable.pkgName] = foundExtension.pkgName
                     this[ExtensionTable.versionName] = foundExtension.versionName

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt
@@ -20,7 +20,7 @@ import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass
 import uy.kohesive.injekt.injectLazy
 
 object ExtensionGithubApi {
-    private const val REPO_URL_PREFIX = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/"
+    const val REPO_URL_PREFIX = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/"
     private const val FALLBACK_REPO_URL_PREFIX = "https://gcore.jsdelivr.net/gh/tachiyomiorg/tachiyomi-extensions@repo/"
     private val logger = KotlinLogging.logger {}
     private val json: Json by injectLazy()
@@ -49,13 +49,13 @@ object ExtensionGithubApi {
 
     private var requiresFallbackSource = false
 
-    suspend fun findExtensions(): List<OnlineExtension> {
+    suspend fun findExtensions(repo: String): List<OnlineExtension> {
         val githubResponse =
             if (requiresFallbackSource) {
                 null
             } else {
                 try {
-                    client.newCall(GET("${REPO_URL_PREFIX}index.min.json")).awaitSuccess()
+                    client.newCall(GET("${repo.repoUrlReplace()}index.min.json")).awaitSuccess()
                 } catch (e: Throwable) {
                     logger.error(e) { "Failed to get extensions from GitHub" }
                     requiresFallbackSource = true
@@ -65,18 +65,18 @@ object ExtensionGithubApi {
 
         val response =
             githubResponse ?: run {
-                client.newCall(GET("${FALLBACK_REPO_URL_PREFIX}index.min.json")).awaitSuccess()
+                client.newCall(GET("${repo.fallbackRepoUrlReplace()}index.min.json")).awaitSuccess()
             }
 
         return with(json) {
             response
                 .parseAs<List<ExtensionJsonObject>>()
-                .toExtensions()
+                .toExtensions(repo.repoUrlReplace())
         }
     }
 
     fun getApkUrl(extension: ExtensionDataClass): String {
-        return "$REPO_URL_PREFIX/apk/${extension.apkName}"
+        return "${extension.repo!!.repoUrlReplace()}/apk/${extension.apkName}"
     }
 
     private val client by lazy {
@@ -91,14 +91,15 @@ object ExtensionGithubApi {
             .build()
     }
 
-    private fun List<ExtensionJsonObject>.toExtensions(): List<OnlineExtension> {
+    private fun List<ExtensionJsonObject>.toExtensions(repo: String): List<OnlineExtension> {
         return this
             .filter {
                 val libVersion = it.version.substringBeforeLast('.').toDouble()
                 libVersion in LIB_VERSION_MIN..LIB_VERSION_MAX
             }
             .map {
                 OnlineExtension(
+                    repo = repo,
                     name = it.name.substringAfter("Tachiyomi: "),
                     pkgName = it.pkg,
                     versionName = it.version,
@@ -109,7 +110,7 @@ object ExtensionGithubApi {
                     hasChangelog = it.hasChangelog == 1,
                     sources = it.sources?.toExtensionSources() ?: emptyList(),
                     apkName = it.apk,
-                    iconUrl = "${REPO_URL_PREFIX}icon/${it.pkg}.png",
+                    iconUrl = "${repo}icon/${it.pkg}.png",
                 )
             }
     }
@@ -124,4 +125,22 @@ object ExtensionGithubApi {
             )
         }
     }
+
+    private fun String.repoUrlReplace() =
+        replace(repoMatchRegex) {
+            "https://raw.githubusercontent.com/${it.groupValues[1]}/${it.groupValues[2]}/" +
+                "${it.groupValues.getOrNull(3)?.ifBlank { null } ?: "repo"}/"
+        }
+
+    private fun String.fallbackRepoUrlReplace() =
+        replace(repoMatchRegex) {
+            "https://gcore.jsdelivr.net/gh/${it.groupValues[1]}/${it.groupValues[2]}@" +
+                "${it.groupValues.getOrNull(3)?.ifBlank { null } ?: "repo"}/"
+        }
+
+    private val repoMatchRegex =
+        (
+            "https:\\/\\/(?:www|raw)?(?:github|githubusercontent)\\.com" +
+                "\\/([^\\/]+)\\/([^\\/]+)(?:\\/(?:tree|blob)\\/(.*))?\\/?"
+        ).toRegex()
 }
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/OnlineExtension.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/OnlineExtension.kt
@@ -15,6 +15,7 @@ data class OnlineExtensionSource(
 )
 
 data class OnlineExtension(
+    val repo: String,
     val name: String,
     val pkgName: String,
     val apkName: String,

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/PackageTools.kt
@@ -44,8 +44,7 @@ object PackageTools {
     const val LIB_VERSION_MAX = 1.5
 
     private const val OFFICIAL_SIGNATURE = "7ce04da7773d41b489f4693a366c36bcd0a11fc39b547168553c285bd7348e23" // inorichi's key
-    private const val UNOFFICIAL_SIGNATURE = "64feb21075ba97ebc9cc981243645b331595c111cef1b0d084236a0403b00581" // ArMor's key
-    val trustedSignatures = setOf(OFFICIAL_SIGNATURE, UNOFFICIAL_SIGNATURE)
+    val trustedSignatures = setOf(OFFICIAL_SIGNATURE)
 
     /**
      * Convert dex to jar, a wrapper for the dex2jar library

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionDataClass.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ExtensionDataClass.kt
@@ -8,6 +8,7 @@ package suwayomi.tachidesk.manga.model.dataclass
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 data class ExtensionDataClass(
+    val repo: String?,
     val apkName: String,
     val iconUrl: String,
     val name: String,

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ExtensionTable.kt
@@ -11,6 +11,7 @@ import org.jetbrains.exposed.dao.id.IntIdTable
 
 object ExtensionTable : IntIdTable() {
     val apkName = varchar("apk_name", 1024)
+    val repo = varchar("repo", 1024).nullable()
 
     // default is the local source icon from tachiyomi
     @Suppress("ktlint:standard:max-line-length")

diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt
@@ -80,6 +80,6 @@ enum class MangaStatus(val value: Int) {
     ;
 
     companion object {
-        fun valueOf(value: Int): MangaStatus = values().find { it.value == value } ?: UNKNOWN
+        fun valueOf(value: Int): MangaStatus = entries.find { it.value == value } ?: UNKNOWN
     }
 }