fixes #208 (#209)

dev-claw · Dec 27, 2024 · c74d711 · c74d711
1 parent 095e69d
commit c74d711
Show file tree

Hide file tree

Showing 23 changed files with 206 additions and 204 deletions.
diff --git a/pom.xml b/pom.xml
@@ -25,7 +25,7 @@
         <koin.version>4.0.0</koin.version>
         <h2.version>2.2.224</h2.version>
         <htmlcleaner.version>2.29</htmlcleaner.version>
-        <failsafe.version>2.4.4</failsafe.version>
+        <failsafe.version>3.3.2</failsafe.version>
         <caffeine.version>3.1.8</caffeine.version>
         <jna-platform.version>5.14.0</jna-platform.version>
         <kotlinx-serialization-json.version>1.7.3</kotlinx-serialization-json.version>
@@ -142,8 +142,8 @@
                 <version>${htmlcleaner.version}</version>
             </dependency>
             <dependency>
+                <groupId>dev.failsafe</groupId>
                 <artifactId>failsafe</artifactId>
-                <groupId>net.jodah</groupId>
                 <version>${failsafe.version}</version>
             </dependency>
             <dependency>

diff --git a/vripper-core/pom.xml b/vripper-core/pom.xml
@@ -69,7 +69,7 @@
         </dependency>
         <dependency>
             <artifactId>failsafe</artifactId>
-            <groupId>net.jodah</groupId>
+            <groupId>dev.failsafe</groupId>
         </dependency>
         <dependency>
             <artifactId>caffeine</artifactId>

diff --git a/vripper-core/src/main/kotlin/me/vripper/Module.kt b/vripper-core/src/main/kotlin/me/vripper/Module.kt
@@ -2,11 +2,11 @@ package me.vripper
 
 import me.vripper.data.repositories.ImageRepository
 import me.vripper.data.repositories.MetadataRepository
-import me.vripper.data.repositories.PostDownloadStateRepository
+import me.vripper.data.repositories.PostRepository
 import me.vripper.data.repositories.ThreadRepository
 import me.vripper.data.repositories.impl.ImageRepositoryImpl
 import me.vripper.data.repositories.impl.MetadataRepositoryImpl
-import me.vripper.data.repositories.impl.PostDownloadStateRepositoryImpl
+import me.vripper.data.repositories.impl.PostRepositoryImpl
 import me.vripper.data.repositories.impl.ThreadRepositoryImpl
 import me.vripper.download.DownloadService
 import me.vripper.event.EventBus
@@ -29,8 +29,8 @@ val coreModule = module {
     single<ImageRepository> {
         ImageRepositoryImpl()
     }
-    single<PostDownloadStateRepository> {
-        PostDownloadStateRepositoryImpl()
+    single<PostRepository> {
+        PostRepositoryImpl()
     }
     single<MetadataRepository> {
         MetadataRepositoryImpl()

diff --git a/...positories/PostDownloadStateRepository.kt → ...ipper/data/repositories/PostRepository.kt b/...positories/PostDownloadStateRepository.kt → ...ipper/data/repositories/PostRepository.kt
@@ -1,12 +1,11 @@
 package me.vripper.data.repositories
 
 import me.vripper.entities.PostEntity
-import java.util.*
 
-internal interface PostDownloadStateRepository {
+internal interface PostRepository {
     fun save(postEntities: List<PostEntity>): List<PostEntity>
-    fun findByPostId(postId: Long): Optional<PostEntity>
-    fun findById(id: Long): Optional<PostEntity>
+    fun findByPostId(postId: Long): PostEntity?
+    fun findById(id: Long): PostEntity?
     fun findCompleted(): List<Long>
     fun findAll(): List<PostEntity>
     fun existByPostId(postId: Long): Boolean

diff --git a/...s/impl/PostDownloadStateRepositoryImpl.kt → ...a/repositories/impl/PostRepositoryImpl.kt b/...s/impl/PostDownloadStateRepositoryImpl.kt → ...a/repositories/impl/PostRepositoryImpl.kt
@@ -1,17 +1,16 @@
 package me.vripper.data.repositories.impl
 
-import me.vripper.data.repositories.PostDownloadStateRepository
+import me.vripper.data.repositories.PostRepository
 import me.vripper.data.tables.PostTable
 import me.vripper.entities.PostEntity
 import me.vripper.entities.Status
 import org.jetbrains.exposed.sql.*
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
 import org.jetbrains.exposed.sql.transactions.TransactionManager
 import java.sql.Connection
-import java.util.*
 
-internal class PostDownloadStateRepositoryImpl :
-    PostDownloadStateRepository {
+internal class PostRepositoryImpl :
+    PostRepository {
 
     private val delimiter = ";"
 
@@ -37,15 +36,11 @@ internal class PostDownloadStateRepositoryImpl :
         }.map(::transform)
     }
 
-    override fun findByPostId(postId: Long): Optional<PostEntity> {
+    override fun findByPostId(postId: Long): PostEntity? {
         val result = PostTable.selectAll().where {
             PostTable.postId eq postId
         }.map(::transform)
-        return if (result.isEmpty()) {
-            Optional.empty()
-        } else {
-            Optional.of(result.first())
-        }
+        return result.firstOrNull()
     }
 
     override fun findCompleted(): List<Long> {
@@ -54,16 +49,11 @@ internal class PostDownloadStateRepositoryImpl :
         }.map { it[PostTable.postId] }
     }
 
-    override fun findById(id: Long): Optional<PostEntity> {
+    override fun findById(id: Long): PostEntity? {
         val result = PostTable.selectAll().where {
             PostTable.id eq id
         }.map { transform(it) }
-
-        return if (result.isEmpty()) {
-            Optional.empty()
-        } else {
-            Optional.of(result.first())
-        }
+        return result.firstOrNull()
     }
 
     override fun findAll(): List<PostEntity> {

diff --git a/vripper-core/src/main/kotlin/me/vripper/download/DownloadService.kt b/vripper-core/src/main/kotlin/me/vripper/download/DownloadService.kt
@@ -1,7 +1,8 @@
 package me.vripper.download
 
+import dev.failsafe.Failsafe
+import dev.failsafe.RetryPolicy
 import kotlinx.coroutines.Runnable
-import kotlinx.coroutines.launch
 import me.vripper.entities.ImageEntity
 import me.vripper.entities.PostEntity
 import me.vripper.entities.Status
@@ -16,10 +17,8 @@ import me.vripper.services.DataTransaction
 import me.vripper.services.RetryPolicyService
 import me.vripper.services.SettingsService
 import me.vripper.services.VGAuthService
-import me.vripper.utilities.GlobalScopeCoroutine
 import me.vripper.utilities.LoggerDelegate
-import net.jodah.failsafe.Failsafe
-import net.jodah.failsafe.RetryPolicy
+import me.vripper.utilities.executorService
 import org.jetbrains.exposed.sql.transactions.transaction
 import java.util.concurrent.locks.ReentrantLock
 import kotlin.concurrent.withLock
@@ -46,14 +45,14 @@ internal class DownloadService(
                 lock.withLock {
                     candidates.addAll(getCandidates(candidateCount()))
                     candidates.forEach {
-                        if (canRun(it.context.imageEntity.host)) {
+                        if (canRun(it.imageEntity.host)) {
                             accepted.add(it)
-                            running[it.context.imageEntity.host]!!.add(it)
-                            log.debug("${it.context.imageEntity.url} accepted to run")
+                            running[it.imageEntity.host]!!.add(it)
+                            log.debug("${it.imageEntity.url} accepted to run")
                         }
                     }
                     accepted.forEach {
-                        pending[it.context.imageEntity.host]?.remove(it)
+                        pending[it.imageEntity.host]?.remove(it)
                         scheduleForDownload(it)
                     }
                     accepted.clear()
@@ -144,18 +143,18 @@ internal class DownloadService(
     }
 
     private fun isPending(postId: Long): Boolean {
-        return pending.values.flatten().any { it.context.imageEntity.postId == postId }
+        return pending.values.flatten().any { it.imageEntity.postId == postId }
     }
 
     private fun isRunning(postId: Long): Boolean {
-        return running.values.flatten().any { it.context.imageEntity.postId == postId }
+        return running.values.flatten().any { it.imageEntity.postId == postId }
     }
 
     private fun stopAll() {
         lock.withLock {
             pending.values.clear()
             running.values.flatten().forEach { obj: ImageDownloadRunnable -> obj.stop() }
-            while (running.values.flatten().count { !it.context.completed } > 0) {
+            while (running.values.flatten().count { !it.completed } > 0) {
                 Thread.sleep(100)
             }
             dataTransaction.findAllNonCompletedPostIds().forEach {
@@ -169,13 +168,13 @@ internal class DownloadService(
         lock.withLock {
             for (postId in postIds) {
                 pending.values.forEach { pending ->
-                    pending.removeIf { it.context.imageEntity.postId == postId }
+                    pending.removeIf { it.imageEntity.postId == postId }
                 }
                 running.values.flatten()
-                    .filter { p: ImageDownloadRunnable -> p.context.imageEntity.postId == postId }
+                    .filter { p: ImageDownloadRunnable -> p.imageEntity.postId == postId }
                     .forEach { obj: ImageDownloadRunnable -> obj.stop() }
                 while (running.values.flatten()
-                        .count { !it.context.completed && it.context.imageEntity.postId == postId } > 0
+                        .count { !it.completed && it.imageEntity.postId == postId } > 0
                 ) {
                     Thread.sleep(100)
                 }
@@ -213,7 +212,7 @@ internal class DownloadService(
 
             val list: List<ImageDownloadRunnable> =
                 pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.postRank }
-                    .thenComparingInt { it.context.imageEntity.index })
+                    .thenComparingInt { it.imageEntity.index })
 
             for (imageDownloadRunnable in list) {
                 val count = hostIntegerMap[host] ?: 0
@@ -229,47 +228,42 @@ internal class DownloadService(
     }
 
     private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) {
-        log.debug("Scheduling a job for ${imageDownloadRunnable.context.imageEntity.url}")
-        GlobalScopeCoroutine.launch {
-            eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
-            try {
-                Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicyForDownload("Failed to download ${imageDownloadRunnable.context.imageEntity.url}: "))
-                    .onFailure {
-                        log.error(
-                            "Failed to download ${imageDownloadRunnable.context.imageEntity.url} after ${it.attemptCount} tries",
-                            it.failure
-                        )
-                        val image = imageDownloadRunnable.context.imageEntity
-                        image.status = Status.ERROR
-                        dataTransaction.updateImage(image)
-                    }
-                    .onComplete {
-                        afterJobFinish(imageDownloadRunnable)
-                        eventBus.publishEvent(
-                            QueueStateEvent(
-                                QueueState(
-                                    runningCount(), pendingCount()
-                                )
-                            )
-                        )
-                        eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
-                        log.debug(
-                            "Finished downloading ${imageDownloadRunnable.context.imageEntity.url}"
-                        )
-                    }.run(imageDownloadRunnable::run)
-            } catch (e: Exception) {
-                log.error("Download Failure", e)
+        log.debug("Scheduling a job for ${imageDownloadRunnable.imageEntity.url}")
+        eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
+        Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${imageDownloadRunnable.imageEntity.url}: "))
+            .with(executorService)
+            .onFailure {
+                log.error(
+                    "Failed to download ${imageDownloadRunnable.imageEntity.url} after ${it.attemptCount} tries",
+                    it.exception
+                )
+                val image = imageDownloadRunnable.imageEntity
+                image.status = Status.ERROR
+                dataTransaction.updateImage(image)
             }
-        }
+            .onComplete {
+                afterJobFinish(imageDownloadRunnable)
+                eventBus.publishEvent(
+                    QueueStateEvent(
+                        QueueState(
+                            runningCount(), pendingCount()
+                        )
+                    )
+                )
+                eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
+                log.debug(
+                    "Finished downloading ${imageDownloadRunnable.imageEntity.url}"
+                )
+            }.runAsync(imageDownloadRunnable)
     }
 
     private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
         lock.withLock {
-            val image = imageDownloadRunnable.context.imageEntity
+            val image = imageDownloadRunnable.imageEntity
             running[image.host]!!.remove(imageDownloadRunnable)
             if (!isPending(image.postId) && !isRunning(
                     image.postId
-                ) && !imageDownloadRunnable.context.stopped
+                ) && !imageDownloadRunnable.stopped
             ) {
                 dataTransaction.finishPost(image.postId, true)
             }

diff --git a/vripper-core/src/main/kotlin/me/vripper/download/ImageDownloadContext.kt b/vripper-core/src/main/kotlin/me/vripper/download/ImageDownloadContext.kt
@@ -15,8 +15,6 @@ internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings:
         HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
     val requests = mutableListOf<HttpUriRequestBase>()
     val postId = imageEntity.postIdRef
-    var stopped = false
-    var completed = false
 
     fun cancelCoroutines() {
         runBlocking {