fixes #165

dev-claw · Feb 25, 2024 · ed9b26b · ed9b26b
1 parent 37b54d5
commit ed9b26b
Show file tree

Hide file tree

Showing 66 changed files with 2,391 additions and 364 deletions.
diff --git a/vripper-core/src/main/kotlin/me/vripper/event/Event.kt b/vripper-core/src/main/kotlin/me/vripper/event/Event.kt
@@ -11,6 +11,7 @@ data class PostUpdateEvent(val posts: List<Post>)
 data class PostDeleteEvent(val postIds: List<Long>)
 data class ImageEvent(val images: List<Image>)
 data class ThreadCreateEvent(val thread: Thread)
+data class ThreadUpdateEvent(val thread: Thread)
 data class ThreadDeleteEvent(val threadId: Long)
 class ThreadClearEvent
 data class VGUserLoginEvent(val username: String)

diff --git a/vripper-core/src/main/kotlin/me/vripper/parser/PostLookupAPIParser.kt b/vripper-core/src/main/kotlin/me/vripper/parser/PostLookupAPIParser.kt
@@ -0,0 +1,75 @@
+package me.vripper.parser
+
+import me.vripper.entities.LogEntry
+import me.vripper.exception.DownloadException
+import me.vripper.exception.PostParseException
+import me.vripper.model.PostItem
+import me.vripper.services.*
+import me.vripper.utilities.Tasks
+import me.vripper.utilities.formatToString
+import net.jodah.failsafe.Failsafe
+import net.jodah.failsafe.function.CheckedSupplier
+import org.apache.hc.client5.http.classic.HttpClient
+import org.apache.hc.client5.http.classic.methods.HttpGet
+import org.apache.hc.core5.net.URIBuilder
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+import javax.xml.parsers.SAXParserFactory
+
+class PostLookupAPIParser(private val threadId: Long, private val postId: Long) : KoinComponent {
+    private val log by me.vripper.delegate.LoggerDelegate()
+    private val settingsService: SettingsService by inject()
+    private val retryPolicyService: RetryPolicyService by inject()
+    private val httpService: HTTPService by inject()
+    private val vgAuthService: VGAuthService by inject()
+    private val dataTransaction: DataTransaction by inject()
+
+    @Throws(PostParseException::class)
+    fun parse(): PostItem {
+        log.debug("Parsing post $postId")
+        val httpGet =
+            HttpGet(URIBuilder("${settingsService.settings.viperSettings.host}/vr.php").also {
+                it.setParameter(
+                    "p", postId.toString()
+                )
+            }.build())
+        val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
+        log.debug("Requesting {}", httpGet)
+        Tasks.increment()
+        return try {
+            Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>()).onFailure {
+                log.error(
+                    "parsing failed for thread $threadId, post $postId", it.failure
+                )
+                dataTransaction.saveLog(
+                    LogEntry(
+                        type = LogEntry.Type.POST, status = LogEntry.Status.ERROR, message = """
+                    Failed to process thread $threadId, post $postId
+                    ${it.failure.formatToString()}
+                    """.trimIndent()
+                    )
+                )
+            }.get(CheckedSupplier {
+                val connection: HttpClient = httpService.client
+                connection.execute(
+                    httpGet, vgAuthService.context
+                ) { response ->
+                    if (response.code / 100 != 2) {
+                        throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
+                    }
+                    factory.newSAXParser()
+                        .parse(response.entity.content, threadLookupAPIResponseHandler)
+                    threadLookupAPIResponseHandler.result.postItemList.first()
+                }
+            })
+        } catch (e: Exception) {
+            throw PostParseException(e)
+        } finally {
+            Tasks.decrement()
+        }
+    }
+
+    companion object {
+        private val factory = SAXParserFactory.newInstance()
+    }
+}
diff --git a/vripper-core/src/main/kotlin/me/vripper/parser/ThreadLookupAPIParser.kt b/vripper-core/src/main/kotlin/me/vripper/parser/ThreadLookupAPIParser.kt
@@ -1,12 +1,12 @@
 package me.vripper.parser
 
+import me.vripper.entities.LogEntry
 import me.vripper.exception.DownloadException
 import me.vripper.exception.PostParseException
 import me.vripper.model.ThreadItem
-import me.vripper.services.HTTPService
-import me.vripper.services.RetryPolicyService
-import me.vripper.services.SettingsService
-import me.vripper.services.VGAuthService
+import me.vripper.services.*
+import me.vripper.utilities.Tasks
+import me.vripper.utilities.formatToString
 import net.jodah.failsafe.Failsafe
 import net.jodah.failsafe.function.CheckedSupplier
 import org.apache.hc.client5.http.classic.methods.HttpGet
@@ -23,6 +23,7 @@ class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent {
     private val retryPolicyService: RetryPolicyService by inject()
     private val vgAuthService: VGAuthService by inject()
     private val settingsService: SettingsService by inject()
+    private val dataTransaction: DataTransaction by inject()
 
     @Throws(PostParseException::class)
     fun parse(): ThreadItem {
@@ -36,12 +37,21 @@ class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent {
             }.build())
         val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
         log.debug("Requesting {}", httpGet)
+        Tasks.increment()
         return try {
             Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>()).onFailure {
                 log.error(
                     "parsing failed for thread $threadId",
                     it.failure
                 )
+                dataTransaction.saveLog(
+                    LogEntry(
+                        type = LogEntry.Type.THREAD, status = LogEntry.Status.ERROR, message = """
+                    Failed to process thread $threadId
+                    ${it.failure.formatToString()}
+                    """.trimIndent()
+                    )
+                )
             }.get(CheckedSupplier {
                 cm.client.execute(
                     httpGet, vgAuthService.context
@@ -58,6 +68,8 @@ class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent {
             })
         } catch (e: Exception) {
             throw PostParseException(e)
+        } finally {
+            Tasks.decrement()
         }
     }
 

diff --git a/vripper-core/src/main/kotlin/me/vripper/parser/ThreadLookupAPIResponseHandler.kt b/vripper-core/src/main/kotlin/me/vripper/parser/ThreadLookupAPIResponseHandler.kt
@@ -78,7 +78,7 @@ class ThreadLookupAPIResponseHandler : KoinComponent, DefaultHandler() {
                         postCounter,
                         postTitle,
                         imageItemList.size,
-                        "${settingsService.settings.viperSettings.host}/threads/?p=$postId&viewfull=1#post$postId",
+                        "${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId",
                         hostMap.toMap().map { Pair(it.key.hostName, it.value) },
                         securityToken,
                         forum,

diff --git a/vripper-core/src/main/kotlin/me/vripper/repositories/ThreadRepository.kt b/vripper-core/src/main/kotlin/me/vripper/repositories/ThreadRepository.kt
@@ -5,6 +5,7 @@ import java.util.*
 
 interface ThreadRepository {
     fun save(thread: Thread): Thread
+    fun update(thread: Thread)
     fun findByThreadId(threadId: Long): Optional<Thread>
     fun findAll(): List<Thread>
     fun findById(id: Long): Optional<Thread>

diff --git a/vripper-core/src/main/kotlin/me/vripper/repositories/impl/ThreadRepositoryImpl.kt b/vripper-core/src/main/kotlin/me/vripper/repositories/impl/ThreadRepositoryImpl.kt
@@ -19,6 +19,13 @@ class ThreadRepositoryImpl : ThreadRepository {
         return thread.copy(id = id)
     }
 
+    override fun update(thread: Thread) {
+        ThreadTable.update({ ThreadTable.id eq thread.id }) {
+            it[title] = thread.title
+            it[total] = thread.total
+        }
+    }
+
     override fun findByThreadId(threadId: Long): Optional<Thread> {
         val result = ThreadTable.select {
             ThreadTable.threadId eq threadId

diff --git a/vripper-core/src/main/kotlin/me/vripper/services/AppEndpointService.kt b/vripper-core/src/main/kotlin/me/vripper/services/AppEndpointService.kt
@@ -1,6 +1,7 @@
 package me.vripper.services
 
 import me.vripper.download.DownloadService
+import me.vripper.entities.LogEntry
 import me.vripper.exception.PostParseException
 import me.vripper.model.PostItem
 import me.vripper.model.ThreadPostId
@@ -34,6 +35,13 @@ class AppEndpointService(
         for (link in urlList) {
             log.debug("Starting to process thread: $link")
             if (!link.startsWith(settingsService.settings.viperSettings.host)) {
+                dataTransaction.saveLog(
+                    LogEntry(
+                        type = LogEntry.Type.SCAN,
+                        status = LogEntry.Status.ERROR,
+                        message = "Invalid link $link, only links starting with ${settingsService.settings.viperSettings.host} can be scanned"
+                    )
+                )
                 continue
             }
             var threadId: Long
@@ -61,6 +69,13 @@ class AppEndpointService(
                 }
             } else {
                 log.error("Cannot retrieve thread id from URL $link")
+                dataTransaction.saveLog(
+                    LogEntry(
+                        type = LogEntry.Type.SCAN,
+                        status = LogEntry.Status.ERROR,
+                        message = "Invalid link $link, link is missing the threadId"
+                    )
+                )
                 continue
             }
         }

diff --git a/vripper-core/src/main/kotlin/me/vripper/services/DataTransaction.kt b/vripper-core/src/main/kotlin/me/vripper/services/DataTransaction.kt
@@ -53,6 +53,13 @@ class DataTransaction(
         }
     }
 
+    fun update(thread: Thread) {
+        transaction { threadRepository.update(thread) }
+        coroutineScope.launch {
+            eventBus.publishEvent(ThreadUpdateEvent(thread))
+        }
+    }
+
     fun updateImages(images: List<Image>) {
         transaction { imageRepository.update(images) }
         coroutineScope.launch {

diff --git a/vripper-core/src/main/kotlin/me/vripper/services/ThreadCacheService.kt b/vripper-core/src/main/kotlin/me/vripper/services/ThreadCacheService.kt
@@ -36,4 +36,8 @@ class ThreadCacheService(val eventBus: EventBus) {
     operator fun get(threadId: Long): ThreadItem {
         return cache[threadId] ?: throw NoSuchElementException("$threadId does not exist")
     }
+
+    fun getIfPresent(threadId: Long): ThreadItem? {
+        return cache.getIfPresent(threadId)
+    }
 }
diff --git a/vripper-core/src/main/kotlin/me/vripper/tasks/AddPostRunnable.kt b/vripper-core/src/main/kotlin/me/vripper/tasks/AddPostRunnable.kt
@@ -1,33 +1,23 @@
 package me.vripper.tasks
 
 import me.vripper.download.DownloadService
-import me.vripper.entities.LogEntry.Status.*
-import me.vripper.exception.DownloadException
-import me.vripper.exception.PostParseException
 import me.vripper.model.PostItem
 import me.vripper.model.ThreadPostId
-import me.vripper.parser.ThreadLookupAPIResponseHandler
-import me.vripper.services.*
+import me.vripper.parser.PostLookupAPIParser
+import me.vripper.services.DataTransaction
+import me.vripper.services.MetadataService
+import me.vripper.services.SettingsService
+import me.vripper.services.ThreadCacheService
 import me.vripper.utilities.Tasks
-import net.jodah.failsafe.Failsafe
-import net.jodah.failsafe.function.CheckedSupplier
-import org.apache.hc.client5.http.classic.HttpClient
-import org.apache.hc.client5.http.classic.methods.HttpGet
-import org.apache.hc.core5.net.URIBuilder
 import org.koin.core.component.KoinComponent
 import org.koin.core.component.inject
-import java.util.*
-import javax.xml.parsers.SAXParserFactory
 
 class AddPostRunnable(private val items: List<ThreadPostId>) : KoinComponent, Runnable {
     private val log by me.vripper.delegate.LoggerDelegate()
     private val dataTransaction: DataTransaction by inject()
     private val settingsService: SettingsService by inject()
-    private val vgAuthService: VGAuthService by inject()
-    private val httpService: HTTPService by inject()
-    private val retryPolicyService: RetryPolicyService by inject()
     private val downloadService: DownloadService by inject()
-    private val cacheService: ThreadCacheService by inject()
+    private val threadCacheService: ThreadCacheService by inject()
     private val metadataService: MetadataService by inject()
 
     override fun run() {
@@ -40,9 +30,12 @@ class AddPostRunnable(private val items: List<ThreadPostId>) : KoinComponent, Ru
                     continue
                 }
 
-                val threadItem = cacheService[threadId]
+                val threadItem = threadCacheService.getIfPresent(threadId)
                 val postItem: PostItem =
-                    threadItem.postItemList.find { it.postId == postId } ?: parse(postId, threadId)
+                    threadItem?.postItemList?.find { it.postId == postId } ?: PostLookupAPIParser(
+                        threadId,
+                        postId
+                    ).parse()
                 toProcess.add(postItem)
             }
 
@@ -61,42 +54,4 @@ class AddPostRunnable(private val items: List<ThreadPostId>) : KoinComponent, Ru
             Tasks.decrement()
         }
     }
-
-    @Throws(PostParseException::class)
-    fun parse(postId: Long, threadId: Long): PostItem {
-        log.debug("Parsing post $postId")
-        val httpGet =
-            HttpGet(URIBuilder("${settingsService.settings.viperSettings.host}/vr.php").also {
-                it.setParameter(
-                    "p", postId.toString()
-                )
-            }.build())
-        val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
-        log.debug("Requesting {}", httpGet)
-        return try {
-            Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>()).onFailure {
-                log.error(
-                    "parsing failed for thread $threadId, post $postId", it.failure
-                )
-            }.get(CheckedSupplier {
-                val connection: HttpClient = httpService.client
-                connection.execute(
-                    httpGet, vgAuthService.context
-                ) { response ->
-                    if (response.code / 100 != 2) {
-                        throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
-                    }
-                    factory.newSAXParser()
-                        .parse(response.entity.content, threadLookupAPIResponseHandler)
-                    threadLookupAPIResponseHandler.result.postItemList.first()
-                }
-            })
-        } catch (e: Exception) {
-            throw PostParseException(e)
-        }
-    }
-
-    companion object {
-        private val factory = SAXParserFactory.newInstance()
-    }
 }