yes parallell

NDLANO · Nov 15, 2023 · 8e821d5 · 8e821d5
1 parent 9eb9502
commit 8e821d5
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 21 deletions.
diff --git a/search-api/src/main/scala/no/ndla/searchapi/integration/TaxonomyApiClient.scala b/search-api/src/main/scala/no/ndla/searchapi/integration/TaxonomyApiClient.scala
@@ -117,28 +117,26 @@ trait TaxonomyApiClient {
           ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(numThreads))
 
         val pages = pageRange.map(pageNum =>
-          fetchPage(params :+ ("page" -> s"$pageNum")).map(page =>
-            page.results.foreach(n =>
-              n.contentUri.foreach(uri => {
-                val pub        = if (published) "published" else "draft"
-                val path       = s"$tmpDir/${uri}_$pub.json"
-                val f          = new java.io.File(path)
-                val fileExists = f.exists()
-                val stream     = new FileOutputStream(f, true)
-
-                if (fileExists) stream.write('\n')
-                val json = Serialization.write(n)
-                stream.write(json.getBytes)
-                stream.flush()
-                stream.close()
-              })
+          Future {
+            fetchPage(params :+ ("page" -> s"$pageNum")).map(page =>
+              page.results.foreach(n =>
+                n.contentUri.foreach(uri => {
+                  val pub    = if (published) "published" else "draft"
+                  val path   = s"$tmpDir/${uri}_$pub.json"
+                  val f      = new java.io.File(path)
+                  val stream = new FileOutputStream(f, true)
+                  val json = Serialization.write(n)
+                  stream.write(json.getBytes :+ '\n'.toByte)
+                  stream.flush()
+                  stream.close()
+                })
+              )
             )
-          )
+          }
         )
-        pages.toList.sequence.map(_ => ())
-//        val mergedFuture = Future.sequence(pages)
-//        val awaited      = Await.result(mergedFuture, timeoutSeconds)
-//        awaited.toList.sequence.map(_ => ())
+        val mergedFuture = Future.sequence(pages)
+        val awaited      = Await.result(mergedFuture, timeoutSeconds)
+        awaited.toList.sequence.map(_ => ())
       })
     }
   }

diff --git a/search-api/src/main/scala/no/ndla/searchapi/model/taxonomy/TaxonomyBundle.scala b/search-api/src/main/scala/no/ndla/searchapi/model/taxonomy/TaxonomyBundle.scala
@@ -37,7 +37,7 @@ case class TmpNodes(tmpDir: String, isPublished: Boolean) extends NodeStorage wi
     val f    = new java.io.File(path)
     if (f.exists()) {
       val stream = new FileInputStream(f)
-      val strs   = new String(stream.readAllBytes()).split("\n").toList
+      val strs   = new String(stream.readAllBytes()).split("\n").filter(_.nonEmpty).toList
       val nodes = strs.traverse(str => {
         Try(Serialization.read[Node](str)(TaxonomyBundle.formats, implicitly[Manifest[Node]]))
       })