fix(service-food): fix PR issues, improve log, use URI instead of URL

service-food/build.gradle

@@ -16,6 +16,7 @@ apply plugin: 'kotlin'
 
 repositories {
     jcenter()
+    mavenCentral()
     maven { url "https://dl.bintray.com/hpi/hpi-cloud-mvn" }
 }
 
@@ -32,6 +33,3 @@ application {
 
     mainClassName = "de.hpi.cloud.food.FoodServiceKt"
 }
-repositories {
-    mavenCentral()
-}

service-food/src/main/kotlin/de/hpi/cloud/food/crawler/Main.kt

@@ -12,7 +12,6 @@ import de.hpi.cloud.food.entities.Counter
 import de.hpi.cloud.food.entities.Label
 import de.hpi.cloud.food.entities.MenuItem
 import java.io.IOException
-import java.net.URL
 import java.time.LocalDate
 import java.util.Locale.GERMAN
 
@@ -31,11 +30,10 @@ fun main(args: Array<String>) {
 
     println("Starting $NAME")
     withBucket("food") { bucket ->
-        // TODO: clear previous data
         KNOWN_CANTEENS
             .map { OpenMensaCrawler(it) }
             .forEach { crawler ->
-                println("Starting crawler $CRAWLER_INFO - ${crawler.canteenData}")
+                println("\nStarting crawler $CRAWLER_INFO - ${crawler.canteenData}")
                 println("Using User-Agent=\"$USER_AGENT\"")
 
                 try {
@@ -44,37 +42,39 @@ fun main(args: Array<String>) {
                         else crawler.queryDays()
                             .filter { it.isOpen }
                             .map { it.date }
-                    days.forEach { date ->
-                        print("Date $date: ")
+                    if (days.isEmpty())
+                        println("No days available")
+                    else days.forEach { date ->
+                        println("\nQuerying date $date:")
 
                         val meals = crawler.queryMeals(date)
                         meals.forEach { meal ->
-                            val counterId = meal.counter ?: "unbekannt"
-
                             // populate counters
-                            // TODO: remove later
-                            val counter = Counter(
-                                crawler.canteenData.id,
-                                title = counterId.l10n(GERMAN),
-                                iconUrl = URL("")
-                            )
-                            bucket.upsert(counter.createNewWrapper(FOOD_CONTEXT, Id(counterId)))
+                            // TODO: remove when we've gathered all counters in the database
+                            val counterId = meal.counterName?.let { Id<Counter>(it) }
+                            val counter = meal.counterName?.let { name ->
+                                Counter(
+                                    crawler.canteenData.id,
+                                    title = name.l10n(GERMAN)
+                                )
+                            }
+                            if (counter != null)
+                                bucket.upsert(counter.createNewWrapper(FOOD_CONTEXT, counterId!!))
 
                             val menuItem = MenuItem(
                                 openMensaId = meal.openMensaId.toString(),
                                 date = meal.date,
                                 restaurantId = crawler.canteenData.id,
                                 offerTitle = meal.offerName.l10n(GERMAN),
                                 title = meal.title.l10n(GERMAN),
-                                counterId = Id(meal.counter ?: "null"),
+                                counterId = counter?.let { counterId },
                                 labelIds = meal.labelIds.map { Id<Label>(it) },
                                 prices = meal.prices.mapValues { (_, price) -> Money.eur(price) }
                             )
-                            println(meal)
-                            println(menuItem)
+                            println("- $meal\n$menuItem\n")
                             bucket.upsert(menuItem.createNewWrapper(FOOD_CONTEXT, Id(meal.id)))
                         }
-                        println("${meals.size} meals")
+                        println("${meals.size} meals found for $date")
                     }
                 } catch (ex: IOException) {
                     ex.printStackTrace()

service-food/src/main/kotlin/de/hpi/cloud/food/crawler/OpenMensaCrawler.kt

@@ -14,9 +14,8 @@ class OpenMensaCrawler(
     companion object {
         private const val OPENMENSA_API_VERSION = "v2"
 
-        val BASE_URI: URI = URI("https://openmensa.org/api/$OPENMENSA_API_VERSION/")
+        val BASE_URI = URI("https://openmensa.org/api/$OPENMENSA_API_VERSION/")
         val DATE_FORMAT_ISO8601 = DateTimeFormatter.ISO_LOCAL_DATE!!
-        val DATE_FORMAT_COMPACT = DateTimeFormatter.ofPattern("yyyyMMdd")!!
     }
 
     private val klaxon = Klaxon()
@@ -28,7 +27,7 @@ class OpenMensaCrawler(
     private fun streamJsonApi(query: String) = BASE_URI.resolve(query).toURL()
         .openStreamWith(
             "User-agent" to USER_AGENT,
-            "Accept" to "application/json, text/plain"
+            "Accept" to "application/json"
         )
 
     fun queryDays() = streamJsonApi(daysQuery())

service-food/src/main/kotlin/de/hpi/cloud/food/crawler/ParsedMensaMeal.kt

@@ -3,33 +3,36 @@ package de.hpi.cloud.food.crawler
 import de.hpi.cloud.food.crawler.openmensa.CanteenData
 import de.hpi.cloud.food.crawler.openmensa.OpenMensaMeal
 import java.time.LocalDate
+import java.time.format.DateTimeFormatter
 
 data class ParsedMensaMeal(
     private val canteenData: CanteenData,
     val openMensaMeal: OpenMensaMeal,
     val date: LocalDate,
-    val counter: String?,
+    val counterName: String?,
     val title: String,
     val offerName: String,
     val labelIds: List<String>,
     val uniqueIdSuffix: Int? = null
 ) {
     companion object {
+        val DATE_FORMAT_COMPACT = DateTimeFormatter.ofPattern("yyyyMMdd")!!
+
         fun parseMeals(canteenData: CanteenData, date: LocalDate, openMensaMeal: OpenMensaMeal) =
             ParsedMensaMeal(
                 canteenData = canteenData,
                 openMensaMeal = openMensaMeal,
                 date = date,
-                counter = canteenData.findCounter(openMensaMeal),
+                counterName = canteenData.findCounter(openMensaMeal),
                 title = openMensaMeal.name.replace("\n", "").trim(),
                 offerName = canteenData.findOfferName(openMensaMeal),
                 labelIds = canteenData.findLabels(openMensaMeal)
             ).let { listOf(it) }
     }
 
     val id get() = canteenData.id.value +
-                "-${date.format(OpenMensaCrawler.DATE_FORMAT_COMPACT)}" +
-                "-${counter ?: openMensaId}" +
+                "-${date.format(DATE_FORMAT_COMPACT)}" +
+                "-${counterName ?: openMensaId}" +
                 (uniqueIdSuffix?.let { "_$it" } ?: "")
     val openMensaId get() = openMensaMeal.id
 

service-food/src/main/kotlin/de/hpi/cloud/food/entities/Counter.kt

@@ -5,17 +5,17 @@ import de.hpi.cloud.common.entity.Entity
 import de.hpi.cloud.common.entity.Id
 import de.hpi.cloud.common.entity.Wrapper
 import de.hpi.cloud.common.protobuf.builder
-import de.hpi.cloud.common.serializers.json.UrlSerializer
+import de.hpi.cloud.common.serializers.json.UriSerializer
 import de.hpi.cloud.common.types.L10n
 import kotlinx.serialization.Serializable
-import java.net.URL
+import java.net.URI
 import de.hpi.cloud.food.v1test.Counter as ProtoCounter
 
 @Serializable
 data class Counter(
     val restaurantId: Id<Restaurant>,
     val title: L10n<String>,
-    val iconUrl: @Serializable(UrlSerializer::class) URL
+    val iconUri: @Serializable(UriSerializer::class) URI? = null
 ) : Entity<Counter>() {
     companion object : Entity.Companion<Counter>("counter")
 
@@ -27,7 +27,7 @@ data class Counter(
             ProtoCounter.newBuilder().builder(entity) {
                 restaurantId = it.restaurantId.value
                 title = it.title[context]
-                icon = it.iconUrl.toString()
+                icon = it.iconUri.toString()
             }
     }
 }

service-food/src/main/kotlin/de/hpi/cloud/food/entities/Label.kt

@@ -6,19 +6,19 @@ import de.hpi.cloud.common.entity.Id
 import de.hpi.cloud.common.entity.Wrapper
 import de.hpi.cloud.common.entity.asId
 import de.hpi.cloud.common.protobuf.builder
-import de.hpi.cloud.common.serializers.json.UrlSerializer
+import de.hpi.cloud.common.serializers.json.UriSerializer
 import de.hpi.cloud.common.types.L10n
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.Transient
-import java.net.URL
+import java.net.URI
 import java.util.Locale.GERMAN
 import de.hpi.cloud.food.v1test.Label as ProtoLabel
 
 @Serializable
 data class Label(
     val title: L10n<String>,
-    val iconUrl: @Serializable(UrlSerializer::class) URL,
-    val aliases: Set<String> = setOf()
+    val aliases: Set<String> = setOf(),
+    val iconUri: @Serializable(UriSerializer::class) URI? = null
 ) : Entity<Label>() {
     companion object : Entity.Companion<Label>("label")
 
@@ -32,7 +32,7 @@ data class Label(
         override fun toProtoBuilder(entity: Label, context: Context): ProtoLabel.Builder =
             ProtoLabel.newBuilder().builder(entity) {
                 title = it.title[context]
-                icon = it.iconUrl.toString()
+                icon = it.iconUri.toString()
             }
     }
 
@@ -45,4 +45,8 @@ data class Label(
     }
 }
 
-fun Wrapper<Label>.toProto(context: Context): ProtoLabel = Label.ProtoSerializer.toProto(this, context)
+fun ProtoLabel.parse(context: Context): Label =
+    Label.ProtoSerializer.fromProto(this, context)
+
+fun Wrapper<Label>.toProto(context: Context): ProtoLabel =
+    Label.ProtoSerializer.toProto(this, context)

service-food/src/main/kotlin/de/hpi/cloud/food/entities/MenuItem.kt

@@ -25,7 +25,7 @@ data class MenuItem(
     val restaurantId: Id<Restaurant>,
     val offerTitle: L10n<String>,
     val title: L10n<String>,
-    val counterId: Id<Counter>,
+    val counterId: Id<Counter>?,
     val labelIds: List<Id<Label>>,
     val prices: Map<String, Money>
 ) : Entity<MenuItem>() {
@@ -50,8 +50,8 @@ data class MenuItem(
                 restaurantId = it.restaurantId.value
                 offerTitle = it.offerTitle[context]
                 title = it.title[context]
-                counterId = it.counterId.value
-                addAllLabelIds(it.labelIds.map { l -> l.value })
+                counterId = it.counterId?.value
+                addAllLabelIds(it.labelIds.map { label -> label.value })
                 putAllPrices(it.prices.mapValues { (_, money) -> money.toProto(context) })
             }
     }

service-food/src/main/kotlin/de/hpi/cloud/food/fixtures/Labels.kt

@@ -6,69 +6,69 @@ import de.hpi.cloud.common.entity.Wrapper
 import de.hpi.cloud.common.types.L10n
 import de.hpi.cloud.food.FOOD_CONTEXT
 import de.hpi.cloud.food.entities.Label
-import java.net.URL
+import java.net.URI
 
 val labelFixture: List<Label> = listOf(
     Label(
         title = L10n.from(de = "Alkohol", en = "Alcohol"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_alkohol_a.png")
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_alkohol_a.png")
     ),
     Label(
         title = L10n.from(de = "Knoblauch", en = "Garlic"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_knoblauch_k_dick.png")
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_knoblauch_k_dick.png")
     ),
     Label(
         title = L10n.from(de = "Schwein", en = "Pork"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_schwein_s.png"),
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_schwein_s.png"),
         aliases = setOf("Schweinefleisch")
     ),
     Label(
         title = L10n.from(de = "Rind", en = "Beef"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_rind_r.png"),
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_rind_r.png"),
         aliases = setOf("Rindfleisch")
     ),
     Label(
         title = L10n.from(de = "Lamm", en = "Lamb"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_lamm_l.png")
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_lamm_l.png")
     ),
     Label(
         title = L10n.from(de = "Fisch", en = "Fish"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_fisch_f.png")
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_fisch_f.png")
     ),
     Label(
         title = L10n.from(de = "Geflügel", en = "Poultry"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_hahn_g.png"),
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_hahn_g.png"),
         aliases = setOf("Gefluegel", "Huhn", "Hahn", "Hühnchen", "Huehnchen", "Ente", "Gans", "Truthahn")
     ),
     Label(
         title = L10n.from(de = "Wild", en = "Venison"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_wild_h.png"),
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_wild_h.png"),
         aliases = setOf("Wildfleisch")
     ),
     Label(
         title = L10n.from(de = "Vegan", en = "Vegan"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_vegan_w.png")
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_vegan_w.png")
     ),
     Label(
         title = L10n.from(de = "Vegetarisch", en = "Vegetarian"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_vegetarisch_v.png")
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_vegetarisch_v.png")
     ),
     Label(
         title = L10n.from(de = "Vital", en = "Vital"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_vital_m.png")
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_vital_m.png")
     ),
     Label(
         title = L10n.from(de = "Regional", en = "Regional"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_regional.png")
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_regional.png")
     ),
     Label(
         title = L10n.from(de = "Draußen", en = "Outdoor"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_outdoor_o.png"),
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_outdoor_o.png"),
         aliases = setOf("Draussen")
     ),
     Label(
-        title = L10n.from(de = "Foottruck", en = "Foottruck"),
-        iconUrl = URL("https://xml.stw-potsdam.de/images/icons/su_foottruck_o.png")
+        title = L10n.from(de = "Foodtruck", en = "Food Truck"),
+        iconUri = URI("https://xml.stw-potsdam.de/images/icons/su_foottruck_o.png")
     )
 )