Merge branch 'release/6.0.0'

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 100

.gitignore

@@ -15,3 +15,7 @@ project/plugins/project/
 # Scala-IDE specific
 .scala_dependencies
 .worksheet
+.bloop/
+.bsp/
+.metals/
+project/project/

CHANGELOG.md

@@ -1,3 +1,11 @@
+## 6.0.0
+
+- Upgrades the GeoTrellis library to `3.7.0`, the latest at this time
+- Upgrade Scala to `2.13`, sbt to `1.9.7`
+- Switch to using `build.sbt` instead of `build.scala`
+- Add `scripts/test` to run black-box tests on running service
+- **NOTE:** This is a major version upgrade because of the significant GeoTrellis, Scala, and SBT upgrades, but there are no changes for the service API itself
+
 ## 5.4.0
 
 - Add printing environment variables on startup to better observe the runtime

api/src/main/scala/ErrorHandler.scala

@@ -10,11 +10,11 @@ case class MissingTargetRasterException() extends GeoprocessingException()
 case class MissingVectorCRSException() extends GeoprocessingException()
 case class MissingVectorException() extends GeoprocessingException()
 case class MissingStreamLinesException() extends GeoprocessingException()
-case class InvalidOperationException(val message: String) extends GeoprocessingException()
-case class UnknownCRSException(val crs: String) extends GeoprocessingException()
+case class InvalidOperationException(message: String) extends GeoprocessingException()
+case class UnknownCRSException(crs: String) extends GeoprocessingException()
 
 trait ErrorHandler {
-  val geoprocessingExceptionHandler = ExceptionHandler {
+  val geoprocessingExceptionHandler: ExceptionHandler = ExceptionHandler {
     case InvalidOperationException(e) => {
       println(s"Invalid operation type: $e")
       complete(HttpResponse(BadRequest, entity = e))

api/src/main/scala/Geoprocessing.scala

@@ -3,15 +3,15 @@ package org.wikiwatershed.mmw.geoprocessing
 import java.util.concurrent.atomic.{LongAdder, DoubleAdder, DoubleAccumulator}
 import scala.concurrent.Future
 import scala.concurrent.ExecutionContext.Implicits.global
+import scala.collection.parallel.CollectionConverters._
 
 import collection.concurrent.TrieMap
 
+import geotrellis.layer._
 import geotrellis.raster._
 import geotrellis.raster.rasterize._
 import geotrellis.vector._
 
-import geotrellis.spark._
-
 import cats.implicits._
 
 trait Geoprocessing extends Utils {
@@ -53,7 +53,7 @@ trait Geoprocessing extends Utils {
         } yield {
           val results = op.name match {
             case "RasterGroupedCount" =>
-              rasterGroupedCount(layers, shape, opts).mapValues(_.toDouble)
+              rasterGroupedCount(layers, shape, opts).fmap(_.toDouble)
 
             case "RasterGroupedAverage" =>
               targetLayer match {
@@ -66,10 +66,9 @@ trait Geoprocessing extends Utils {
 
             case "RasterLinesJoin" =>
               input.streamLines match {
-                case Some(mls) => {
+                case Some(mls) =>
                   val lines = cropLinesToAOI(mls.map(parseMultiLineString), shape)
-                  rasterLinesJoin(layers, lines).mapValues(_.toDouble)
-                }
+                  rasterLinesJoin(layers, lines).fmap(_.toDouble)
                 case None =>
                   throw new MissingStreamLinesException
               }
@@ -81,7 +80,7 @@ trait Geoprocessing extends Utils {
     }
 
     val nested: Future[Map[HucID, Map[OperationID, Map[String, Double]]]] = tabular.map { list =>
-      list.groupBy { case (a, _, _) => a }.mapValues {
+      list.groupBy { case (a, _, _) => a }.fmap {
         grouped => grouped.map {case (_, b, c) => (b, c) }.toMap
       }
     }
@@ -210,7 +209,7 @@ trait Geoprocessing extends Utils {
     */
   private def rasterLinesJoin(
     rasterLayers: Seq[TileLayerCollection[SpatialKey]],
-    lines: Seq[MultiLine]
+    lines: Seq[MultiLineString]
   ): Map[String, Int] = {
     val metadata = rasterLayers.head.metadata
     val pixelGroups: TrieMap[(List[Int], TilePixel), Int] = TrieMap.empty
@@ -232,7 +231,7 @@ trait Geoprocessing extends Utils {
 
     pixelGroups
       .groupBy(_._1._1.toString)
-      .mapValues(_.size)
+      .fmap(_.size)
   }
 
   /**
@@ -309,7 +308,7 @@ trait Geoprocessing extends Utils {
             metadata.layout.tileRows)
 
         Rasterizer.foreachCellByMultiPolygon(multiPolygon, re, opts) { case (col, row) =>
-          val pixelKey: List[Int] = tiles.map(_.get(col, row)).toList
+          val pixelKey: List[Int] = tiles.map(_.get(col, row))
           val pixelValues = pixelGroups.getOrElseUpdate(pixelKey, init())
           val targetLayerData = targetTile.getDouble(col, row)
 
@@ -319,9 +318,11 @@ trait Geoprocessing extends Utils {
 
           update(targetLayerValue, pixelValues)
         }
+        case (_,_) => ()
       })
 
     pixelGroups
+      .view
       .mapValues { case (accumulator, counter) => accumulator.sum / counter.sum }
       .map { case (k, v) => k.toString -> v }
       .toMap
@@ -360,6 +361,7 @@ trait Geoprocessing extends Utils {
       })
 
     pixelGroups
+      .view
       .mapValues(_.sum().toInt)
       .map { case (k, v) => k.toString -> v}
       .toMap

api/src/main/scala/Utils.scala

@@ -5,23 +5,22 @@ import java.util.function.DoubleBinaryOperator
 import scala.concurrent.Future
 import scala.concurrent.ExecutionContext.Implicits.global
 
-import spray.json._
-
 import geotrellis.proj4.{CRS, ConusAlbers, LatLng, WebMercator}
 
+import geotrellis.layer._
 import geotrellis.raster._
 import geotrellis.raster.rasterize._
+import geotrellis.store._
+import geotrellis.store.s3._
 import geotrellis.vector._
 import geotrellis.vector.io._
-import geotrellis.spark._
-import geotrellis.spark.io._
-import geotrellis.spark.io.s3._
+import geotrellis.vector.io.json.GeoJson
 
 import com.typesafe.config.ConfigFactory
 
 trait Utils {
-  val s3bucket = ConfigFactory.load().getString("geoprocessing.s3bucket")
-  val baseLayerReader = S3CollectionLayerReader(s3bucket, "")
+  val s3bucket: String = ConfigFactory.load().getString("geoprocessing.s3bucket")
+  val baseLayerReader: S3CollectionLayerReader = S3CollectionLayerReader(s3bucket, "")
 
   /**
     * Given a zoom level & area of interest, transform a list of raster
@@ -68,7 +67,7 @@ trait Utils {
     val parseGeom =
       parseGeometry(_: String, getCRS(input.polygonCRS), getCRS(input.rasterCRS))
 
-    input.polygon.map { str => parseGeom(str).buffer(0).asMultiPolygon.get }
+    input.polygon.map { str => regularizeMultiPolygon(parseGeom(str)) }
       .unionGeometries
       .asMultiPolygon
       .get
@@ -85,7 +84,7 @@ trait Utils {
     val parseGeom =
       parseGeometry(_: String, getCRS(input.polygonCRS), getCRS(input.rasterCRS))
 
-    input.polygon.map { str => parseGeom(str).buffer(0).asMultiPolygon.get }
+    input.polygon.map { str => regularizeMultiPolygon(parseGeom(str)) }
   }
 
   /**
@@ -95,10 +94,7 @@ trait Utils {
     * @return         A MultiPolygon
     */
   def normalizeHuc(huc: HUC): MultiPolygon = {
-    parseGeometry(huc.shape, LatLng, ConusAlbers)
-      .buffer(0)
-      .asMultiPolygon
-      .get
+    regularizeMultiPolygon(parseGeometry(huc.shape, LatLng, ConusAlbers))
   }
 
   /**
@@ -131,13 +127,21 @@ trait Utils {
     * @return           A MultiPolygon
     */
   def parseGeometry(geoJson: String, srcCRS: CRS, destCRS: CRS): MultiPolygon = {
-    geoJson.parseJson.convertTo[Geometry] match {
+    GeoJson.parse[Geometry](geoJson) match {
       case p: Polygon => MultiPolygon(p.reproject(srcCRS, destCRS))
       case mp: MultiPolygon => mp.reproject(srcCRS, destCRS)
       case _ => MultiPolygon()
     }
   }
 
+  def regularizeMultiPolygon(mp: MultiPolygon): MultiPolygon = {
+    mp.buffer(0) match {
+      case p: Polygon if p.isEmpty => MultiPolygon()
+      case p: Polygon => MultiPolygon(Seq(p))
+      case mp: MultiPolygon => mp
+    }
+  }
+
   /**
     * Given an input vector along with a vectorCRS and rasterCRS, return a Seq
     * of MultiLines
@@ -151,7 +155,7 @@ trait Utils {
     vector: List[String],
     vectorCRS: String,
     rasterCRS: String
-  ): Seq[MultiLine] = {
+  ): Seq[MultiLineString] = {
     val parseVector =
       parseMultiLineString(_: String, getCRS(vectorCRS), getCRS(rasterCRS))
 
@@ -167,18 +171,18 @@ trait Utils {
     * @param   destCRS  The CRS that the outgoing geometry should be in
     * @return           A MultiLine
     */
-  def parseMultiLineString(geoJson: String, srcCRS: CRS, destCRS: CRS): MultiLine = {
-    geoJson.parseJson.convertTo[Geometry] match {
-      case l: Line => MultiLine(l.reproject(srcCRS, destCRS))
-      case ml: MultiLine => ml.reproject(srcCRS, destCRS)
-      case _ => MultiLine()
+  def parseMultiLineString(geoJson: String, srcCRS: CRS, destCRS: CRS): MultiLineString = {
+    GeoJson.parse[Geometry](geoJson) match {
+      case l: LineString => MultiLineString(l.reproject(srcCRS, destCRS))
+      case ml: MultiLineString => ml.reproject(srcCRS, destCRS)
+      case _ => MultiLineString()
     }
   }
 
   /**
     * Convenience flavor of the above with defaults
     */
-  def parseMultiLineString(geoJson: String): MultiLine =
+  def parseMultiLineString(geoJson: String): MultiLineString =
     parseMultiLineString(geoJson, LatLng, ConusAlbers)
 
   /**
@@ -189,8 +193,8 @@ trait Utils {
     * @param   aoi    Area of Interest
     * @return         A sequence of MultiLines that intersect with the Area of Interest
     */
-  def cropLinesToAOI(lines: Seq[MultiLine], aoi: MultiPolygon): Seq[MultiLine] = {
-    lines.flatMap(line => (line & aoi).asMultiLine)
+  def cropLinesToAOI(lines: Seq[MultiLineString], aoi: MultiPolygon): Seq[MultiLineString] = {
+    lines.flatMap(line => (line & aoi).asMultiLineString)
   }
 
   /**
@@ -205,7 +209,7 @@ trait Utils {
     case "LatLng" => LatLng
     case "WebMercator" => WebMercator
     case "ConusAlbers" => ConusAlbers
-    case s: String => throw new UnknownCRSException(s)
+    case s: String => throw UnknownCRSException(s)
   }
 
   /**
@@ -243,7 +247,7 @@ trait Utils {
       .where(Intersects(shape))
       .result
 
-  def printConfiguration() = {
+  def printConfiguration(): Unit = {
     val config = ConfigFactory.load()
 
     val host = config.getString("geoprocessing.hostname")

api/src/main/scala/WebServer.scala

@@ -49,16 +49,16 @@ case class MultiInput (
 )
 
 object PostRequestProtocol extends DefaultJsonProtocol {
-  implicit val inputFormat = jsonFormat10(InputData)
-  implicit val postFormat = jsonFormat1(PostRequest)
-  implicit val resultFormat = jsonFormat1(ResultInt)
-  implicit val resultDoubleFormat = jsonFormat1(ResultDouble)
-  implicit val resultSummaryFormat = jsonFormat1(ResultSummary)
-  implicit val resultManyIntFormat = jsonFormat1(ResultManyInt)
+  implicit val inputFormat: RootJsonFormat[InputData] = jsonFormat10(InputData)
+  implicit val postFormat: RootJsonFormat[PostRequest] = jsonFormat1(PostRequest)
+  implicit val resultFormat: RootJsonFormat[ResultInt] = jsonFormat1(ResultInt)
+  implicit val resultDoubleFormat: RootJsonFormat[ResultDouble] = jsonFormat1(ResultDouble)
+  implicit val resultSummaryFormat: RootJsonFormat[ResultSummary] = jsonFormat1(ResultSummary)
+  implicit val resultManyIntFormat: RootJsonFormat[ResultManyInt] = jsonFormat1(ResultManyInt)
 
-  implicit val hucFormat = jsonFormat2(HUC)
-  implicit val operationFormat = jsonFormat5(Operation)
-  implicit val multiInputFormat = jsonFormat3(MultiInput)
+  implicit val hucFormat: RootJsonFormat[HUC] = jsonFormat2(HUC)
+  implicit val operationFormat: RootJsonFormat[Operation] = jsonFormat5(Operation)
+  implicit val multiInputFormat: RootJsonFormat[MultiInput] = jsonFormat3(MultiInput)
 }
 
 object WebServer extends HttpApp with App with LazyLogging with Geoprocessing with ErrorHandler {
@@ -88,7 +88,7 @@ object WebServer extends HttpApp with App with LazyLogging with Geoprocessing wi
                 complete(getRasterSummary(data.input))
               case _ => {
                 val message = s"Unknown operationType: ${data.input.operationType}"
-                throw new InvalidOperationException(message)
+                throw InvalidOperationException(message)
               }
             }
           }

api/src/main/scala/package.scala

@@ -1,6 +1,6 @@
 package org.wikiwatershed.mmw
 
-import geotrellis.spark._
+import geotrellis.layer._
 
 package object geoprocessing {
   type HucID = String