diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/creo/CreoS3Utils.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/creo/CreoS3Utils.scala index 560832192..ca8f16b44 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/creo/CreoS3Utils.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/creo/CreoS3Utils.scala @@ -1,19 +1,29 @@ package org.openeo.geotrellis.creo +import geotrellis.store.s3.AmazonS3URI +import org.apache.commons.io.FileUtils import org.openeo.geotrelliss3.S3Utils +import org.slf4j.LoggerFactory import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} import software.amazon.awssdk.awscore.retry.conditions.RetryOnErrorCodeCondition import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration import software.amazon.awssdk.core.retry.RetryPolicy import software.amazon.awssdk.core.retry.backoff.FullJitterBackoffStrategy import software.amazon.awssdk.core.retry.conditions.{OrRetryCondition, RetryCondition} +import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.model._ import software.amazon.awssdk.services.s3.{S3AsyncClient, S3Client, S3Configuration} import java.net.URI +import java.nio.file.{FileAlreadyExistsException, Files, Path} import java.time.Duration +import scala.collection.JavaConverters._ +import scala.collection.immutable.Iterable +import scala.util.control.Breaks.{break, breakable} object CreoS3Utils { + private val logger = LoggerFactory.getLogger(getClass) private val cloudFerroRegion: Region = Region.of("RegionOne") @@ -66,8 +76,174 @@ object CreoS3Utils { overrideConfig } - def deleteCreoSubFolder(bucket_name: String, subfolder: String) = { + //noinspection ScalaWeakerAccess + def deleteCreoSubFolder(bucket_name: String, subfolder: String): Unit = { val s3Client = getCreoS3Client() S3Utils.deleteSubFolder(s3Client, bucket_name, subfolder) } + + def isS3(path: String): Boolean = { + path.toLowerCase.startsWith("s3:/") + } + + private def toAmazonS3URI(path: String): AmazonS3URI = { + val correctS3Path = path.replaceFirst("(?i)s3:/(?!/)", "s3://") + new AmazonS3URI(correctS3Path) + } + + // In the following functions an asset path could be a local path or an S3 path. + + /** + * S3 does not have folders, so we interpret the path as a prefix. + */ + def assetDeleteFolders(paths: Iterable[String]): Unit = { + for (path <- paths) { + if (isS3(path)) { + val s3Uri = toAmazonS3URI(path) + deleteCreoSubFolder(s3Uri.getBucket, s3Uri.getKey) + } else { + val p = Path.of(path) + if (Files.exists(p)) { + if (Files.isDirectory(p)) { + FileUtils.deleteDirectory(p.toFile) + } else { + throw new IllegalArgumentException(f"Can only delete directory here: $path") + } + } + } + } + } + + def assetDelete(path: String): Unit = { + if (isS3(path)) { + val s3Uri = toAmazonS3URI(path) + val keys = Seq(path) + val deleteObjectsRequest = DeleteObjectsRequest.builder + .bucket(s3Uri.getBucket) + .delete(Delete.builder.objects(keys.map(key => ObjectIdentifier.builder.key(key).build).asJavaCollection).build) + .build + getCreoS3Client().deleteObjects(deleteObjectsRequest) + } else { + val p = Path.of(path) + if (Files.isDirectory(p)) { + throw new IllegalArgumentException(f"Cannot delete directory like this: $path") + } else { + Files.deleteIfExists(p) + } + } + } + + def asseetPathListDirectChildren(path: String): Set[String] = { + if (isS3(path)) { + val s3Uri = toAmazonS3URI(path) + val listObjectsRequest = ListObjectsRequest.builder + .bucket(s3Uri.getBucket) + .prefix(s3Uri.getKey) + .build + val listObjectsResponse = getCreoS3Client().listObjects(listObjectsRequest) + listObjectsResponse.contents.asScala.map(o => f"s3://${s3Uri.getBucket}/${o.key}").toSet + } else { + Files.list(Path.of(path)).toArray.map(_.toString).toSet + } + } + + def assetExists(path: String): Boolean = { + if (isS3(path)) { + try { + // https://stackoverflow.com/a/56038360/1448736 + val s3Uri = toAmazonS3URI(path) + val objectRequest = HeadObjectRequest.builder + .bucket(s3Uri.getBucket) + .key(s3Uri.getKey) + .build + getCreoS3Client().headObject(objectRequest) + true + } catch { + case _: NoSuchKeyException => false + } + } else { + Files.exists(Path.of(path)) + } + } + + def copyAsset(pathOrigin: String, pathDestination: String): Unit = { + if (isS3(pathOrigin) && isS3(pathDestination)) { + val s3UriOrigin = toAmazonS3URI(pathOrigin) + val s3UriDestination = toAmazonS3URI(pathDestination) + val copyRequest = CopyObjectRequest.builder + .sourceBucket(s3UriOrigin.getBucket) + .sourceKey(s3UriOrigin.getKey) + .destinationBucket(s3UriDestination.getBucket) + .destinationKey(s3UriDestination.getKey) + .build + getCreoS3Client().copyObject(copyRequest) + } else if (!isS3(pathOrigin) && !isS3(pathDestination)) { + Files.copy(Path.of(pathOrigin), Path.of(pathDestination)) + } else if (!isS3(pathOrigin) && isS3(pathDestination)) { + uploadToS3(Path.of(pathOrigin), pathDestination) + } else if (isS3(pathOrigin) && !isS3(pathDestination)) { + // TODO: Download + throw new IllegalArgumentException(f"S3->local not supported here yet ($pathOrigin, $pathDestination)") + } else { + throw new IllegalArgumentException(f"Should be impossible to get here ($pathOrigin, $pathDestination)") + } + } + + def moveAsset(pathOrigin: String, pathDestination: String): Unit = { + // This could be optimized using move when on file system. + copyAsset(pathOrigin, pathDestination) + assetDelete(pathOrigin) + } + + def waitTillPathAvailable(path: Path): Unit = { + var retry = 0 + val maxTries = 20 + while (!assetExists(path.toString)) { + if (retry < maxTries) { + retry += 1 + val seconds = 5 + logger.info(f"Waiting for path to be available. Try $retry/$maxTries (sleep:$seconds seconds): $path") + Thread.sleep(seconds * 1000) + } else { + logger.warn(f"Path is not available after $maxTries tries: $path") + // Throw error instead? + return + } + } + } + + def moveOverwriteWithRetries(oldPath: String, newPath: String): Unit = { + var try_count = 1 + breakable { + while (true) { + try { + if (assetExists(newPath)) { + // It might be a partial result of a previous failing task. + logger.info(f"Will replace $newPath. (try $try_count)") + assetDelete(newPath) + } + moveAsset(oldPath, newPath) + break + } catch { + case e: Exception => + // Here if another executor wrote the file between the delete and the move statement. + try_count += 1 + if (try_count > 5) { + throw e + } + } + } + } + } + + def uploadToS3(localFile: Path, s3Path: String) = { + val s3Uri = toAmazonS3URI(s3Path) + val objectRequest = PutObjectRequest.builder + .bucket(s3Uri.getBucket) + .key(s3Uri.getKey) + .build + + getCreoS3Client().putObject(objectRequest, RequestBody.fromFile(localFile)) + s3Path + } } diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/package.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/package.scala index c3f5c8ae6..ea5771a8e 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/package.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/package.scala @@ -15,7 +15,7 @@ import geotrellis.spark.pyramid.Pyramid import geotrellis.store.s3._ import geotrellis.util._ import geotrellis.vector.{ProjectedExtent, _} -import org.apache.commons.io.FilenameUtils +import org.apache.commons.io.{FileUtils, FilenameUtils} import org.apache.spark.SparkContext import org.apache.spark.broadcast.Broadcast import org.apache.spark.rdd.RDD @@ -27,20 +27,16 @@ import org.openeo.geotrellis.netcdf.NetCDFRDDWriter.fixedTimeOffset import org.openeo.geotrellis.stac.STACItem import org.openeo.geotrellis.tile_grid.TileGrid import org.slf4j.LoggerFactory -import software.amazon.awssdk.core.sync.RequestBody -import software.amazon.awssdk.services.s3.model.PutObjectRequest import spire.math.Integral import spire.syntax.cfor.cfor import java.io.IOException -import java.nio.channels.FileChannel -import java.nio.file.{FileAlreadyExistsException, Files, NoSuchFileException, Path, Paths} +import java.nio.file.{Files, Path, Paths} import java.time.Duration import java.time.format.DateTimeFormatter import java.util.{ArrayList, Collections, Map, List => JList} import scala.collection.JavaConverters._ import scala.reflect._ -import scala.util.control.Breaks.{break, breakable} package object geotiff { @@ -105,6 +101,46 @@ package object geotiff { ret.map(t => (t._1, t._2, t._3)).asJava } + private val executorAttemptDirectoryPrefix = "executorAttemptDirectory" + + private def createExecutorAttemptDirectory(parentDirectory: String): Path = { + createExecutorAttemptDirectory(Path.of(parentDirectory)) + } + + private def createExecutorAttemptDirectory(parentDirectory: Path): Path = { + // Multiple executors with the same task can run at the same time. + // Writing their output to the same path would create a racing condition. + // Let's provide a unique directory for each executor: + val rand = new java.security.SecureRandom().nextLong() + val uniqueFolderName = executorAttemptDirectoryPrefix + java.lang.Long.toUnsignedString(rand) + val executorAttemptDirectory = Paths.get(parentDirectory + "/" + uniqueFolderName) + if (!CreoS3Utils.isS3(parentDirectory.toString)) { + Files.createDirectories(executorAttemptDirectory) + } + executorAttemptDirectory + } + + private def moveFromExecutorAttemptDirectory(parentDirectory: Path, absoluteFilePath: String, fileExists: Boolean): Path = { + // Move output file to standard location. (On S3, a move is more a copy and delete): + val relativeFilePath = parentDirectory.relativize(Path.of(absoluteFilePath)).toString + if (!relativeFilePath.startsWith(executorAttemptDirectoryPrefix)) throw new Exception() + // Remove the executorAttemptDirectory part from the path: + val destinationPath = parentDirectory.resolve(relativeFilePath.substring(relativeFilePath.indexOf("/") + 1)) + if (fileExists) { + CreoS3Utils.waitTillPathAvailable(Path.of(absoluteFilePath)) + if (!CreoS3Utils.isS3(parentDirectory.toString)) { + Files.createDirectories(destinationPath.getParent) + } + CreoS3Utils.moveOverwriteWithRetries(absoluteFilePath, destinationPath.toString) + } + destinationPath + } + + private def cleanUpExecutorAttemptDirectory(parentDirectory: String): Unit = { + val list = CreoS3Utils.asseetPathListDirectChildren(parentDirectory).filter(_.contains(executorAttemptDirectoryPrefix)) + CreoS3Utils.assetDeleteFolders(list) + } + /** * Save temporal rdd, on the executors * @@ -137,7 +173,7 @@ package object geotiff { val bandSegmentCount = totalCols * totalRows val bandLabels = formatOptions.tags.bandTags.map(_("DESCRIPTION")) - preprocessedRdd.flatMap { case (key: SpaceTimeKey, multibandTile: MultibandTile) => + val res = preprocessedRdd.flatMap { case (key: SpaceTimeKey, multibandTile: MultibandTile) => var bandIndex = -1 //Warning: for deflate compression, the segmentcount and index is not really used, making it stateless. //Not sure how this works out for other types of compression!!! @@ -177,9 +213,12 @@ package object geotiff { val bandIndices = sequence.map(_._3).toSet.toList.asJava val segmentCount = bandSegmentCount * tiffBands - val absolutePath = Paths.get(path).resolve(filename) - absolutePath.toFile.getParentFile.mkdirs() - val thePath = absolutePath.toString + + // Each executor writes to a unique folder to avoid conflicts: + val executorAttemptDirectory = createExecutorAttemptDirectory(path) + val absoluteFilePath = executorAttemptDirectory.resolve(filename) + absoluteFilePath.toFile.getParentFile.mkdirs() + val thePath = absoluteFilePath.toString // filter band tags that match bandIndices val fo = formatOptions.deepClone() @@ -188,12 +227,19 @@ package object geotiff { .map { case (bandTags, _) => bandTags } fo.setBandTags(newBandTags) - val correctedPath = writeTiff(thePath, tiffs, gridBounds, croppedExtent, preprocessedRdd.metadata.crs, + val (correctedPath, fileExists) = writeTiff(thePath, tiffs, gridBounds, croppedExtent, preprocessedRdd.metadata.crs, tileLayout, compression, cellTypes.head, tiffBands, segmentCount, fo, ) - (correctedPath, timestamp, croppedExtent, bandIndices) - }.collect().toList.asJava + (correctedPath, fileExists, timestamp, croppedExtent, bandIndices) + }.collect().map { + case (absoluteFilePath, fileExists, timestamp, croppedExtent, bandIndices) => + val destinationPath = moveFromExecutorAttemptDirectory(Path.of(path), absoluteFilePath, fileExists) + (destinationPath.toString, timestamp, croppedExtent, bandIndices) + }.toList.asJava + cleanUpExecutorAttemptDirectory(path) + + res } @@ -242,10 +288,11 @@ package object geotiff { ((name, bandIndex), (key, t)) } } - rdd_per_band.groupByKey().map { case ((name, bandIndex), tiles) => + val res = rdd_per_band.groupByKey().map { case ((name, bandIndex), tiles) => val fixedPath = if (path.endsWith("out")) { - path.substring(0, path.length - 3) + name + val executorAttemptDirectory = createExecutorAttemptDirectory(path.substring(0, path.length - 3)) + executorAttemptDirectory + "/" + name } else { path @@ -263,7 +310,23 @@ package object geotiff { (stitchAndWriteToTiff(tiles, fixedPath, layout, crs, extent, None, None, compression, Some(fo)), Collections.singletonList(bandIndex)) - }.collect().toList.sortBy(_._1).asJava + }.collect().map { + case ((absoluteFilePath, fileExists), bandIndices) => + if (path.endsWith("out")) { + val beforeOut = path.substring(0, path.length - "out".length) + val destinationPath = moveFromExecutorAttemptDirectory(Path.of(beforeOut), absoluteFilePath, fileExists) + (destinationPath.toString, bandIndices) + } else { + (absoluteFilePath, bandIndices) + } + }.toList.sortBy(_._1).asJava + + if (path.endsWith("out")) { + val beforeOut = path.substring(0, path.length - "out".length) + cleanUpExecutorAttemptDirectory(beforeOut) + } + + res } else { val tmp = saveRDDGeneric(rdd, bandCount, path, zLevel, cropBounds, formatOptions).asScala tmp.map(t => (t, (0 until bandCount).toList.asJava)).asJava @@ -426,7 +489,7 @@ package object geotiff { val metadata = new STACItem() metadata.asset(fixedPath) metadata.write(stacItemPath) - val finalPath = writeTiff( fixedPath,tiffs, gridBounds, croppedExtent, preprocessedRdd.metadata.crs, preprocessedRdd.metadata.tileLayout, compression, cellType, detectedBandCount, segmentCount,formatOptions = formatOptions, overviews = overviews) + val finalPath = writeTiff( fixedPath,tiffs, gridBounds, croppedExtent, preprocessedRdd.metadata.crs, preprocessedRdd.metadata.tileLayout, compression, cellType, detectedBandCount, segmentCount,formatOptions = formatOptions, overviews = overviews)._1 return Collections.singletonList(finalPath) }finally { preprocessedRdd.unpersist() @@ -578,12 +641,17 @@ package object geotiff { val segmentCount = bandSegmentCount * detectedBandCount val newPath = newFilePath(path, name) - writeTiff(newPath, tiffs, gridBounds, extent.intersection(croppedExtent).get, preprocessedRdd.metadata.crs, tileLayout, compression, cellType, detectedBandCount, segmentCount) + writeTiff(newPath, tiffs, gridBounds, extent.intersection(croppedExtent).get, preprocessedRdd.metadata.crs, tileLayout, compression, cellType, detectedBandCount, segmentCount)._1 }.collect() .toList } - private def writeTiff(path: String, tiffs: collection.Map[Int, Array[Byte]], gridBounds: GridBounds[Int], croppedExtent: Extent, crs: CRS, tileLayout: TileLayout, compression: DeflateCompression, cellType: CellType, detectedBandCount: Double, segmentCount: Int, formatOptions:GTiffOptions = new GTiffOptions,overviews: List[GeoTiffMultibandTile] = Nil) = { + private def writeTiff(path: String, tiffs: collection.Map[Int, Array[Byte]], + gridBounds: GridBounds[Int], croppedExtent: Extent, crs: CRS, + tileLayout: TileLayout, compression: DeflateCompression, cellType: CellType, + detectedBandCount: Double, segmentCount: Int, + formatOptions: GTiffOptions = new GTiffOptions, overviews: List[GeoTiffMultibandTile] = Nil + ): (String, Boolean) = { logger.info(s"Writing geotiff to $path with type ${cellType.toString()} and bands $detectedBandCount") val tiffTile: GeoTiffMultibandTile = toTiff(tiffs, gridBounds, tileLayout, compression, cellType, detectedBandCount, segmentCount) val options = if(formatOptions.colorMap.isDefined){ @@ -693,7 +761,7 @@ package object geotiff { val layout = rdd.metadata.layout val crs = rdd.metadata.crs - rdd.flatMap { + val res = rdd.flatMap { case (key, tile) => features.filter { case (_, extent) => val tileBounds = layout.mapTransform(extent) @@ -702,19 +770,28 @@ package object geotiff { ((name, extent), (key, tile)) } }.groupByKey() - .map { case ((name, extent), tiles) => - val filePath = newFilePath(path, name) + .map { case ((tileId, extent), tiles) => + // Each executor writes to a unique folder to avoid conflicts: + val executorAttemptDirectory = createExecutorAttemptDirectory(Path.of(path).getParent) + val filePath = executorAttemptDirectory + "/" + newFilePath(Path.of(path).getFileName.toString, tileId) (stitchAndWriteToTiff(tiles, filePath, layout, crs, extent, croppedExtent, cropDimensions, compression), extent) - }.collect() - .toList.asJava + }.collect().map { + case ((absoluteFilePath, fileExists), croppedExtent) => + val destinationPath = moveFromExecutorAttemptDirectory(Path.of(path).getParent, absoluteFilePath, fileExists) + (destinationPath.toString, croppedExtent) + }.toList.asJava + + cleanUpExecutorAttemptDirectory(Path.of(path).getParent.toString) + + res } private def stitchAndWriteToTiff(tiles: Iterable[(SpatialKey, MultibandTile)], filePath: String, layout: LayoutDefinition, crs: CRS, geometry: Geometry, croppedExtent: Option[Extent], cropDimensions: Option[java.util.ArrayList[Int]], compression: Compression, formatOptions: Option[GTiffOptions] = None - ) = { + ):(String, Boolean) = { val raster: Raster[MultibandTile] = ContextSeq(tiles, layout).stitch() val re = raster.rasterExtent @@ -834,74 +911,40 @@ package object geotiff { val filename = s"${filenamePrefix.getOrElse("openEO")}_${DateTimeFormatter.ISO_DATE.format(time)}_$name.tif" val filePath = Paths.get(path).resolve(filename).toString val timestamp = time format DateTimeFormatter.ISO_ZONED_DATE_TIME - (stitchAndWriteToTiff(tiles, filePath, layout, crs, geometry, croppedExtent, cropDimensions, compression), + (stitchAndWriteToTiff(tiles, filePath, layout, crs, geometry, croppedExtent, cropDimensions, compression)._1, timestamp, geometry.extent) } .collect() .toList.asJava } - def writeGeoTiff(geoTiff: MultibandGeoTiff, path: String, gtiffOptions: Option[GTiffOptions]): String = { - if (path.startsWith("s3:/")) { - val tempFile = Files.createTempFile(null, null) - geoTiff.write(tempFile.toString, optimizedOrder = true) + def writeGeoTiff(geoTiff: MultibandGeoTiff, path: String, gtiffOptions: Option[GTiffOptions]): (String, Boolean) = { + val tempFile = getTempFile(null, ".tif") + geoTiff.write(tempFile.toString, optimizedOrder = true) + val fileExists = Files.exists(tempFile) + if (fileExists) { gtiffOptions.foreach(options => embedGdalMetadata(tempFile, options.tagsAsGdalMetadataXml)) - uploadToS3(tempFile, path.replaceFirst("s3:/(?!/)", "s3://")) } else { - val tempFile = getTempFile(null, ".tif") - // TODO: Try to run fsync on the file opened by GeoTrellis (without the temporary copy) - geoTiff.write(tempFile.toString, optimizedOrder = true) - gtiffOptions.foreach(options => embedGdalMetadata(tempFile, options.tagsAsGdalMetadataXml)) - - // TODO: Write to unique path instead to avoid collisions between executors. Let the driver choose the paths. - moveOverwriteWithRetries(tempFile, Path.of(path)) - - // Call fsync on the parent path to assure the fusemount is up-to-date. - // The equivalent of Python's os.fsync - try { - FileChannel.open(Path.of(path)).force(true) - } catch { - case _: NoSuchFileException => // Ignore. The file may already be deleted by another executor - } - - path + logger.warn("writeGeoTiff() File was not created: " + path) } - } - - def moveOverwriteWithRetries(oldPath: Path, newPath: Path): Unit = { - var try_count = 1 - breakable { - while (true) { - try { - if (newPath.toFile.exists()) { - // It might be a partial result of a previous failing task. - logger.info(f"Will replace $newPath. (try $try_count)") - } - Files.deleteIfExists(newPath) - Files.move(oldPath, newPath) - break - } catch { - case e: FileAlreadyExistsException => - // Here if another executor wrote the file between the delete and the move statement. - try_count += 1 - if (try_count > 5) { - throw e - } - } + if (CreoS3Utils.isS3(path)) { + // Converting to Path and back could change the s3:// prefix to s3:/ + // The following line corrects this: + val correctS3Path = path.replaceFirst("s3:/(?!/)", "s3://") + if (fileExists) { + CreoS3Utils.uploadToS3(tempFile, correctS3Path) + } + (correctS3Path, fileExists) + } else { + // Retry should not be needed at this point, but it is almost free to keep it. + if (fileExists) { + CreoS3Utils.moveOverwriteWithRetries(tempFile.toString, path) } + (path, fileExists) } } - def uploadToS3(localFile: Path, s3Path: String) = { - val s3Uri = new AmazonS3URI(s3Path) - val objectRequest = PutObjectRequest.builder - .bucket(s3Uri.getBucket) - .key(s3Uri.getKey) - .build - CreoS3Utils.getCreoS3Client().putObject(objectRequest, RequestBody.fromFile(localFile)) - s3Path - } case class ContextSeq[K, V, M](tiles: Iterable[(K, V)], metadata: LayoutDefinition) extends Seq[(K, V)] with Metadata[LayoutDefinition] { override def length: Int = tiles.size diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/png/package.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/png/package.scala index 1da4f2b4b..acd5fcfec 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/png/package.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/png/package.scala @@ -6,7 +6,8 @@ import geotrellis.raster.render.RGBA import geotrellis.raster.{MultibandTile, UByteCellType} import geotrellis.spark._ import geotrellis.vector.{Extent, ProjectedExtent} -import org.openeo.geotrellis.geotiff.{SRDD, uploadToS3} +import org.openeo.geotrellis.creo.CreoS3Utils.uploadToS3 +import org.openeo.geotrellis.geotiff.SRDD import java.io.File import java.nio.file.{Files, Paths} diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/stac/STACItem.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/stac/STACItem.scala index 9abfeaf60..5b39bcfa7 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/stac/STACItem.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/stac/STACItem.scala @@ -1,6 +1,6 @@ package org.openeo.geotrellis.stac -import org.openeo.geotrellis.geotiff.uploadToS3 +import org.openeo.geotrellis.creo.CreoS3Utils.uploadToS3 import org.openeo.geotrellis.getTempFile import org.slf4j.LoggerFactory diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/TileGridTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/TileGridTest.scala index a32b02599..d7cdd3cc1 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/TileGridTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/TileGridTest.scala @@ -1,8 +1,5 @@ package org.openeo.geotrellis.geotiff -import java.time.LocalTime.MIDNIGHT -import java.time.ZoneOffset.UTC -import java.time.{LocalDate, ZonedDateTime} import geotrellis.proj4.{CRS, LatLng} import geotrellis.raster.io.geotiff.compression.DeflateCompression import geotrellis.spark._ @@ -10,19 +7,25 @@ import geotrellis.spark.util.SparkUtils import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.SparkContext import org.apache.spark.storage.StorageLevel.DISK_ONLY -import org.junit._ +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.api.{BeforeAll, Test} +import org.junit.{AfterClass, Assert} import org.openeo.geotrellis.LayerFixtures.rgbLayerProvider import org.openeo.geotrellis.png.PngTest import org.openeo.geotrellis.tile_grid.TileGrid import org.openeo.geotrellis.{LayerFixtures, geotiff} +import java.nio.file.Path +import java.time.LocalTime.MIDNIGHT +import java.time.ZoneOffset.UTC import java.time.format.DateTimeFormatter.ISO_ZONED_DATE_TIME +import java.time.{LocalDate, ZonedDateTime} import scala.collection.JavaConverters._ object TileGridTest { private var sc: SparkContext = _ - @BeforeClass + @BeforeAll def setupSpark(): Unit = { // originally geotrellis.spark.util.SparkUtils.createLocalSparkContext val conf = SparkUtils.createSparkConf @@ -47,7 +50,7 @@ class TileGridTest { import TileGridTest._ @Test - def testSaveStitchWithTileGrids(): Unit = { + def testSaveStitchWithTileGrids(@TempDir outDir: Path): Unit = { val date = ZonedDateTime.of(LocalDate.of(2020, 4, 5), MIDNIGHT, UTC) val bbox = ProjectedExtent(Extent(1.95, 50.95, 2.05, 51.05), LatLng) @@ -57,8 +60,13 @@ class TileGridTest { .toSpatial() .persist(DISK_ONLY) - val tiles = geotiff.saveStitchedTileGrid(spatialLayer, "/tmp/testSaveStitched.tiff", "10km", DeflateCompression(6)) - val expectedPaths = Set("/tmp/testSaveStitched-31UDS_3_4.tiff", "/tmp/testSaveStitched-31UDS_2_4.tiff", "/tmp/testSaveStitched-31UDS_3_5.tiff", "/tmp/testSaveStitched-31UDS_2_5.tiff") + val tiles = geotiff.saveStitchedTileGrid(spatialLayer, outDir + "/testSaveStitched.tiff", "10km", DeflateCompression(6)) + val expectedPaths = Set( + outDir + "/testSaveStitched-31UDS_3_4.tiff", + outDir + "/testSaveStitched-31UDS_2_4.tiff", + outDir + "/testSaveStitched-31UDS_3_5.tiff", + outDir + "/testSaveStitched-31UDS_2_5.tiff", + ) // TODO: check if extents (in the layer CRS) are 10000m wide/high (in UTM) Assert.assertEquals(expectedPaths, tiles.asScala.map { case (path, _) => path }.toSet) @@ -66,8 +74,13 @@ class TileGridTest { val extent = bbox.reproject(spatialLayer.metadata.crs) val cropBounds = mapAsJavaMap(Map("xmin" -> extent.xmin, "xmax" -> extent.xmax, "ymin" -> extent.ymin, "ymax" -> extent.ymax)) - val croppedTiles = geotiff.saveStitchedTileGrid(spatialLayer, "/tmp/testSaveStitched_cropped.tiff", "10km", cropBounds, DeflateCompression(6)) - val expectedCroppedPaths = Set("/tmp/testSaveStitched_cropped-31UDS_3_4.tiff", "/tmp/testSaveStitched_cropped-31UDS_2_4.tiff", "/tmp/testSaveStitched_cropped-31UDS_3_5.tiff", "/tmp/testSaveStitched_cropped-31UDS_2_5.tiff") + val croppedTiles = geotiff.saveStitchedTileGrid(spatialLayer, outDir + "/testSaveStitched_cropped.tiff", "10km", cropBounds, DeflateCompression(6)) + val expectedCroppedPaths = Set( + outDir + "/testSaveStitched_cropped-31UDS_3_4.tiff", + outDir + "/testSaveStitched_cropped-31UDS_2_4.tiff", + outDir + "/testSaveStitched_cropped-31UDS_3_5.tiff", + outDir + "/testSaveStitched_cropped-31UDS_2_5.tiff", + ) // TODO: also check extents Assert.assertEquals(expectedCroppedPaths, croppedTiles.asScala.map { case (path, _) => path }.toSet) diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/WriteRDDToGeotiffTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/WriteRDDToGeotiffTest.scala index faac7086e..c1cdfdf77 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/WriteRDDToGeotiffTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/geotiff/WriteRDDToGeotiffTest.scala @@ -1,5 +1,6 @@ package org.openeo.geotrellis.geotiff +import better.files.File.apply import geotrellis.layer.{CRSWorldExtent, SpaceTimeKey, SpatialKey, ZoomedLayoutScheme} import geotrellis.proj4.LatLng import geotrellis.raster.io.geotiff.GeoTiff @@ -13,14 +14,15 @@ import geotrellis.vector._ import geotrellis.vector.io.json.GeoJson import org.apache.spark.{SparkConf, SparkContext, SparkEnv} import org.junit.Assert._ -import org.junit._ +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.api.{BeforeAll, Test} import org.junit.rules.TemporaryFolder +import org.junit.{AfterClass, Rule} import org.openeo.geotrellis.{LayerFixtures, OpenEOProcesses, ProjectedPolygons} -import org.openeo.sparklisteners.GetInfoSparkListener import org.slf4j.{Logger, LoggerFactory} -import java.nio.file.{Files, Paths} -import java.time.{LocalDate, LocalTime, ZoneOffset, ZonedDateTime} +import java.nio.file.{Files, Path, Paths} +import java.time.{LocalTime, ZoneOffset, ZonedDateTime} import java.util import java.util.zip.Deflater._ import scala.annotation.meta.getter @@ -34,7 +36,7 @@ object WriteRDDToGeotiffTest{ var sc: SparkContext = _ - @BeforeClass + @BeforeAll def setupSpark() = { sc = { val conf = new SparkConf().setMaster("local[2]").setAppName(getClass.getSimpleName) @@ -68,7 +70,7 @@ class WriteRDDToGeotiffTest { @Test - def testWriteRDD(): Unit ={ + def testWriteRDD(@TempDir tempDir: Path): Unit ={ val layoutCols = 8 val layoutRows = 4 @@ -76,7 +78,7 @@ class WriteRDDToGeotiffTest { val imageTile = ByteArrayTile(intImage,layoutCols*256, layoutRows*256) val tileLayerRDD = TileLayerRDDBuilders.createMultibandTileLayerRDD(WriteRDDToGeotiffTest.sc,MultibandTile(imageTile),TileLayout(layoutCols,layoutRows,256,256),LatLng) - val filename = "out.tif" + val filename = (tempDir / "out.tif").toString() saveRDD(tileLayerRDD.withContext{_.repartition(layoutCols*layoutRows)},1,filename,formatOptions = allOverviewOptions) @@ -150,7 +152,7 @@ class WriteRDDToGeotiffTest { } @Test - def testWriteRDD_apply_neighborhood(): Unit ={ + def testWriteRDD_apply_neighborhood(@TempDir outDir: Path): Unit = { val layoutCols = 8 val layoutRows = 4 @@ -159,15 +161,16 @@ class WriteRDDToGeotiffTest { val tileLayerRDD = LayerFixtures.buildSingleBandSpatioTemporalDataCube(util.Arrays.asList(imageTile),Seq("2017-03-01T00:00:00Z")) - val filename = "openEO_2017-03-01Z.tif" + val filename = outDir + "/openEO_2017-03-01Z.tif" val p = new OpenEOProcesses() val buffered: MultibandTileLayerRDD[SpaceTimeKey] = p.remove_overlap(p.retileGeneric(tileLayerRDD,224,224,16,16),224,224,16,16) val cropBounds = Extent(-115, -65, 5.0, 56) - saveRDDTemporal(buffered,"./",cropBounds = Some(cropBounds)) + saveRDDTemporal(buffered, outDir.toString, cropBounds = Some(cropBounds)) val croppedRaster: Raster[MultibandTile] = tileLayerRDD.toSpatial().stitch().crop(cropBounds) - val referenceFile = "croppedRaster.tif" + val referenceFile = outDir + "/croppedRaster.tif" + Files.deleteIfExists(Path.of(referenceFile)) GeoTiff(croppedRaster,LatLng).write(referenceFile) val result = GeoTiff.readMultiband(filename).raster @@ -178,7 +181,7 @@ class WriteRDDToGeotiffTest { } @Test - def testWriteMultibandRDD(): Unit ={ + def testWriteMultibandRDD(@TempDir tempDir: Path): Unit ={ val layoutCols = 8 val layoutRows = 4 @@ -189,7 +192,7 @@ class WriteRDDToGeotiffTest { val thirdBand = imageTile.map{x => if(x >= 5 ) 50 else 200 } val tileLayerRDD = TileLayerRDDBuilders.createMultibandTileLayerRDD(WriteRDDToGeotiffTest.sc,MultibandTile(imageTile,secondBand,thirdBand),TileLayout(layoutCols,layoutRows,256,256),LatLng) - val filename = "outRGB.tif" + val filename = (tempDir / "outRGB.tif").toString() saveRDD(tileLayerRDD.withContext{_.repartition(layoutCols*layoutRows)},3,filename) val result = GeoTiff.readMultiband(filename).raster.tile assertArrayEquals(imageTile.toArray(),result.band(0).toArray()) @@ -199,7 +202,7 @@ class WriteRDDToGeotiffTest { @Test - def testWriteCroppedRDD(): Unit ={ + def testWriteCroppedRDD(@TempDir tempDir: Path): Unit ={ val layoutCols = 8 val layoutRows = 4 @@ -215,9 +218,9 @@ class WriteRDDToGeotiffTest { val cropBounds = Extent(-115, -65, 5.0, 56) val croppedRaster: Raster[MultibandTile] = tileLayerRDD.stitch().crop(cropBounds) - val referenceFile = "croppedRaster.tif" + val referenceFile = (tempDir / "croppedRaster.tif").toString() GeoTiff(croppedRaster,LatLng).write(referenceFile) - val filename = "outRGBCropped3.tif" + val filename = (tempDir / "outRGBCropped3.tif").toString() saveRDD(tileLayerRDD.withContext{_.repartition(layoutCols*layoutRows)},3,filename,cropBounds = Some(cropBounds)) val result = GeoTiff.readMultiband(filename).raster val reference = GeoTiff.readMultiband(referenceFile).raster @@ -228,7 +231,7 @@ class WriteRDDToGeotiffTest { } @Test - def testWriteRDDGlobalLayout(): Unit ={ + def testWriteRDDGlobalLayout(@TempDir tempDir: Path): Unit ={ val layoutCols = 8 val layoutRows = 8 @@ -244,10 +247,10 @@ class WriteRDDToGeotiffTest { val cropBounds = Extent(0, -90, 180, 90) val croppedRaster: Raster[MultibandTile] = tileLayerRDD.stitch().crop(cropBounds) - val referenceFile = "croppedRasterGlobalLayout.tif" + val referenceFile = (tempDir / "croppedRasterGlobalLayout.tif").toString() GeoTiff(croppedRaster,LatLng).write(referenceFile) - val filename = "outCropped.tif" + val filename = (tempDir / "outCropped.tif").toString() saveRDD(tileLayerRDD.withContext{_.repartition(tileLayerRDD.count().toInt)},3,filename,cropBounds = Some(cropBounds)) val resultRaster = GeoTiff.readMultiband(filename).raster @@ -259,7 +262,7 @@ class WriteRDDToGeotiffTest { } @Test - def testWriteEmptyRdd(): Unit ={ + def testWriteEmptyRdd(@TempDir tempDir: Path): Unit ={ val layoutCols = 8 val layoutRows = 4 @@ -268,7 +271,7 @@ class WriteRDDToGeotiffTest { val tileLayerRDD = TileLayerRDDBuilders.createMultibandTileLayerRDD(WriteRDDToGeotiffTest.sc,MultibandTile(imageTile),TileLayout(layoutCols,layoutRows,256,256),LatLng) val empty = tileLayerRDD.withContext{_.filter(_ => false)} - val filename = "outEmpty.tif" + val filename = (tempDir / "outEmpty.tif").toString() val cropBounds = Extent(-115, -65, 5.0, 56) saveRDD(empty,-1,filename,cropBounds = Some(cropBounds)) diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala index d9396a706..575b14dbe 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala @@ -4,7 +4,6 @@ import cats.data.NonEmptyList import geotrellis.layer.{FloatingLayoutScheme, LayoutTileSource, SpaceTimeKey, SpatialKey, TileLayerMetadata} import geotrellis.proj4.{CRS, LatLng} import geotrellis.raster.gdal.{GDALIOException, GDALRasterSource} -import geotrellis.raster.geotiff.GeoTiffRasterSource import geotrellis.raster.io.geotiff.GeoTiff import geotrellis.raster.resample.{Bilinear, CubicConvolution, ResampleMethod} import geotrellis.raster.summary.polygonal.Summary @@ -19,27 +18,27 @@ import geotrellis.vector._ import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD import org.junit.jupiter.api.Assertions.{assertEquals, assertNotSame, assertSame, assertTrue} -import org.junit.jupiter.api.{AfterAll, BeforeAll, Disabled, Test, Timeout} +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.api._ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource import org.openeo.geotrellis.TestImplicits._ import org.openeo.geotrellis.file.PyramidFactory -import org.openeo.geotrellis.layers.FileLayerProvider.rasterSourceRDD import org.openeo.geotrellis.geotiff._ +import org.openeo.geotrellis.layers.FileLayerProvider.rasterSourceRDD import org.openeo.geotrellis.netcdf.{NetCDFOptions, NetCDFRDDWriter} -import org.openeo.geotrellis.{LayerFixtures, OpenEOProcesses, ProjectedPolygons} +import org.openeo.geotrellis.{LayerFixtures, ProjectedPolygons} import org.openeo.geotrelliscommon.DatacubeSupport._ import org.openeo.geotrelliscommon.{ConfigurableSpaceTimePartitioner, DataCubeParameters, DatacubeSupport, NoCloudFilterStrategy, SpaceTimeByMonthPartitioner, SparseSpaceTimePartitioner} import org.openeo.opensearch.OpenSearchResponses.{CreoFeatureCollection, FeatureCollection, Link} import org.openeo.opensearch.backends.CreodiasClient import org.openeo.opensearch.{OpenSearchClient, OpenSearchResponses} import org.openeo.sparklisteners.GetInfoSparkListener -import org.slf4j.LoggerFactory import ucar.nc2.NetcdfFile import ucar.nc2.util.CompareNetcdf2 import java.net.{URI, URL} -import java.nio.file.{Files, Paths} +import java.nio.file.{Files, Path, Paths} import java.time.ZoneOffset.UTC import java.time.{LocalDate, ZoneId, ZonedDateTime} import java.util @@ -47,7 +46,6 @@ import java.util.Formatter import java.util.concurrent.TimeUnit import scala.collection.immutable import scala.io.Source -import scala.reflect.io.Directory object FileLayerProviderTest { private var _sc: Option[SparkContext] = None @@ -1079,7 +1077,7 @@ class FileLayerProviderTest extends RasterMatchers{ } @Test - def testPixelValueOffsetNeededCorner(): Unit = { + def testPixelValueOffsetNeededCorner(@TempDir outDir: Path): Unit = { // This selection will go over a corner that has nodata pixels val layer = testPixelValueOffsetNeeded( "/org/openeo/geotrellis/testPixelValueOffsetNeededCorner.json", @@ -1087,14 +1085,14 @@ class FileLayerProviderTest extends RasterMatchers{ LocalDate.of(2023, 4, 5), ) val cubeSpatial = layer.toSpatial() - cubeSpatial.writeGeoTiff("tmp/testPixelValueOffsetNeededCorner.tiff") + cubeSpatial.writeGeoTiff(f"$outDir/testPixelValueOffsetNeededCorner.tiff") val arr = cubeSpatial.collect().array assertTrue(isNoData(arr(1)._2.toArrayTile().band(0).get(162, 250))) assertEquals(172, arr(0)._2.toArrayTile().band(0).get(5, 5), 1) } @Test - def testPixelValueOffsetNeededDark(): Unit = { + def testPixelValueOffsetNeededDark(@TempDir outDir: Path): Unit = { // This will cover an area where pixels go under 0 val layer = testPixelValueOffsetNeeded( "/org/openeo/geotrellis/testPixelValueOffsetNeededDark.json", @@ -1102,7 +1100,7 @@ class FileLayerProviderTest extends RasterMatchers{ LocalDate.of(2023, 1, 17), ) val cubeSpatial = layer.toSpatial() - cubeSpatial.writeGeoTiff("tmp/testPixelValueOffsetNeededDark.tiff") + cubeSpatial.writeGeoTiff(f"$outDir/testPixelValueOffsetNeededDark.tiff") val band = cubeSpatial.collect().array(0)._2.toArrayTile().band(0) assertEquals(888, band.get(0, 0), 1) @@ -1122,11 +1120,7 @@ class FileLayerProviderTest extends RasterMatchers{ @Test - def testMissingS2(): Unit = { - val outDir = Paths.get("tmp/FileLayerProviderTest/") - new Directory(outDir.toFile).deleteRecursively() - Files.createDirectories(outDir) - + def testMissingS2(@TempDir outDir: Path): Unit = { val from = ZonedDateTime.parse("2024-03-24T00:00:00Z") val extent = Extent(-162.2501, 70.1839, -161.2879, 70.3401) @@ -1276,8 +1270,7 @@ class FileLayerProviderTest extends RasterMatchers{ } @Test - def testSamplingLoadPerProduct():Unit = { - + def testSamplingLoadPerProduct(@TempDir outDir: Path):Unit = { val srs32631 = "EPSG:32631" val projected_polygons_native_crs = ProjectedPolygons.fromExtent(Extent(703109 - 100, 5600100, 709000, 5610000 - 100), srs32631) val dataCubeParameters = new DataCubeParameters() @@ -1289,16 +1282,16 @@ class FileLayerProviderTest extends RasterMatchers{ val cube = LayerFixtures.sentinel2Cube(LocalDate.of(2023, 4, 5), projected_polygons_native_crs, "/org/openeo/geotrellis/testPixelValueOffsetNeededCorner.json",dataCubeParameters) val opts = new GTiffOptions opts.setFilenamePrefix("load_per_product") - saveRDDTemporal(cube,"./", formatOptions = opts) + saveRDDTemporal(cube,outDir.toString, formatOptions = opts) dataCubeParameters.loadPerProduct = false val cube_ref = LayerFixtures.sentinel2Cube(LocalDate.of(2023, 4, 5), projected_polygons_native_crs, "/org/openeo/geotrellis/testPixelValueOffsetNeededCorner.json",dataCubeParameters) opts.setFilenamePrefix("load_regular") - saveRDDTemporal(cube_ref,"./", formatOptions = opts) + saveRDDTemporal(cube_ref, outDir.toString, formatOptions = opts) - val reference = GeoTiff.readMultiband("./load_regular_2023-04-05Z.tif").raster - val actual = GeoTiff.readMultiband("./load_per_product_2023-04-05Z.tif").raster + val reference = GeoTiff.readMultiband(f"$outDir/load_regular_2023-04-05Z.tif").raster + val actual = GeoTiff.readMultiband(f"$outDir/load_per_product_2023-04-05Z.tif").raster assertRastersEqual(actual,reference) @@ -1328,7 +1321,7 @@ class FileLayerProviderTest extends RasterMatchers{ } @Test - def testMultibandCOGViaSTAC(): Unit = { + def testMultibandCOGViaSTAC(@TempDir outDir: Path): Unit = { val factory = LayerFixtures.STACCOGCollection() val extent = Extent(-162.2501, 70.1839, -161.2879, 70.3401) @@ -1341,7 +1334,7 @@ class FileLayerProviderTest extends RasterMatchers{ bands.add("temperature-mean") bands.add("precipitation-flux") - val outLocation = "tmp/testMultibandCOGViaSTAC.nc" + val outLocation = f"$outDir/testMultibandCOGViaSTAC.nc" val referenceFile = "https://artifactory.vgt.vito.be/artifactory/testdata-public/openeo/geotrellis_extrensions/testMultibandCOGViaSTAC.nc" writeToNetCDFAndCompare(projected_polygons_native_crs, dataCubeParameters, bands, factory, outLocation, referenceFile) @@ -1350,7 +1343,7 @@ class FileLayerProviderTest extends RasterMatchers{ @Test - def testMultibandCOGViaSTACResample(): Unit = { + def testMultibandCOGViaSTACResample(@TempDir outDir: Path): Unit = { val factory = LayerFixtures.STACCOGCollection(resolution = CellSize(10.0,10.0)) val extent = Extent(-162.2501, 70.1839, -161.2879, 70.3401) @@ -1365,11 +1358,13 @@ class FileLayerProviderTest extends RasterMatchers{ bands.add("temperature-mean") bands.add("precipitation-flux") - writeToNetCDFAndCompare(projected_polygons_native_crs, dataCubeParameters, bands, factory, "tmp/testMultibandCOGViaSTACResampledCubic.nc", "https://artifactory.vgt.vito.be/artifactory/testdata-public/openeo/geotrellis_extrensions/testMultibandCOGViaSTACResampledCubic.nc") + val referenceFile = "https://artifactory.vgt.vito.be/artifactory/testdata-public/openeo/geotrellis_extrensions/testMultibandCOGViaSTACResampledCubic.nc" + writeToNetCDFAndCompare(projected_polygons_native_crs, dataCubeParameters, bands, factory, + f"$outDir/testMultibandCOGViaSTACResampledCubic.nc", referenceFile) } @Test - def testMultibandCOGViaSTACResampleReadOneBand(): Unit = { + def testMultibandCOGViaSTACResampleReadOneBand(@TempDir outDir: Path): Unit = { val factory = LayerFixtures.STACCOGCollection(resolution = CellSize(10.0,10.0),util.Arrays.asList("precipitation-flux")) val extent = Extent(-162.2501, 70.1839, -161.2879, 70.3401) @@ -1381,7 +1376,9 @@ class FileLayerProviderTest extends RasterMatchers{ val bands: util.ArrayList[String] = new util.ArrayList[String]() bands.add("precipitation-flux") - writeToNetCDFAndCompare(projected_polygons_native_crs, dataCubeParameters, bands, factory, "tmp/testSinglebandCOGViaSTACResampled.nc", "https://artifactory.vgt.vito.be/artifactory/testdata-public/openeo/geotrellis_extrensions/testSinglebandCOGViaSTACResampled.nc") + val referenceFile = "https://artifactory.vgt.vito.be/artifactory/testdata-public/openeo/geotrellis_extrensions/testSinglebandCOGViaSTACResampled.nc" + writeToNetCDFAndCompare(projected_polygons_native_crs, dataCubeParameters, bands, factory, + f"$outDir/testSinglebandCOGViaSTACResampled.nc", referenceFile) } private def datacubeParams(polygonsAOI: ProjectedPolygons, resampleMethod: ResampleMethod) = {