Skip to content

Commit

Permalink
Merge branch 'main' into 258-reporting-2
Browse files Browse the repository at this point in the history
  • Loading branch information
jakimowb committed Sep 5, 2023
2 parents 4413281 + ffaf3d2 commit 07e59de
Show file tree
Hide file tree
Showing 84 changed files with 1,153 additions and 111 deletions.
2 changes: 1 addition & 1 deletion enmapbox/dependencycheck.py
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ def missingTestData() -> bool:
"""
try:
import enmapbox.exampledata
assert os.path.isfile(enmapbox.exampledata.enmap_potsdam)
assert os.path.isfile(enmapbox.exampledata.enmap)
return False
except Exception as ex:
print(ex, file=sys.stderr)
Expand Down
8 changes: 4 additions & 4 deletions enmapbox/exampledata/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from os.path import dirname, join

aerial_potsdam = join(dirname(__file__), 'aerial_potsdam.tif')
enmap_potsdam = join(dirname(__file__), 'enmap_potsdam.tif')
landcover_potsdam_point = join(dirname(__file__), 'landcover_potsdam_point.gpkg')
landcover_potsdam_polygon = join(dirname(__file__), 'landcover_potsdam_polygon.gpkg')
hires = join(dirname(__file__), 'aerial_potsdam.tif')
enmap = join(dirname(__file__), 'enmap_potsdam.tif')
landcover_point = join(dirname(__file__), 'landcover_potsdam_point.gpkg')
landcover_polygon = join(dirname(__file__), 'landcover_potsdam_polygon.gpkg')
11 changes: 5 additions & 6 deletions enmapbox/gui/enmapboxgui.py
Original file line number Diff line number Diff line change
Expand Up @@ -856,8 +856,8 @@ def onOpenPythonConsole(self):

def onAddBerlinDataset(self):
# example datasets are stored here: https://github.com/EnMAP-Box/enmap-box-exampledata
from enmapboxexampledata.berlin import enmap_berlin, hires_berlin, landcover_berlin_polygon, \
landcover_berlin_point, veg_cover_fraction_berlin_point, enmap_srf_library, library_berlin
from enmapboxtestdata import enmap_berlin, hires_berlin, landcover_berlin_polygon, landcover_berlin_point, \
veg_cover_fraction_berlin_point, enmap_srf_library, library_berlin
layers = [
QgsRasterLayer(enmap_berlin, basename(enmap_berlin)),
QgsRasterLayer(hires_berlin, basename(hires_berlin)),
Expand All @@ -870,8 +870,7 @@ def onAddBerlinDataset(self):

def onAddPotsdamDataset(self):
# example datasets are stored here: https://github.com/EnMAP-Box/enmap-box-exampledata
from enmapboxexampledata.potsdam import enmap_potsdam, aerial_potsdam, landcover_potsdam_polygon, \
landcover_potsdam_point
from enmapboxtestdata import enmap_potsdam, aerial_potsdam, landcover_potsdam_polygon, landcover_potsdam_point
layers = [
QgsRasterLayer(enmap_potsdam, basename(enmap_potsdam)),
QgsRasterLayer(aerial_potsdam, basename(aerial_potsdam)),
Expand Down Expand Up @@ -1880,9 +1879,9 @@ def niceLayerOrder(lyr: QgsMapLayer) -> (int, int):
lyrs = [lyr for lyr in sorted(lyrs, key=niceLayerOrder)]

# quick fix for issue #555
from enmapbox.exampledata import enmap_potsdam, aerial_potsdam
from enmapbox.exampledata import enmap, hires
lyrNames = [basename(lyr.source()) for lyr in lyrs]
a, b = lyrNames.index(basename(aerial_potsdam)), lyrNames.index(basename(enmap_potsdam))
a, b = lyrNames.index(basename(hires)), lyrNames.index(basename(enmap))
lyrs[b], lyrs[a] = lyrs[a], lyrs[b] # we just switch positions

for lyr in lyrs:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
class ClassFractionFromCategorizedVectorAlgorithm(EnMAPProcessingAlgorithm):
P_CATEGORIZED_VECTOR, _CATEGORIZED_VECTOR = 'categorizedVector', 'Categorized vector layer'
P_GRID, _GRID = 'grid', 'Grid'
P_COVERAGE, _COVERAGE = 'coverage', 'Minimum pixel coverage'
P_COVERAGE, _COVERAGE = 'coverage', 'Minimum pixel coverage [%]'
P_OUTPUT_FRACTION_RASTER, _OUTPUT_FRACTION_RASTER = 'outputFraction', 'Output class fraction layer'

def displayName(self):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,12 @@ def processAlgorithm(
assert ds is not None

# allow counter variables for band names (see issue #539)
if '{bandNo}' in parameters[self.P_NAMES] or '{bandName}' in parameters[self.P_NAMES]:
reader = RasterReader(source)
bandNames = [reader.bandName(bandNo) for bandNo in reader.bandNumbers()]
parameters[self.P_NAMES] = \
f"[f'{parameters[self.P_NAMES]}' for bandNo, bandName in enumerate({bandNames}, 1)]"
if self.P_NAMES in parameters:
if '{bandNo}' in parameters[self.P_NAMES] or '{bandName}' in parameters[self.P_NAMES]:
reader = RasterReader(source)
bandNames = [reader.bandName(bandNo) for bandNo in reader.bandNumbers()]
parameters[self.P_NAMES] = \
f"[f'{parameters[self.P_NAMES]}' for bandNo, bandName in enumerate({bandNames}, 1)]"

names = self.parameterAsStringValues(parameters, self.P_NAMES, context)
wavelengths = self.parameterAsFloatValues(parameters, self.P_WAVELENGTHS, context)
Expand Down
10 changes: 7 additions & 3 deletions enmapboxprocessing/algorithm/importenmapl2aalgorithm.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
class ImportEnmapL2AAlgorithm(EnMAPProcessingAlgorithm):
P_FILE, _FILE = 'file', 'Metadata file'
P_SET_BAD_BANDS, _SET_BAD_BANDS = 'setBadBands', 'Set bad bands'
P_EXCLUDE_BAD_BANDS, _EXCLUDE_BAD_BANDS, = 'excludeBadBands', 'Exclude bad bands'
P_EXCLUDE_BAD_BANDS, _EXCLUDE_BAD_BANDS, = 'excludeBadBands', 'Mark no data bands as bad bands'
P_DETECTOR_OVERLAP, _DETECTOR_OVERLAP = 'detectorOverlap', 'Detector overlap region'
O_DETECTOR_OVERLAP = [
'Order by detector (VNIR, SWIR)', 'Order by wavelength (default order)', 'Moving average filter', 'VNIR only',
Expand All @@ -44,7 +44,7 @@ def helpParameters(self) -> List[Tuple[str, str]]:
'Instead of executing this algorithm, '
'you may drag&drop the metadata XML file directly from your system file browser onto '
'the EnMAP-Box map view area.'),
(self._SET_BAD_BANDS, 'Whether to find and set the bad bands list by evaluating the image data.'),
(self._SET_BAD_BANDS, 'Whether to mark no data bands as bad bands.'),
(self._EXCLUDE_BAD_BANDS, 'Whether to exclude bands.'),
(self._DETECTOR_OVERLAP, 'Different options for handling the detector overlap region from 900 to 1000 '
'nanometers. For the Moving average filter, a kernel size of 3 is used.'),
Expand Down Expand Up @@ -124,14 +124,18 @@ def processAlgorithm(
int(text) for text in root.find('specific/swirProductQuality/expectedChannelsList').text.split(',')
]

overlapStart = wavelength[swirBandNumbers[0]]
overlapEnd = wavelength[vnirBandNumbers[-1]]
if detectorOverlap == self.OrderByDetectorOverlapOption:
bandList = vnirBandNumbers + swirBandNumbers
elif detectorOverlap == self.OrderByWavelengthOverlapOption:
bandList = list(range(1, len(wavelength) + 1))
elif detectorOverlap == self.VnirOnlyOverlapOption:
bandList = vnirBandNumbers
bandList.extend([bandNo for bandNo in swirBandNumbers if wavelength[bandNo - 1] > overlapEnd])
elif detectorOverlap == self.SwirOnlyOverlapOption:
bandList = swirBandNumbers
bandList = [bandNo for bandNo in vnirBandNumbers if wavelength[bandNo - 1] < overlapStart]
bandList.extend(swirBandNumbers)
else:
raise ValueError()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class PrepareClassificationDatasetFromCategorizedVectorAlgorithm(EnMAPProcessing
P_FEATURE_RASTER, _FEATURE_RASTER = 'featureRaster', 'Raster layer with features'
P_EXCLUDE_BAD_BANDS, _EXCLUDE_BAD_BANDS, = 'excludeBadBands', 'Exclude bad bands'
P_CATEGORY_FIELD, _CATEGORY_FIELD = 'categoryField', 'Field with class values'
P_COVERAGE, _COVERAGE = 'coverage', 'Minimum pixel coverage'
P_COVERAGE, _COVERAGE = 'coverage', 'Minimum pixel coverage [%]'
P_MAJORITY_VOTING, _MAJORITY_VOTING = 'majorityVoting', 'Majority voting'
P_OUTPUT_DATASET, _OUTPUT_DATASET = 'outputClassificationDataset', 'Output dataset'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
class RasterizeCategorizedVectorAlgorithm(EnMAPProcessingAlgorithm):
P_CATEGORIZED_VECTOR, _CATEGORIZED_VECTOR = 'categorizedVector', 'Categorized vector layer'
P_GRID, _GRID = 'grid', 'Grid'
P_COVERAGE, _COVERAGE = 'coverage', 'Minimum pixel coverage'
P_COVERAGE, _COVERAGE = 'coverage', 'Minimum pixel coverage [%]'
P_MAJORITY_VOTING, _MAJORITY_VOTING = 'majorityVoting', 'Majority voting'
P_OUTPUT_CATEGORIZED_RASTER, _OUTPUT_CATEGORIZED_RASTER = 'outputRasterizedCategories', \
'Output categorized raster layer'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ class RasterMathAlgorithm(EnMAPProcessingAlgorithm):

linkNumpy = EnMAPProcessingAlgorithm.htmlLink('https://numpy.org/doc/stable/reference/', 'NumPy')
linkRecipe = EnMAPProcessingAlgorithm.htmlLink(
'https://enmap-box.readthedocs.io/en/latest/usr_section/usr_cookbook/raster_math.html',
'RasterMath cookbook recipe')
'https://enmap-box.readthedocs.io/en/latest/usr_section/usr_manual/applications.html#raster-math',
'User Manual')

def displayName(self) -> str:
return 'Raster math'
Expand Down
110 changes: 110 additions & 0 deletions scripts/EnFireMAP/ingest_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
from collections import defaultdict
from os import listdir, makedirs, sep
from os.path import join, isdir, dirname, exists, normpath
from typing import Optional

from osgeo.gdal import BuildVRT, Warp, WarpOptions, Translate, TranslateOptions

from enmapbox.qgispluginsupport.qps.utils import SpatialExtent
from enmapboxprocessing.algorithm.importenmapl2aalgorithm import ImportEnmapL2AAlgorithm
from enmapboxprocessing.driver import Driver
from enmapboxprocessing.rasterreader import RasterReader
from qgis.core import QgsVectorLayer

rootData = r'D:\data\EnFireMap\data'
rootCube = r'D:\data\EnFireMap\cube'
rootTmpWarped = r'D:\data\EnFireMap\data\_warped'
rootTmpMosaics = r'D:\data\EnFireMap\data\_mosaics'
tilingScheme = QgsVectorLayer(r'D:\data\EnFireMap\cube\shp\grid.shp')
idField = 'Tile_ID'

products = [
'QL_PIXELMASK.TIF', 'QL_QUALITY_CIRRUS.TIF', 'QL_QUALITY_CLASSES.TIF', 'QL_QUALITY_CLOUD.TIF',
'QL_QUALITY_CLOUDSHADOW.TIF', 'QL_QUALITY_HAZE.TIF', 'QL_QUALITY_SNOW.TIF', 'QL_QUALITY_TESTFLAGS.TIF',
'QL_SWIR.TIF', 'QL_VNIR.TIF', 'SPECTRAL_IMAGE.vrt'
]


def prepareSpectralImages():
for name in listdir(rootData):
if isdir(join(rootData, name)):
xmlFilename = auxFindMetadataXml(join(rootData, name))
if xmlFilename is not None:
alg = ImportEnmapL2AAlgorithm()
parameters = {
alg.P_FILE: xmlFilename,
alg.P_SET_BAD_BANDS: True,
alg.P_EXCLUDE_BAD_BANDS: True,
alg.P_DETECTOR_OVERLAP: alg.OrderByWavelengthOverlapOption,
alg.P_OUTPUT_RASTER: xmlFilename.replace('METADATA.XML', 'SPECTRAL_IMAGE.vrt'),
}
alg.runAlg(alg, parameters)


def ingestData():
# group by date
scenesByDate = defaultdict(list)
for scene in listdir(rootData):
try:
prefix, datestamp, sceneNo = scene.split('_')
assert prefix == 'nc'
except Exception:
continue
scenesByDate[datestamp].append(scene)

# build mosaic for each date and product
filesByDateAndProduct = defaultdict(list)
for datestamp, scenes in scenesByDate.items():
for scene in scenes:
print('warp', scene, flush=True)
xmlFilename = auxFindMetadataXml(join(rootData, scene))
for product in products:
productFilename = xmlFilename.replace('METADATA.XML', product)
tmp = normpath(productFilename).split(sep)
productFilename2 = join(rootTmpWarped, *tmp[-2:])
if not exists(dirname(productFilename2)):
makedirs(dirname(productFilename2))
options = WarpOptions(
format='GTiff', dstSRS=tilingScheme.crs().toWkt(),
creationOptions=Driver.DefaultGTiffCreationOptions
)
Warp(productFilename2, productFilename, options=options)

filesByDateAndProduct[(datestamp, product)].append(productFilename2)

# - mosaic and cut into tiling scheme
for (datestamp, product), filenames in filesByDateAndProduct.items():
print('mosaic', datestamp + '_' + product, flush=True)
if not exists(join(rootTmpMosaics, datestamp)):
makedirs(join(rootTmpMosaics, datestamp))
filename = join(rootTmpMosaics, datestamp, datestamp + '_' + product).replace('.TIF', '.vrt')
BuildVRT(filename, filenames)

reader = RasterReader(filename)
extent = SpatialExtent(reader.crs(), reader.extent())
extent2 = extent.toCrs(tilingScheme.crs())
tilingScheme.selectByRect(extent2)
for feature in tilingScheme.selectedFeatures():
tileName = str(feature.attribute(idField))
print('cut', tileName, datestamp + '_' + product, flush=True)
filename2 = join(rootCube, tileName, datestamp + '_' + product).replace('.vrt', '.TIF')
if not exists(join(rootCube, tileName)):
makedirs(join(rootCube, tileName))
tileExtent = feature.geometry().boundingBox()
projWin = tileExtent.xMinimum(), tileExtent.yMaximum(), tileExtent.xMaximum(), tileExtent.yMinimum()
options = TranslateOptions(
format='GTiff', projWin=projWin, creationOptions=Driver.DefaultGTiffCreationOptions
)
Translate(filename2, filename, options=options)


def auxFindMetadataXml(folder: str) -> Optional[str]:
for name in listdir(folder):
if name.endswith('METADATA.XML'):
return join(folder, name)
return None


# prepareSpectralImages()
ingestData()
print('done')
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from sklearn.decomposition import PCA
from sklearn.preprocessing import RobustScaler, MinMaxScaler

from enmapbox.exampledata import enmap
from enmapboxtestdata import enmap
from enmapboxprocessing.rasterreader import RasterReader
from enmapboxprocessing.testcase import TestCase
from enmapboxprocessing.utils import Utils
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import numpy as np

from enmapbox.exampledata import enmap
from enmapboxtestdata import enmap
from enmapboxprocessing.algorithm.aggregaterasterbandsalgorithm import AggregateRasterBandsAlgorithm
from enmapboxprocessing.algorithm.testcase import TestCase
from enmapboxprocessing.rasterreader import RasterReader
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import numpy as np

from enmapbox.exampledata import enmap, landcover_polygon
from enmapboxtestdata import enmap, landcover_polygon
from enmapboxprocessing.algorithm.applymaskalgorithm import ApplyMaskAlgorithm
from enmapboxprocessing.algorithm.testcase import TestCase
from enmapboxprocessing.rasterreader import RasterReader
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from enmapbox.exampledata import enmap
from enmapboxtestdata import enmap
from enmapboxprocessing.algorithm.build3dcubealgorithm import Build3dCubeAlgorithm
from enmapboxprocessing.algorithm.testcase import TestCase

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def test_50p_coverage(self):
}
self.runalg(alg, parameters)
reader = RasterReader(parameters[alg.P_OUTPUT_FRACTION_RASTER])
self.assertAlmostEqual(247.589, np.mean(reader.array()), 3)
self.assertAlmostEqual(249.092, np.mean(reader.array()), 3)

def test_100p_coverage(self):
alg = ClassFractionFromCategorizedVectorAlgorithm()
Expand All @@ -93,4 +93,4 @@ def test_100p_coverage(self):
}
self.runalg(alg, parameters)
reader = RasterReader(parameters[alg.P_OUTPUT_FRACTION_RASTER])
self.assertAlmostEqual(247.589, np.mean(reader.array()), 3)
self.assertAlmostEqual(250.711, np.mean(reader.array()), 3)
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from math import isnan

from enmapbox.exampledata import landcover_polygon
from enmapboxtestdata import landcover_polygon
from enmapboxprocessing.algorithm.classificationperformancesimplealgorithm import \
ClassificationPerformanceSimpleAlgorithm
from enmapboxprocessing.algorithm.testcase import TestCase
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import numpy as np

from enmapbox.exampledata import landcover_polygon
from enmapboxtestdata import landcover_polygon
from enmapboxprocessing.algorithm.classificationperformancestratifiedalgorithm import (
stratifiedAccuracyAssessment, ClassificationPerformanceStratifiedAlgorithm
)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from sklearn.base import ClassifierMixin

from enmapbox.exampledata import enmap
from enmapboxtestdata import enmap
from enmapboxprocessing.algorithm.classificationworkflowalgorithm import ClassificationWorkflowAlgorithm
from enmapboxprocessing.algorithm.fitclassifieralgorithmbase import FitClassifierAlgorithmBase
from enmapboxprocessing.algorithm.testcase import TestCase
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import numpy as np

from enmapbox.exampledata import enmap
from enmapboxtestdata import enmap
from enmapboxprocessing.algorithm.convexhullalgorithm import ConvexHullAlgorithm
from enmapboxprocessing.algorithm.testcase import TestCase
from enmapboxprocessing.algorithm.translaterasteralgorithm import TranslateRasterAlgorithm
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import unittest

from enmapbox.exampledata import hires
from enmapboxtestdata import hires
from enmapboxprocessing.algorithm.convolutionfilteralgorithmbase import ConvolutionFilterAlgorithmBase
from enmapboxprocessing.algorithm.spatialconvolutionairydisk2dalgorithm import SpatialConvolutionAiryDisk2DAlgorithm
from enmapboxprocessing.algorithm.spatialconvolutionbox2dalgorithm import SpatialConvolutionBox2DAlgorithm
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from enmapbox.exampledata import enmap
from enmapboxtestdata import enmap
from enmapboxprocessing.algorithm.creategridalgorithm import CreateGridAlgorithm
from enmapboxprocessing.algorithm.testcase import TestCase
from qgis.core import QgsRasterLayer
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import numpy as np

from enmapbox.exampledata import hires
from enmapboxtestdata import hires
from enmapboxprocessing.algorithm.createmaskalgorithm import CreateMaskAlgorithm
from enmapboxprocessing.algorithm.testcase import TestCase
from enmapboxprocessing.rasterreader import RasterReader
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import numpy as np
from sklearn.base import ClassifierMixin

from enmapbox.exampledata import enmap
from enmapboxtestdata import enmap
from enmapboxprocessing.algorithm.creatergbimagefromclassprobabilityalgorithm import \
CreateRgbImageFromClassProbabilityAlgorithm
from enmapboxprocessing.algorithm.fitclassifieralgorithmbase import FitClassifierAlgorithmBase
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from osgeo import gdal

from enmapbox.exampledata import enmap
from enmapboxtestdata import enmap
from enmapboxprocessing.algorithm.createspectralindicesalgorithm import CreateSpectralIndicesAlgorithm
from enmapboxprocessing.algorithm.testcase import TestCase
from enmapboxprocessing.rasterreader import RasterReader
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from sklearn.base import ClusterMixin, TransformerMixin

from enmapbox.exampledata import enmap
from enmapboxtestdata import enmap
from enmapboxprocessing.algorithm.fitaffinitypropagationalgorithm import FitAffinityPropagationAlgorithm
from enmapboxprocessing.algorithm.fitbirchalgorithm import FitBirchAlgorithm
from enmapboxprocessing.algorithm.fitclustereralgorithmbase import FitClustererAlgorithmBase
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from sklearn.base import TransformerMixin

from enmapbox.exampledata import enmap
from enmapboxtestdata import enmap
from enmapboxprocessing.algorithm.fitfactoranalysisalgorithm import FitFactorAnalysisAlgorithm
from enmapboxprocessing.algorithm.fitfasticaalgorithm import FitFastIcaAlgorithm
from enmapboxprocessing.algorithm.fitfeatureagglomerationalgorithm import FitFeatureAgglomerationAlgorithm
Expand Down
Loading

0 comments on commit 07e59de

Please sign in to comment.