diff --git a/README.md b/README.md index 689da55..90fd940 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,7 @@ Get a quantized mesh for tile index z, x, y. Set the minZoom value to retrieve e - **cog**: Path or URL to COG file. - **minZoom** : The min zoomlevel for the terrain. Default (0) +- **noData** : The value to use for NoData in COG. Default (0) - **resamplingMethod** : Resampling method for COG: 'nearest', 'bilinear', 'cubic', 'cubic_spline', 'lanczos', 'average', 'mode', 'gauss', 'rms'. Default 'none' - **skipCache** : Set to true to prevent loading tiles from the cache. Default (False) - **meshingMethod**: The Meshing method to use: 'grid', 'martini', 'delatin' @@ -246,7 +247,7 @@ The CTOD service has a very basic tile caching option, tiles can be retrieved an ### Nodata -Nodata values in the COG are automatically set to 0 else it is likely that the meshing will go wrong, for now nodata should be handled in the source data (COG) In a future version we can try to fill up the nodata values based on surrounding pixels. +Nodata values in the COG are automatically set to 0 else it is likely that the meshing will go wrong, for now nodata should be handled in the source data (COG) or pass `noData={value}` to the .terrain request to overwrite the default value `0` ### Used libraries diff --git a/ctod/core/cog/cog_request.py b/ctod/core/cog/cog_request.py index 6056f66..c002043 100644 --- a/ctod/core/cog/cog_request.py +++ b/ctod/core/cog/cog_request.py @@ -25,6 +25,7 @@ def __init__( y, cog_processor: CogProcessor, cog_reader_pool: CogReaderPool, + no_data: int, resampling_method=None, generate_normals=False, ): @@ -35,10 +36,13 @@ def __init__( self.y = y self.cog_processor = cog_processor self.cog_reader_pool = cog_reader_pool + self.no_data = no_data self.resampling_method = resampling_method self.generate_normals = generate_normals - self.key = generate_cog_cache_key(cog, cog_processor.get_name(), z, x, y) - self.tile_bounds = utils.get_tile_bounds(self.tms, self.x, self.y, self.z) + self.key = generate_cog_cache_key( + cog, cog_processor.get_name(), z, x, y) + self.tile_bounds = utils.get_tile_bounds( + self.tms, self.x, self.y, self.z) self.is_out_of_bounds = False self.data = None self.processed_data = None @@ -75,7 +79,7 @@ async def download_tile_async(self, executor: ThreadPoolExecutor): def _download(self, reader: CogReader, loop): kwargs = self.cog_processor.get_reader_kwargs() dowloaded_data = reader.download_tile( - self.x, self.y, self.z, loop, self.resampling_method, **kwargs + self.x, self.y, self.z, loop, self.no_data, self.resampling_method, **kwargs ) if dowloaded_data is not None: diff --git a/ctod/core/cog/reader/cog_reader.py b/ctod/core/cog/reader/cog_reader.py index f7cdd62..499711b 100644 --- a/ctod/core/cog/reader/cog_reader.py +++ b/ctod/core/cog/reader/cog_reader.py @@ -43,6 +43,7 @@ def download_tile( y: int, z: int, loop: AbstractEventLoop, + nodata: int, resampling_method: str = None, **kwargs: Any, ) -> ImageData: @@ -69,12 +70,14 @@ def download_tile( if z < self.safe_level: if not self.unsafe: logging.warning( - f"Skipping unsafe tile {self.cog} {z,x,y}, generate more overviews or use --unsafe to load anyway" + f"""Skipping unsafe tile {self.cog} { + z, x, y}, generate more overviews or use --unsafe to load anyway""" ) return None else: logging.warning( - f"Loading unsafe tile {self.cog} {z,x,y}, consider generating more overviews" + f"""Loading unsafe tile {self.cog} { + z, x, y}, consider generating more overviews""" ) if resampling_method is not None: @@ -91,10 +94,11 @@ def download_tile( image_data = self.rio_reader.tile( tile_z=z, tile_x=x, tile_y=y, align_bounds_with_dataset=True, **kwargs ) - # For now set nodata to 0 if nodata is present in the metadata - # handle this better later + + # Set nodata value if self.nodata_value is not None: - image_data.data[image_data.data == self.nodata_value] = float(0) + image_data.data[image_data.data == + self.nodata_value] = float(nodata) return image_data @@ -129,14 +133,16 @@ def _set_safe_level(self): dataset_width = self.rio_reader.dataset.width dataset_wgs_width = dataset_bounds.right - dataset_bounds.left pixels_per_wgs = dataset_width / dataset_wgs_width - pixels_per_tile_downsampled = 256 * max(self.rio_reader.dataset.overviews(1)) + pixels_per_tile_downsampled = 256 * \ + max(self.rio_reader.dataset.overviews(1)) for z in range(0, 24): tile_bounds = self.tms.xy_bounds(Tile(x=0, y=0, z=z)) tile_wgs = tile_bounds.right - tile_bounds.left tile_wgs_clipped = min(tile_wgs, dataset_wgs_width) tile_pixels_needed = tile_wgs_clipped * pixels_per_wgs - needed_tiles = math.ceil(tile_pixels_needed / pixels_per_tile_downsampled) + needed_tiles = math.ceil( + tile_pixels_needed / pixels_per_tile_downsampled) if needed_tiles <= 4: self.safe_level = z @@ -147,7 +153,8 @@ def _set_nodata_value(self): reader_info = self.rio_reader.info() self.nodata_value = ( - reader_info.nodata_value if hasattr(reader_info, "nodata_value") else None + reader_info.nodata_value if hasattr( + reader_info, "nodata_value") else None ) def __del__(self): diff --git a/ctod/core/cog/reader/cog_reader_mosaic.py b/ctod/core/cog/reader/cog_reader_mosaic.py index cf04b4f..bba423c 100644 --- a/ctod/core/cog/reader/cog_reader_mosaic.py +++ b/ctod/core/cog/reader/cog_reader_mosaic.py @@ -13,7 +13,7 @@ class CogReaderMosaic: """A reader for a Cloud Optimized GeoTIFF. This class is used to pool readers to avoid opening and closing the same file many times. """ - + def __init__(self, pool, config: Any, cog: str, tms: TileMatrixSet, unsafe: bool = False): self.pool = pool self.config = config @@ -21,26 +21,28 @@ def __init__(self, pool, config: Any, cog: str, tms: TileMatrixSet, unsafe: bool self.tms = tms self.unsafe = unsafe self.last_used = time.time() - + def close(self): """Close the reader.""" - + # CogReaderMosaic doesn't have a reader itself, therefore it doesn't need to close pass - + def tiler(self, src_path: str, *args, **kwargs) -> ImageData: - future = asyncio.run_coroutine_threadsafe(self.pool.get_reader(src_path, self.tms), args[3]) + future = asyncio.run_coroutine_threadsafe( + self.pool.get_reader(src_path, self.tms), args[3]) reader = future.result() - - data = reader.download_tile(args[0], args[1], args[2], args[3], **kwargs) + + data = reader.download_tile( + args[0], args[1], args[2], args[3], **kwargs) reader.return_reader() - + if not data: raise TileOutsideBounds return data - - def download_tile(self, x: int, y: int, z: int, loop: asyncio.AbstractEventLoop, resampling_method: str = None, **kwargs: Any) -> ImageData: + + def download_tile(self, x: int, y: int, z: int, loop: asyncio.AbstractEventLoop, no_data: int, resampling_method: str = None, **kwargs: Any) -> ImageData: """Retrieve an image from a Cloud Optimized GeoTIFF based on a tile index. Args: @@ -55,32 +57,39 @@ def download_tile(self, x: int, y: int, z: int, loop: asyncio.AbstractEventLoop, Returns: ImageData: Image data from the Cloud Optimized GeoTIFF. """ - + tile_bounds = self.tms.xy_bounds(Tile(x=x, y=y, z=z)) datasets = self._get_intersecting_datasets(tile_bounds) - + if len(datasets) == 0: return None if not self._tile_intersects(tile_bounds, self.config["extent"]) or len(datasets) == 0: return None - + if not self.unsafe and len(datasets) > 10: - logging.warning(f"Too many datasets intersecting with requested tile {z,x,y}, {len(datasets)}") + logging.warning(f"""Too many datasets intersecting with requested tile { + z, x, y}, {len(datasets)}""") return None - + if resampling_method is not None: kwargs["resampling_method"] = resampling_method try: - img, _ = mosaic_reader(datasets, self.tiler, x, y, z, loop, **kwargs) + img, _ = mosaic_reader(datasets, self.tiler, + x, y, z, loop, **kwargs) + + # Set nodata value + if self.nodata_value is not None: + img.data[img.data == self.nodata_value] = float(no_data) + return img except Exception as e: - return None - + return None + def return_reader(self): """Done with the reader, return it to the pool.""" - + self.last_used = time.time() self.pool.return_reader(self) @@ -89,9 +98,9 @@ def _get_intersecting_datasets(self, tile_bounds: BoundingBox) -> list: for dataset in self.config["datasets"]: if self._tile_intersects(tile_bounds, dataset["extent"]): intersecting_datasets.append(dataset["path"]) - + return intersecting_datasets - + def _tile_intersects(self, tile_bounds: BoundingBox, dataset_bounds: list) -> bool: """Check if a tile intersects with a dataset. Instead of check if inside we check if something is outside and @@ -104,9 +113,9 @@ def _tile_intersects(self, tile_bounds: BoundingBox, dataset_bounds: list) -> bo Returns: bool: True if bounds intersect, False otherwise """ - + if (tile_bounds.left > dataset_bounds[2] or tile_bounds.right < dataset_bounds[0] or - tile_bounds.bottom > dataset_bounds[3] or tile_bounds.top < dataset_bounds[1]): + tile_bounds.bottom > dataset_bounds[3] or tile_bounds.top < dataset_bounds[1]): return False - + return True diff --git a/ctod/core/factory/terrain_factory.py b/ctod/core/factory/terrain_factory.py index 00d2d2c..8417ceb 100644 --- a/ctod/core/factory/terrain_factory.py +++ b/ctod/core/factory/terrain_factory.py @@ -37,6 +37,7 @@ async def handle_request( terrain_request: TerrainRequest, cog_reader_pool: CogReaderPool, processor, + no_data: int, ) -> asyncio.Future: """Handle a TerrainRequest @@ -46,14 +47,15 @@ async def handle_request( Returns: asyncio.Future: Future which will be set when the terrain is ready """ - + async with self.lock: # add terrain_request to terrain_requests cache self.terrain_requests[terrain_request.key] = terrain_request - # loop over wanted files and add to processing queue if needed - processing_queue_keys = set(item.key for item in self.processing_queue._queue) - + # loop over wanted files and add to processing queue if needed + processing_queue_keys = set( + item.key for item in self.processing_queue._queue) + for wanted_file in terrain_request.wanted_files: # Check if the cog request is not already in the cache or processing queue or open requests if ( @@ -70,6 +72,7 @@ async def handle_request( wanted_file.y, processor, cog_reader_pool, + no_data, wanted_file.resampling_method, wanted_file.generate_normals, ) @@ -85,7 +88,7 @@ async def handle_request( asyncio.create_task(self.cache_changed()) return await terrain_request.wait() - + def start_periodic_check(self, interval: int = 5): """Start a task to periodically check the cache for expired items""" @@ -107,7 +110,7 @@ async def _process_queue(self): while not self.processing_queue.empty(): cog_request = await self.processing_queue.get() - + self.open_requests.add(cog_request.key) asyncio.create_task(self._process_cog_request(cog_request)) del cog_request @@ -129,7 +132,7 @@ async def _process_cog_request(self, cog_request): async def cache_changed(self, keys: list = None): """Triggered by the cache when a new item was added""" - + # When checking if a cog request is already in cache, open requests # when a new requests comes in we need to have it somewhere so we don't directly # remove a key from the open_requests until we have it in the cache @@ -138,7 +141,7 @@ async def cache_changed(self, keys: list = None): self.open_requests = self.open_requests - set(keys) # If already processing the list set rerun to True - # We don't want to queue since process_terrain_request should pick up + # We don't want to queue since process_terrain_request should pick up # everything that is available for processing if self.processing_terrain_requests: self.processing_terrain_requests_rerun = True @@ -149,7 +152,7 @@ async def cache_changed(self, keys: list = None): async def _process_terrain_requests(self): """Check and run process on terrain requests when ready for processing""" - try: + try: # Convert to use O(n) complexity instead of O(n^2) with set intersection cache_keys = set(self.cache.keys) terrain_keys = list(self.terrain_requests.items()) @@ -218,7 +221,7 @@ async def _cleanup(self): def _get_executor(self): """Get the ThreadPoolExecutor""" - + if self.executor is None: self.executor = ThreadPoolExecutor(max_workers=20) @@ -230,14 +233,15 @@ def _try_reset_executor(self): This is to try to free up memory when idle but seems to have no effect but has no negative impact either. """ - + if self.executor and self.executor._work_queue.empty(): self.executor.shutdown(wait=False) self.executor = None def _print_debug_info(self): """Print debug info about the factory and it's state""" - + logging.info( - f"Factory: terrain reqs: {len(self.terrain_requests)}, cache size: {len(self.cache.keys)}, open requests: {len(self.open_requests)}, queue size: {self.processing_queue.qsize()}" + f"""Factory: terrain reqs: {len(self.terrain_requests)}, cache size: {len(self.cache.keys)}, open requests: { + len(self.open_requests)}, queue size: {self.processing_queue.qsize()}""" ) diff --git a/ctod/core/terrain/empty_tile.py b/ctod/core/terrain/empty_tile.py index e6e1bce..83a6e1a 100644 --- a/ctod/core/terrain/empty_tile.py +++ b/ctod/core/terrain/empty_tile.py @@ -9,7 +9,8 @@ from morecantile import TileMatrixSet from quantized_mesh_encoder.ecef import to_ecef -def generate_empty_tile(tms: TileMatrixSet, z: int, x: int, y: int) -> bytes: + +def generate_empty_tile(tms: TileMatrixSet, z: int, x: int, y: int, no_data: int) -> bytes: """Generate an empty terrain tile for a tile index with geodetic surface normals Args: @@ -17,23 +18,23 @@ def generate_empty_tile(tms: TileMatrixSet, z: int, x: int, y: int) -> bytes: z (int): z tile index x (int): x tile index y (int): y tile index + no_data (int): no data value Returns: bytes: quantized mesh tile """ - + grid_vertices, grid_triangles = generate_grid(256, 256, 20, 20) - vertices_3d = np.column_stack((grid_vertices, np.zeros(grid_vertices.shape[0]))) - + vertices_3d = np.column_stack( + (grid_vertices, np.full(grid_vertices.shape[0], no_data))) vertices_new = np.array(vertices_3d, dtype=np.float64) triangles_new = np.array(grid_triangles, dtype=np.uint16) - + bounds = utils.get_tile_bounds(tms, x, y, z) rescaled_vertices = rescale_positions(vertices_new, bounds, flip_y=False) cartesian = to_ecef(rescaled_vertices) normals = generate_geodetic_normals(cartesian, triangles_new) quantized = quantize(rescaled_vertices, triangles_new, normals) - + return quantized - \ No newline at end of file diff --git a/ctod/core/terrain/generator/terrain_generator_quantized_mesh_delatin.py b/ctod/core/terrain/generator/terrain_generator_quantized_mesh_delatin.py index 5b1552c..2d67468 100644 --- a/ctod/core/terrain/generator/terrain_generator_quantized_mesh_delatin.py +++ b/ctod/core/terrain/generator/terrain_generator_quantized_mesh_delatin.py @@ -11,7 +11,7 @@ class TerrainGeneratorQuantizedMeshDelatin(TerrainGenerator): """A TerrainGenerator for a delatin based mesh.""" - + def __init__(self): self.ellipsoid: Ellipsoid = WGS84 @@ -24,15 +24,18 @@ def generate(self, terrain_request: TerrainRequest) -> bytes: Returns: quantized_mesh (bytes): The generated quantized mesh """ - + main_cog = terrain_request.get_main_file() - + if main_cog is None or main_cog.data is None or main_cog.is_out_of_bounds: logging.debug("main_cog.processed_data is None") - quantized_empty_tile = generate_empty_tile(main_cog.tms, main_cog.z, main_cog.x, main_cog.y) + quantized_empty_tile = generate_empty_tile( + main_cog.tms, main_cog.z, main_cog.x, main_cog.y, main_cog.no_data) return quantized_empty_tile - - rescaled_vertices = rescale_positions(main_cog.processed_data[0], main_cog.tile_bounds, flip_y=False) - quantized = quantize(rescaled_vertices, main_cog.processed_data[1], main_cog.processed_data[2]) - - return quantized \ No newline at end of file + + rescaled_vertices = rescale_positions( + main_cog.processed_data[0], main_cog.tile_bounds, flip_y=False) + quantized = quantize( + rescaled_vertices, main_cog.processed_data[1], main_cog.processed_data[2]) + + return quantized diff --git a/ctod/core/terrain/generator/terrain_generator_quantized_mesh_grid.py b/ctod/core/terrain/generator/terrain_generator_quantized_mesh_grid.py index 9612c0d..d2af4d3 100644 --- a/ctod/core/terrain/generator/terrain_generator_quantized_mesh_grid.py +++ b/ctod/core/terrain/generator/terrain_generator_quantized_mesh_grid.py @@ -4,15 +4,15 @@ from ctod.core.terrain.quantize import quantize from ctod.core.utils import rescale_positions from ctod.core.terrain.generator.mesh_helper import ( - get_neighbor_files, - get_neighbour_transformed_edge_vertices, - get_neighbour_normals, + get_neighbor_files, + get_neighbour_transformed_edge_vertices, + get_neighbour_normals, average_height_and_normals_to_neighbours) class TerrainGeneratorQuantizedMeshGrid(TerrainGenerator): """A TerrainGenerator for a grid based mesh.""" - + def __init__(self): self.tile_size = 255 @@ -25,21 +25,26 @@ def generate(self, terrain_request: TerrainRequest) -> bytes: Returns: quantized_mesh (bytes): The generated quantized mesh """ - + main_cog = terrain_request.get_main_file() - + if main_cog.processed_data is None or main_cog.is_out_of_bounds: - quantized_empty_tile = generate_empty_tile(main_cog.tms, main_cog.z, main_cog.x, main_cog.y) + quantized_empty_tile = generate_empty_tile( + main_cog.tms, main_cog.z, main_cog.x, main_cog.y, main_cog.no_data) return quantized_empty_tile - + vertices, triangles, normals = main_cog.processed_data n, ne, e, se, s, sw, w, nw = get_neighbor_files(terrain_request) - neighbour_vertices = get_neighbour_transformed_edge_vertices(self.tile_size, n, ne, e, se, s, sw, w, nw) - neighbour_normals = get_neighbour_normals(self.tile_size, n, ne, e, se, s, sw, w, nw) if terrain_request.generate_normals else None - - average_height_and_normals_to_neighbours(vertices, normals, neighbour_vertices, neighbour_normals) - rescaled = rescale_positions(vertices, main_cog.tile_bounds, flip_y=False) + neighbour_vertices = get_neighbour_transformed_edge_vertices( + self.tile_size, n, ne, e, se, s, sw, w, nw) + neighbour_normals = get_neighbour_normals( + self.tile_size, n, ne, e, se, s, sw, w, nw) if terrain_request.generate_normals else None + + average_height_and_normals_to_neighbours( + vertices, normals, neighbour_vertices, neighbour_normals) + rescaled = rescale_positions( + vertices, main_cog.tile_bounds, flip_y=False) quantized = quantize(rescaled, triangles, normals) - return quantized \ No newline at end of file + return quantized diff --git a/ctod/core/terrain/generator/terrain_generator_quantized_mesh_martini.py b/ctod/core/terrain/generator/terrain_generator_quantized_mesh_martini.py index 5448a35..8ee42d0 100644 --- a/ctod/core/terrain/generator/terrain_generator_quantized_mesh_martini.py +++ b/ctod/core/terrain/generator/terrain_generator_quantized_mesh_martini.py @@ -6,16 +6,16 @@ from ctod.core.terrain.quantize import quantize from ctod.core.terrain.generator.mesh_helper import ( average_height_and_normals_to_neighbours, - merge_shared_vertices, + merge_shared_vertices, get_neighbor_files, - get_neighbour_transformed_edge_vertices_from_array, + get_neighbour_transformed_edge_vertices_from_array, get_neighbour_normals_from_array) from ctod.core.utils import rescale_positions class TerrainGeneratorQuantizedMeshMartini(TerrainGenerator): """A TerrainGenerator for a martini based mesh.""" - + def __init__(self): self.tile_size = 256 @@ -28,34 +28,47 @@ def generate(self, terrain_request: TerrainRequest) -> bytes: Returns: quantized_mesh (bytes): The generated quantized mesh """ - + main_cog = terrain_request.get_main_file() - + # should not happen, in case it does return empty tile if main_cog is None or main_cog.data is None or main_cog.is_out_of_bounds: logging.debug("main_cog.processed_data is None") - quantized_empty_tile = generate_empty_tile(main_cog.tms, main_cog.z, main_cog.x, main_cog.y) + quantized_empty_tile = generate_empty_tile( + main_cog.tms, main_cog.z, main_cog.x, main_cog.y, main_cog.no_data) return quantized_empty_tile - + n, ne, e, se, s, sw, w, nw = get_neighbor_files(terrain_request) - + # remesh neighbours - n_v, _, n_n = merge_shared_vertices(self.tile_size, n, e=ne, se=e, s=main_cog, sw=w, w=nw) - ne_v, _, ne_n = merge_shared_vertices(self.tile_size, ne, s=e, sw=main_cog, w=n) - e_v, _, e_n = merge_shared_vertices(self.tile_size, e, n=ne, s=se, sw=s, w=main_cog, nw=n) - se_v, _, se_n = merge_shared_vertices(self.tile_size, se, n=e, nw=main_cog, w=s) - s_v, _, s_n = merge_shared_vertices(self.tile_size, s, e=se, ne=e, n=main_cog, nw=w, w=sw) - sw_v, _, sw_n = merge_shared_vertices(self.tile_size, sw, e=s, ne=main_cog, n=w) - w_v, _, w_n = merge_shared_vertices(self.tile_size, w, s=sw, se=s, e=main_cog, ne=n, n=nw) - nw_v, _, nw_n = merge_shared_vertices(self.tile_size, nw, s=w, e=n, se=main_cog) - main_v, main_t, main_n = merge_shared_vertices(self.tile_size, main_cog, n=n, ne=ne, e=e, se=se, s=s, sw=sw, w=w, nw=nw) - - neighbour_vertices = get_neighbour_transformed_edge_vertices_from_array(self.tile_size, n_v, ne_v, e_v, se_v, s_v, sw_v, w_v, nw_v) - neighbour_normals = get_neighbour_normals_from_array(self.tile_size, n_v, n_n, ne_v, ne_n, e_v, e_n, se_v, se_n, s_v, s_n, sw_v, sw_n, w_v, w_n, nw_v, nw_n) if terrain_request.generate_normals else None - average_height_and_normals_to_neighbours(main_v, main_n, neighbour_vertices, neighbour_normals) - - rescaled = rescale_positions(main_v, main_cog.tile_bounds, flip_y=False) + n_v, _, n_n = merge_shared_vertices( + self.tile_size, n, e=ne, se=e, s=main_cog, sw=w, w=nw) + ne_v, _, ne_n = merge_shared_vertices( + self.tile_size, ne, s=e, sw=main_cog, w=n) + e_v, _, e_n = merge_shared_vertices( + self.tile_size, e, n=ne, s=se, sw=s, w=main_cog, nw=n) + se_v, _, se_n = merge_shared_vertices( + self.tile_size, se, n=e, nw=main_cog, w=s) + s_v, _, s_n = merge_shared_vertices( + self.tile_size, s, e=se, ne=e, n=main_cog, nw=w, w=sw) + sw_v, _, sw_n = merge_shared_vertices( + self.tile_size, sw, e=s, ne=main_cog, n=w) + w_v, _, w_n = merge_shared_vertices( + self.tile_size, w, s=sw, se=s, e=main_cog, ne=n, n=nw) + nw_v, _, nw_n = merge_shared_vertices( + self.tile_size, nw, s=w, e=n, se=main_cog) + main_v, main_t, main_n = merge_shared_vertices( + self.tile_size, main_cog, n=n, ne=ne, e=e, se=se, s=s, sw=sw, w=w, nw=nw) + + neighbour_vertices = get_neighbour_transformed_edge_vertices_from_array( + self.tile_size, n_v, ne_v, e_v, se_v, s_v, sw_v, w_v, nw_v) + neighbour_normals = get_neighbour_normals_from_array( + self.tile_size, n_v, n_n, ne_v, ne_n, e_v, e_n, se_v, se_n, s_v, s_n, sw_v, sw_n, w_v, w_n, nw_v, nw_n) if terrain_request.generate_normals else None + average_height_and_normals_to_neighbours( + main_v, main_n, neighbour_vertices, neighbour_normals) + + rescaled = rescale_positions( + main_v, main_cog.tile_bounds, flip_y=False) quantized = quantize(rescaled, main_t, main_n) - + return quantized - diff --git a/ctod/core/terrain/terrain_request.py b/ctod/core/terrain/terrain_request.py index 595fed4..48d74bd 100644 --- a/ctod/core/terrain/terrain_request.py +++ b/ctod/core/terrain/terrain_request.py @@ -24,6 +24,7 @@ def __init__( z: int, x: int, y: int, + no_data: int, resampling_method: str, cog_processor: CogProcessor, terrain_generator: TerrainGenerator, @@ -35,6 +36,7 @@ def __init__( self.z = z self.x = x self.y = y + self.no_data = no_data, self.resampling_method = resampling_method self.cog_processor = cog_processor self.terrain_generator = terrain_generator @@ -43,7 +45,8 @@ def __init__( self.wanted_files = [] self._generate_wanted_files() self.key = generate_uuid() - self.main_file_key = generate_cog_cache_key(self.cog, self.cog_processor.get_name(), self.z, self.x, self.y) + self.main_file_key = generate_cog_cache_key( + self.cog, self.cog_processor.get_name(), self.z, self.x, self.y) self.wanted_file_keys = self.get_wanted_file_keys() self.processing = False self.future = asyncio.Future() @@ -173,6 +176,7 @@ def _generate_wanted_files(self): self.y, self.cog_processor, self.cog_reader_pool, + self.no_data, self.resampling_method, self.generate_normals, ) @@ -189,6 +193,7 @@ def _generate_wanted_files(self): tile.y, self.cog_processor, self.cog_reader_pool, + self.no_data, self.resampling_method, self.generate_normals, ) diff --git a/ctod/server/fastapi.py b/ctod/server/fastapi.py index 55c7e57..9bceee9 100644 --- a/ctod/server/fastapi.py +++ b/ctod/server/fastapi.py @@ -24,24 +24,30 @@ @asynccontextmanager async def lifespan(app: FastAPI): args = parse_args() - unsafe = get_value(args, "unsafe", os.environ.get("CTOD_UNSAFE", False), False) + unsafe = get_value(args, "unsafe", os.environ.get( + "CTOD_UNSAFE", False), False) tile_cache_path = get_value( - args, "tile_cache_path", os.environ.get("CTOD_TILE_CACHE_PATH", None), None + args, "tile_cache_path", os.environ.get( + "CTOD_TILE_CACHE_PATH", None), None ) - port = get_value(args, "port", int(os.environ.get("CTOD_PORT", 5000)), 5000) + port = get_value(args, "port", int( + os.environ.get("CTOD_PORT", 5000)), 5000) logging_level = get_value( - args, "logging_level", os.environ.get("CTOD_LOGGING_LEVEL", "info"), "info" + args, "logging_level", os.environ.get( + "CTOD_LOGGING_LEVEL", "info"), "info" ) db_name = get_value( - args, "db_name", os.environ.get("CTOD_DB_NAME", "factory_cache.db"), "factory_cache.db" + args, "db_name", os.environ.get( + "CTOD_DB_NAME", "factory_cache.db"), "factory_cache.db" ) factory_cache_ttl = 15 - + patch_occlusion() setup_logging(log_level=getattr(logging, logging_level.upper())) log_ctod_start(port, tile_cache_path) - terrain_factory = TerrainFactory(tile_cache_path, db_name, factory_cache_ttl) + terrain_factory = TerrainFactory( + tile_cache_path, db_name, factory_cache_ttl) await terrain_factory.cache.initialize() globals["terrain_factory"] = terrain_factory @@ -92,7 +98,8 @@ def layer_json( zoomGridSizes: str = queries.query_zoom_grid_sizes, defaultMaxError: int = queries.query_default_max_error, zoomMaxErrors: str = queries.query_zoom_max_errors, - extensions: str = queries.query_extensions + extensions: str = queries.query_extensions, + noData: int = queries.query_no_data ): params = queries.QueryParameters( cog, @@ -105,7 +112,8 @@ def layer_json( zoomGridSizes, defaultMaxError, zoomMaxErrors, - extensions + extensions, + noData ) return get_layer_json(globals["tms"], params) @@ -132,6 +140,7 @@ async def terrain( defaultMaxError: int = queries.query_default_max_error, zoomMaxErrors: str = queries.query_zoom_max_errors, extensions: str = queries.query_extensions, + noData: int = queries.query_no_data ): params = queries.QueryParameters( cog, @@ -144,7 +153,8 @@ async def terrain( zoomGridSizes, defaultMaxError, zoomMaxErrors, - extensions + extensions, + noData ) use_extensions = get_extensions(extensions, request) diff --git a/ctod/server/handlers/terrain.py b/ctod/server/handlers/terrain.py index fcbbd18..15821ba 100644 --- a/ctod/server/handlers/terrain.py +++ b/ctod/server/handlers/terrain.py @@ -79,6 +79,7 @@ async def get( meshing_method = qp.get_meshing_method() resampling_method = qp.get_resampling_method() min_zoom = qp.get_min_zoom() + no_data = qp.get_no_data() # Try handling the request from the cache if not skip_cache: @@ -98,7 +99,7 @@ async def get( # Always return an empty tile at 0 or requested zoom level is lower than min_zoom if z == 0 or z < min_zoom: empty_tile = await self._return_empty_terrain( - tms, cog, meshing_method, z, x, y + tms, cog, meshing_method, z, x, y, no_data ) return Response(content=empty_tile, media_type="application/octet-stream") @@ -110,6 +111,7 @@ async def get( z, x, y, + no_data, resampling_method, cog_processor, terrain_generator, @@ -117,7 +119,7 @@ async def get( extensions["octvertexnormals"], ) quantized = await self.terrain_factory.handle_request( - tms, terrain_request, self.cog_reader_pool, cog_processor + tms, terrain_request, self.cog_reader_pool, cog_processor, no_data ) await self._try_save_tile_to_cache(cog, tms, meshing_method, z, x, y, quantized) @@ -125,13 +127,13 @@ async def get( del terrain_generator del cog_processor del terrain_request - + # ToDo: Add support for gzip # makes a bit of difference in size but is slower - #if 'gzip' in request.headers.get('Accept-Encoding', ''): + # if 'gzip' in request.headers.get('Accept-Encoding', ''): # quantized = gzip.compress(quantized) # headers = {"Content-Encoding": "gzip"} - #else: + # else: # headers = {} return Response(content=quantized, media_type="application/octet-stream") @@ -169,7 +171,7 @@ def _get_cog_processor( return cog_processor(qp) async def _return_empty_terrain( - self, tms: TileMatrixSet, cog: str, meshing_method: str, z: int, x: int, y: int + self, tms: TileMatrixSet, cog: str, meshing_method: str, z: int, x: int, y: int, no_data: int ): """Return an empty terrain tile generated based on the tile index including geodetic surface normals @@ -179,9 +181,10 @@ async def _return_empty_terrain( z (int): z tile index x (int): x tile index y (int): y tile index + no_data (int): no data value """ - quantized_empty_tile = generate_empty_tile(tms, z, x, y) + quantized_empty_tile = generate_empty_tile(tms, z, x, y, no_data) await self._try_save_tile_to_cache( cog, tms, meshing_method, z, x, y, quantized_empty_tile ) @@ -233,4 +236,3 @@ async def _try_save_tile_to_cache( await save_tile_to_disk( self.tile_cache_path, cog, tms, meshing_method, z, x, y, data ) - diff --git a/ctod/server/queries.py b/ctod/server/queries.py index b7ee5dc..242fdc8 100644 --- a/ctod/server/queries.py +++ b/ctod/server/queries.py @@ -65,6 +65,12 @@ description="Normally supplied trough Accept header but can be set and overridden here, by multiple extensions user separator '-' (octvertexnormals-watermask) Currently only octvertexnormals is supported", ) +query_no_data = Query( + None, + title="No Data", + description="No data value for the COG, Default (0)" +) + class QueryParameters: """ @@ -84,6 +90,7 @@ def __init__( defaultMaxError: int = None, zoomMaxErrors: str = None, extensions: str = None, + noData: int = None, ): self.cog = cog self.minZoom = minZoom @@ -96,6 +103,7 @@ def __init__( self.defaultMaxError = defaultMaxError self.zoomMaxErrors = zoomMaxErrors self.extensions = extensions + self.nodata = noData def get_cog(self) -> str: """Returns the cog if it's not None, otherwise returns a default value""" @@ -122,6 +130,11 @@ def get_meshing_method(self) -> str: return self.meshingMethod if self.meshingMethod is not None else "grid" + def get_no_data(self) -> int: + """Returns the nodata if it's not None, otherwise returns a default value""" + + return self.nodata if self.nodata is not None else 0 + def get_skip_cache(self) -> bool: """Returns the skipCache if it's not None, otherwise returns a default value""" diff --git a/ctod/templates/static/controls.js b/ctod/templates/static/controls.js index 3930633..ca0c8af 100644 --- a/ctod/templates/static/controls.js +++ b/ctod/templates/static/controls.js @@ -7,6 +7,7 @@ var cogValue = "./ctod/files/test_cog.tif"; var resamplingValue = "none"; var skipCacheValue = false; +var noDataValue = 0; document.addEventListener("DOMContentLoaded", async () => { try { @@ -23,6 +24,7 @@ document.addEventListener("DOMContentLoaded", async () => { function setupTweakpane() { minZoomValue = getIntParameterValue("minZoom", minZoomValue); maxZoomValue = getIntParameterValue("maxZoom", maxZoomValue); + noDataValue = getIntParameterValue("noData", noDataValue); cogValue = getStringParameterValue("cog", cogValue); resamplingValue = getStringParameterValue("resamplingMethod", resamplingValue); skipCacheValue = getBoolParameterValue("skipCache", skipCacheValue); @@ -149,6 +151,25 @@ function createTerrainPane() { updateTerrainProvider(); }); + noData = terrainFolder.addBlade({ + view: "slider", + label: "noData", + min: -100, + max: 100, + value: noDataValue, + format: (e) => Math.round(e), + }); + + noData.on("change", (ev) => { + nod = Math.round(ev.value); + if (noDataValue === nod) { + return; + } + + noDataValue = nod; + updateTerrainProvider(); + }); + const resamplingMethod = terrainFolder.addBinding(PARAMS, "resampling", { options: { none: "none", @@ -220,7 +241,7 @@ function createLayerPane() { } function updateTerrainProvider() { - setTerrainProvider(minZoomValue, maxZoomValue, cogValue, resamplingValue, skipCacheValue, meshingMethodValue); + setTerrainProvider(minZoomValue, maxZoomValue, noDataValue, cogValue, resamplingValue, skipCacheValue, meshingMethodValue); } function getStringParameterValue(param, defaultValue) { diff --git a/ctod/templates/static/index.js b/ctod/templates/static/index.js index 1b2a6ab..208a7c0 100644 --- a/ctod/templates/static/index.js +++ b/ctod/templates/static/index.js @@ -47,12 +47,13 @@ function initializeLayers() { const minZoom = getUrlParamIgnoreCase("minZoom") || 1; const maxZoom = getUrlParamIgnoreCase("maxZoom") || 18; + const noData = getUrlParamIgnoreCase("noData") || 0; const cog = getUrlParamIgnoreCase("cog") || "./ctod/files/test_cog.tif"; const skipCache = getUrlParamIgnoreCase("skipCache") || false; const meshingMethod = getUrlParamIgnoreCase("meshingMethod") || "grid"; - setTerrainProvider(minZoom, maxZoom, cog, "none", skipCache, meshingMethod); + setTerrainProvider(minZoom, maxZoom, noData, cog, "none", skipCache, meshingMethod); streetsLayer.show = true; satelliteLayer.show = false; @@ -110,8 +111,8 @@ function configureViewer() { }); } -function setTerrainProvider(minZoom, maxZoom, cog, resamplingMethod, skipCache, meshingMethod) { - let terrainProviderUrl = `${window.location.origin}/tiles?minZoom=${minZoom}&maxZoom=${maxZoom}&cog=${cog}&skipCache=${skipCache}&meshingMethod=${meshingMethod}`; +function setTerrainProvider(minZoom, maxZoom, noData, cog, resamplingMethod, skipCache, meshingMethod) { + let terrainProviderUrl = `${window.location.origin}/tiles?minZoom=${minZoom}&maxZoom=${maxZoom}&noData=${noData}&cog=${cog}&skipCache=${skipCache}&meshingMethod=${meshingMethod}`; if (resamplingMethod !== "none") { terrainProviderUrl += `&resamplingMethod=${resamplingMethod}`;