Merge branch 'develop'

SEEDPOD Ground Risk.spec

@@ -12,6 +12,7 @@ import panel
 import pyviz_comms
 import rtree
 import shiboken2
+import sklearn
 from PyInstaller.building.api import PYZ, EXE, COLLECT
 from PyInstaller.building.build_main import Analysis
 
@@ -62,6 +63,7 @@ a = Analysis(['seedpod_ground_risk/main.py'],
                  ("README.md", '.'),
                  (os.path.join(pyviz_comms.comm_path, "notebook.js"), "pyviz_comms"),
                  (panel.__path__[0], "panel"),
+                 (sklearn.utils.__path__[0], "sklearn/utils"),
                  (datashader.__path__[0], "datashader"),
                  (distributed.__path__[0], "distributed"),
                  (os.path.join(fiona.__path__[0], "*.pyd"), "fiona"),  # Geospatial primitives
@@ -101,7 +103,7 @@ a = Analysis(['seedpod_ground_risk/main.py'],
                  "PyQt5",
                  "PyQt4",
                  "tkinter",
-                 "pydoc",
+                 # "pydoc",
                  "pdb",
                  "IPython",
                  "jupyter",

make_installer.iss

@@ -2,7 +2,7 @@
 ; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES!
 
 #define MyAppName "SEEDPOD Ground Risk"
-#define MyAppVersion "0.6a"
+#define MyAppVersion "0.10.0"
 #define MyAppPublisher "CASCADE UAV"
 #define MyAppURL "https://cascadeuav.com/seedpod/"
 #define MyAppExeName "SEEDPOD Ground Risk.exe"
@@ -22,7 +22,7 @@ DefaultDirName={autopf}\{#MyAppName}
 DefaultGroupName=SEEDPOD
 AllowNoIcons=yes
 ;LicenseFile=D:\PycharmProjects\seedpod_ground_risk\LICENSE
-InfoBeforeFile=D:\PycharmProjects\seedpod_ground_risk\README.md
+InfoBeforeFile=README.md
 ; Remove the following line to run in administrative install mode (install for all users.)
 PrivilegesRequired=lowest
 PrivilegesRequiredOverridesAllowed=commandline
@@ -38,7 +38,7 @@ Name: "english"; MessagesFile: "compiler:Default.isl"
 Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked
 
 [Files]
-Source: "D:\PycharmProjects\seedpod_ground_risk\dist\{#MyAppName}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
+Source: "dist\{#MyAppName}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
 ; NOTE: Don't use "Flags: ignoreversion" on any shared system files
 
 [Icons]

requirements.txt

@@ -91,3 +91,6 @@ typing-extensions==3.7.4.3
 urllib3==1.26.2
 xarray==0.16.2
 zict==2.0.0
+
+fastparquet~=0.5.0
+casex~=1.0.5

seedpod_ground_risk/core/plot_server.py

@@ -42,6 +42,7 @@ class PlotServer:
     def __init__(self, tiles: str = 'Wikipedia', tools: Optional[Iterable[str]] = None,
                  active_tools: Optional[Iterable[str]] = None,
                  cmap: str = 'CET_L18',
+                 raster_resolution: float = 40,
                  plot_size: Tuple[int, int] = (760, 735),
                  progress_callback: Optional[Callable[[str], None]] = None,
                  update_callback: Optional[Callable[[str], None]] = None):
@@ -51,13 +52,13 @@ def __init__(self, tiles: str = 'Wikipedia', tools: Optional[Iterable[str]] = No
         :param str tiles: a geoviews.tile_sources attribute string from http://geoviews.org/gallery/bokeh/tile_sources.html#bokeh-gallery-tile-sources
         :param List[str] tools: the bokeh tools to make available for the plot from https://docs.bokeh.org/en/latest/docs/user_guide/tools.html
         :param List[str] active_tools: the subset of `tools` that should be enabled by default
-        :param bool rasterise: Whether to opportunistically raster layers
         :param cmap: a colorcet attribute string for the colourmap to use from https://colorcet.holoviz.org/user_guide/Continuous.html
+        :param raster_resolution: resolution of a single square of the raster pixel grid in metres
         :param Tuple[int, int] plot_size: the plot size in (width, height) order
         :param progress_callback: an optional callable that takes a string updating progress
         :param update_callback: an optional callable that is called before an plot is rendered
         """
-        self.tools = ['hover', 'crosshair'] if tools is None else tools
+        self.tools = ['crosshair'] if tools is None else tools
         self.active_tools = ['wheel_zoom'] if active_tools is None else active_tools
 
         import colorcet
@@ -73,7 +74,7 @@ def __init__(self, tiles: str = 'Wikipedia', tools: Optional[Iterable[str]] = No
         self._generated_data_layers = {}
         self.data_layer_order = []
         self.data_layers = [ResidentialLayer('Residential Population', buffer_dist=30),
-                            RoadsLayer('Road Traffic Population per Hour')]
+                            RoadsLayer('Road Traffic Population/Hour')]
 
         self.annotation_layers = []
 
@@ -83,6 +84,9 @@ def __init__(self, tiles: str = 'Wikipedia', tools: Optional[Iterable[str]] = No
 
         self._x_range, self._y_range = [-1.45, -1.35], [50.85, 50.95]
 
+        self.raster_resolution_m = raster_resolution
+
+        self._epsg4326_to_epsg3857_proj = None
         self._epsg3857_to_epsg4326_proj = None
         self._preload_started = False
         self._preload_complete = False
@@ -196,31 +200,35 @@ def compose_overlay_plot(self, x_range: Optional[Sequence[float]] = (-1.6, -1.2)
             else:
                 # Construct box around requested bounds
                 bounds_poly = make_bounds_polygon(x_range, y_range)
+                raster_shape = self._get_raster_dimensions(bounds_poly, self.raster_resolution_m)
                 # Ensure bounds are small enough to render without OOM or heat death of universe
                 if bounds_poly.area < 0.2:
                     from time import time
 
                     t0 = time()
-                    self.generate_layers(bounds_poly)
+                    self.generate_layers(bounds_poly, raster_shape)
                     self._progress_callback("Rendering new map...")
                     plot = Overlay([res[0] for res in self._generated_data_layers.values()])
                     print("Generated all layers in ", time() - t0)
                     if self.annotation_layers:
                         import matplotlib.pyplot as mpl
                         plot = Overlay([res[0] for res in self._generated_data_layers.values()])
                         raw_datas = [res[2] for res in self._generated_data_layers.values()]
-                        raster_indices = dict(Longitude=np.linspace(x_range[0], x_range[1], num=400),
-                                              Latitude=np.linspace(y_range[0], y_range[1], num=400))
-                        raster_grid = np.zeros((400, 400), dtype=np.float64)
+                        raster_indices = dict(Longitude=np.linspace(x_range[0], x_range[1], num=raster_shape[0]),
+                                              Latitude=np.linspace(y_range[0], y_range[1], num=raster_shape[1]))
+                        raster_grid = np.zeros((raster_shape[1], raster_shape[0]), dtype=np.float64)
                         for res in self._generated_data_layers.values():
                             layer_raster_grid = res[1]
                             nans = np.isnan(layer_raster_grid)
                             layer_raster_grid[nans] = 0
-                            raster_grid += res[1]
-
+                            raster_grid += layer_raster_grid
+                        raster_grid = np.flipud(raster_grid)
+                        raster_indices['Latitude'] = np.flip(raster_indices['Latitude'])
                         annotations = []
+                        print('Annotating Layers...')
                         for layer in self.annotation_layers:
-                            annotation = layer.annotate(raw_datas, (raster_indices, raster_grid))
+                            annotation = layer.annotate(raw_datas, (raster_indices, raster_grid),
+                                                        resolution=self.raster_resolution_m)
                             if annotation:
                                 annotations.append(annotation)
 
@@ -250,13 +258,15 @@ def compose_overlay_plot(self, x_range: Optional[Sequence[float]] = (-1.6, -1.2)
         except Exception as e:
             # Catch-all to prevent plot blanking out and/or crashing app
             # Just display map tiles in case this was transient
+            import traceback
+            traceback.print_exc()
             print(e)
             plot = self._base_tiles
 
         return plot.opts(width=self.plot_size[0], height=self.plot_size[1],
                          tools=self.tools, active_tools=self.active_tools)
 
-    def generate_layers(self, bounds_poly: sg.Polygon) -> NoReturn:
+    def generate_layers(self, bounds_poly: sg.Polygon, raster_shape: Tuple[int, int]) -> NoReturn:
         """
         Generate static layers of map
 
@@ -270,8 +280,8 @@ def generate_layers(self, bounds_poly: sg.Polygon) -> NoReturn:
         layers = {}
         self._progress_callback('Generating layer data')
         with ThreadPoolExecutor() as pool:
-            layer_futures = [pool.submit(self.generate_layer, layer, bounds_poly, self._time_idx) for layer in
-                             self.data_layers]
+            layer_futures = [pool.submit(self.generate_layer, layer, bounds_poly, raster_shape, self._time_idx,
+                                         self.raster_resolution_m) for layer in self.data_layers]
         # Store generated layers as they are completed
         for future in as_completed(layer_futures):
             key, result = future.result()
@@ -293,8 +303,10 @@ def generate_layers(self, bounds_poly: sg.Polygon) -> NoReturn:
                 {k: layers[k] for k in layers.keys() if k not in self._generated_data_layers})
 
     @staticmethod
-    def generate_layer(layer: DataLayer, bounds_poly: sg.Polygon, hour: int) -> Union[
+    def generate_layer(layer: DataLayer, bounds_poly: sg.Polygon, raster_shape: Tuple[int, int], hour: int,
+                       resolution: float) -> Union[
         Tuple[str, Tuple[Geometry, np.ndarray, gpd.GeoDataFrame]], Tuple[str, None]]:
+
         import shapely.ops as so
 
         from_cache = False
@@ -308,9 +320,12 @@ def generate_layer(layer: DataLayer, bounds_poly: sg.Polygon, hour: int) -> Unio
             layer_bounds_poly = bounds_poly.difference(layer.cached_area)
         layer.cached_area = so.unary_union([layer.cached_area, bounds_poly])
         try:
-            result = layer.key, layer.generate(layer_bounds_poly, from_cache=from_cache, hour=hour)
+            result = layer.key, layer.generate(layer_bounds_poly, raster_shape, from_cache=from_cache, hour=hour,
+                                               resolution=resolution)
             return result
         except Exception as e:
+            import traceback
+            traceback.print_tb(e.__traceback__)
             print(e)
             return layer.key + ' FAILED', None
 
@@ -337,3 +352,30 @@ def remove_layer(self, layer):
 
     def set_layer_order(self, layer_order):
         self.data_layer_order = layer_order
+
+    def export_path_geojson(self, layer, filepath):
+        import os
+        if layer in self.annotation_layers:
+            layer.dataframe.to_file(os.path.join(os.sep, f'{filepath}', 'path.geojson'), driver='GeoJSON')
+
+    def _get_raster_dimensions(self, bounds_poly: sg.Polygon, raster_resolution_m: float) -> Tuple[int, int]:
+        """
+        Return a the (x,y) shape of a raster grid given its EPSG4326 envelope and desired raster resolution
+        :param bounds_poly: EPSG4326 Shapely Polygon specifying bounds
+        :param raster_resolution_m: raster resolution in metres
+        :return: 2-tuple of (width, height)
+        """
+
+        import pyproj
+
+        if self._epsg4326_to_epsg3857_proj is None:
+            self._epsg4326_to_epsg3857_proj = pyproj.Transformer.from_crs(pyproj.CRS.from_epsg('4326'),
+                                                                          pyproj.CRS.from_epsg('3857'),
+                                                                          always_xy=True)
+        bounds = bounds_poly.bounds
+
+        min_x, min_y = self._epsg4326_to_epsg3857_proj.transform(bounds[1], bounds[0])
+        max_x, max_y = self._epsg4326_to_epsg3857_proj.transform(bounds[3], bounds[2])
+        raster_width = int(abs(max_x - min_x) // raster_resolution_m)
+        raster_height = int(abs(max_y - min_y) // raster_resolution_m)
+        return raster_width, raster_height

seedpod_ground_risk/core/plot_worker.py

@@ -75,6 +75,14 @@ def set_time(self, hour):
     def layers_reorder(self, layer_order):
         self.plot_server.set_layer_order(layer_order)
 
+    @Slot(Layer, str)
+    def export_path_json(self, layer, filepath):
+        self.plot_server.export_path_geojson(layer, filepath)
+
+    @Slot(int, int)
+    def resize_plot(self, width, height):
+        self.plot_server.plot_size = (width, height)
+
     def layers_update(self, layers):
         self.signals.update_layers.emit(layers)
 

seedpod_ground_risk/layers/arbitrary_obstacle_layer.py

@@ -0,0 +1,44 @@
+from typing import NoReturn, Tuple
+
+import geopandas as gpd
+import numpy as np
+from holoviews.element import Geometry
+from shapely import geometry as sg
+
+from seedpod_ground_risk.layers.blockable_data_layer import BlockableDataLayer
+
+
+class ArbitraryObstacleLayer(BlockableDataLayer):
+    def __init__(self, key, filepath: str = '', **kwargs):
+        super(ArbitraryObstacleLayer, self).__init__(key, **kwargs)
+        self.filepath = filepath
+
+    def preload_data(self) -> NoReturn:
+
+        self.dataframe = gpd.read_file(self.filepath)
+        if self.buffer_dist:
+            epsg3857_geom = self.dataframe.to_crs('EPSG:3857').geometry
+            self.buffer_poly = gpd.GeoDataFrame(
+                {'geometry': epsg3857_geom.buffer(self.buffer_dist).to_crs('EPSG:4326')}
+            )
+
+    def generate(self, bounds_polygon: sg.Polygon, raster_shape: Tuple[int, int], from_cache: bool = False, **kwargs) -> \
+            Tuple[Geometry, np.ndarray, gpd.GeoDataFrame]:
+        import geoviews as gv
+        from holoviews.operation.datashader import rasterize
+
+        bounds = bounds_polygon.bounds
+
+        bounded_df = self.dataframe.cx[bounds[1]:bounds[3], bounds[0]:bounds[2]]
+
+        polys = gv.Polygons(bounded_df).opts(style={'alpha': 0.8, 'color': self._colour})
+        raster = rasterize(polys, width=raster_shape[0], height=raster_shape[1],
+                           x_range=(bounds[1], bounds[3]), y_range=(bounds[0], bounds[2]), dynamic=False)
+        raster_grid = np.copy(list(raster.data.data_vars.items())[0][1].data.astype(np.float))
+        if self.blocking:
+            raster_grid[raster_grid != 0] = -1
+
+        return polys, raster_grid, gpd.GeoDataFrame(self.dataframe)
+
+    def clear_cache(self):
+        pass

seedpod_ground_risk/layers/blockable_data_layer.py

@@ -0,0 +1,27 @@
+import random
+from abc import ABCMeta, abstractmethod
+
+from seedpod_ground_risk.layers.data_layer import DataLayer
+
+
+class BlockableDataLayer(DataLayer, metaclass=ABCMeta):
+    def __init__(self, key, colour: str = None,
+                 blocking=False, buffer_dist=0):
+        super().__init__(key)
+        self.blocking = blocking
+        self.buffer_dist = buffer_dist
+        # Set a random colour
+        self._colour = colour if colour is not None else "#" + ''.join(
+            [random.choice('0123456789ABCDEF') for _ in range(6)])
+
+    @abstractmethod
+    def preload_data(self):
+        pass
+
+    @abstractmethod
+    def generate(self, bounds_polygon, raster_shape, from_cache: bool = False, **kwargs):
+        pass
+
+    @abstractmethod
+    def clear_cache(self):
+        pass

seedpod_ground_risk/layers/data_layer.py

@@ -23,10 +23,11 @@ def __init__(self, key):
         self.cached_area = sg.Polygon()
 
     @abc.abstractmethod
-    def generate(self, bounds_polygon: sg.Polygon, from_cache: bool = False, **kwargs) -> Tuple[
+    def generate(self, bounds_polygon: sg.Polygon, raster_shape: Tuple[int, int], from_cache: bool = False, **kwargs) -> Tuple[
         Geometry, np.ndarray, gpd.GeoDataFrame]:
         """
         Generate the map of this layer. This is called asynchronously, so cannot access plot_server members.
+        :param raster_shape:
         :param shapely.geometry.Polygon bounds_polygon: the bounding polygon for which to generate the map
         :param bool from_cache: flag to indicate whether to use cached data to fulfill this request
         :return: an Overlay-able holoviews layer with specific options