Merge pull request #192 from CosmiQ/dev

Bumping to 0.1.1

docs/conf.py

@@ -24,8 +24,8 @@
 copyright = u'2018-{}, CosmiQ Works: an IQT Lab'.format(time.strftime("%Y"))
 
 # The full version, including alpha/beta/rc tags
-release = '0.1.0'
-version = '0.1.0'
+release = '0.1.1'
+version = '0.1.1'
 
 # -- General configuration ---------------------------------------------------
 

environment-gpu.yml

@@ -27,8 +27,8 @@ dependencies:
   - torchvision=0.3.0
   - pyyaml=5.1
   - pyproj=2.1.3
+  - affine=2.2.2
   - pip:
-    - affine==2.2.2
     - albumentations==0.2.3
-    - rio-tiler==1.2.7
+#    - rio-tiler==1.2.7
     - rio-cogeo==1.0.0

environment.yml

@@ -26,8 +26,8 @@ dependencies:
   - torchvision=0.3.0
   - pyyaml=5.1
   - pyproj=2.1.3
+  - affine=2.2.2
   - pip:
-    - affine==2.2.2
     - albumentations==0.2.3
-    - rio-tiler==1.2.7
+#    - rio-tiler==1.2.7
     - rio-cogeo==1.0.0

requirements.txt

@@ -17,7 +17,6 @@ torch==1.1.0
 torchvision==0.3.0
 affine==2.2.2
 albumentations==0.2.3
-rio-tiler==1.2.7
 urllib3==1.24.3
 pyyaml==5.1.1
 pyproj==2.2.0

setup.py

@@ -81,7 +81,7 @@ def check_output(cmd):
                  'matplotlib>=3.1.0',
                  'affine>=2.2.2',
                  'albumentations>=0.2.3',
-                 'rio-tiler>=1.2.7',
+                 # 'rio-tiler>=1.2.7',
                  'pyyaml>=5.1',
                  'torchvision>=0.3.0']
 

solaris/__init__.py

@@ -1,3 +1,3 @@
 from . import bin, data, eval, nets, raster, tile, utils, vector
 
-__version__ = "0.1.0"
+__version__ = "0.1.1"

solaris/nets/train.py

@@ -41,6 +41,10 @@ def __init__(self, config, custom_model_dict=None):
         self.verbose = self.config['training']['verbose']
         if self.framework in ['torch', 'pytorch']:
             self.gpu_available = torch.cuda.is_available()
+            if self.gpu_available:
+                self.gpu_count = torch.cuda.device_count()
+            else:
+                self.gpu_count = 0
         elif self.framework == 'keras':
             self.gpu_available = tf.test.is_gpu_available()
 
@@ -62,6 +66,8 @@ def initialize_model(self):
         elif self.framework == 'torch':
             if self.gpu_available:
                 self.model = self.model.cuda()
+                if self.gpu_count > 1:
+                    self.model = torch.nn.DataParallel(self.model)
             # create optimizer
             if self.config['training']['opt_args'] is not None:
                 self.optimizer = self.optimizer(

solaris/tile/raster_tile.py

@@ -9,7 +9,8 @@
 import math
 from rio_cogeo.cogeo import cog_validate, cog_translate
 from ..utils.core import _check_crs, _check_rasterio_im_load
-from ..utils.tile import read_cog_tile
+# removing the following until COG functionality is implemented
+# from ..utils.tile import read_cog_tile
 from ..utils.geo import latlon_to_utm_epsg, reproject_geometry, reproject
 from ..utils.geo import raster_get_projection_unit
 from tqdm import tqdm
@@ -267,7 +268,8 @@ def tile_generator(self, src, dest_dir=None, channel_idxs=None,
             self.get_tile_bounds()
 
         for tb in self.tile_bounds:
-            if not self.is_cog or self.force_load_cog:
+            # removing the following line until COG functionality implemented
+            if True:  # not self.is_cog or self.force_load_cog:
                 vrt = self.load_src_vrt()
                 window = vrt.window(*tb)
                 if self.src.count != 1:
@@ -293,15 +295,15 @@ def tile_generator(self, src, dest_dir=None, channel_idxs=None,
                 else:
                     mask = None  # placeholder
 
-            else:
-                tile_data, mask, window, aff_xform = read_cog_tile(
-                    src=self.src,
-                    bounds=tb,
-                    tile_size=self.dest_tile_size,
-                    indexes=channel_idxs,
-                    nodata=self.nodata,
-                    resampling_method=self.resampling
-                    )
+            # else:
+            #     tile_data, mask, window, aff_xform = read_cog_tile(
+            #         src=self.src,
+            #         bounds=tb,
+            #         tile_size=self.dest_tile_size,
+            #         indexes=channel_idxs,
+            #         nodata=self.nodata,
+            #         resampling_method=self.resampling
+            #         )
             profile = self.src.profile
             profile.update(width=self.dest_tile_size[1],
                            height=self.dest_tile_size[0],

solaris/tile/vector_tile.py

@@ -4,7 +4,7 @@
 import geopandas as gpd
 from ..utils.core import _check_gdf_load, _check_crs
 from ..utils.tile import save_empty_geojson
-from ..utils.geo import gdf_get_projection_unit
+from ..utils.geo import gdf_get_projection_unit, split_multi_geometries
 from tqdm import tqdm
 
 
@@ -34,8 +34,8 @@ def __init__(self, dest_dir=None, dest_crs=None, output_format='GeoJSON',
             print('Initialization done.')
 
     def tile(self, src, tile_bounds, geom_type='Polygon',
-             split_multi_geometries=True, min_partial_perc=0.0,
-             dest_fname_base='geoms'):
+             split_multi_geoms=True, min_partial_perc=0.0,
+             dest_fname_base='geoms', obj_id_col=None):
         """Tile `src` into vector data tiles bounded by `tile_bounds`.
 
         Arguments
@@ -50,7 +50,7 @@ def tile(self, src, tile_bounds, geom_type='Polygon',
         geom_type : str, optional (default: "Polygon")
             The type of geometries contained within `src`. Defaults to
             ``"Polygon"``, can also be ``"LineString"``.
-        split_multi_geometries : bool, optional (default: True)
+        split_multi_geoms : bool, optional (default: True)
             Should multi-polygons or multi-linestrings generated by clipping
             a geometry into discontinuous pieces be separated? Defaults to yes
             (``True``).
@@ -64,10 +64,16 @@ def tile(self, src, tile_bounds, geom_type='Polygon',
             The base filename to use when creating outputs. The lower left
             corner coordinates of the tile's bounding box will be appended
             when saving.
+        obj_id_col : str, optional (default: None)
+            If ``split_multi_geoms=True``, the name of a column that specifies
+            a unique identifier for each geometry (e.g. the ``"BuildingId"``
+            column in many SpaceNet datasets.) See
+            :func:`solaris.utils.geo.split_multi_geometries` for more.
         """
         tile_gen = self.tile_generator(src, tile_bounds, geom_type,
-                                       split_multi_geometries,
-                                       min_partial_perc)
+                                       split_multi_geoms,
+                                       min_partial_perc,
+                                       obj_id_col=obj_id_col)
         for tile_gdf, tb in tqdm(tile_gen):
             if self.proj_unit not in ['meter', 'metre']:
                 out_path = os.path.join(
@@ -85,7 +91,8 @@ def tile(self, src, tile_bounds, geom_type='Polygon',
                 save_empty_geojson(out_path, self.dest_crs)
 
     def tile_generator(self, src, tile_bounds, geom_type='Polygon',
-                       split_multi_geometries=True, min_partial_perc=0.0):
+                       split_multi_geoms=True, min_partial_perc=0.0,
+                       obj_id_col=None):
         """Generate `src` vector data tiles bounded by `tile_bounds`.
 
         Arguments
@@ -100,7 +107,7 @@ def tile_generator(self, src, tile_bounds, geom_type='Polygon',
         geom_type : str, optional (default: "Polygon")
             The type of geometries contained within `src`. Defaults to
             ``"Polygon"``, can also be ``"LineString"``.
-        split_multi_geometries : bool, optional (default: True)
+        split_multi_geoms : bool, optional (default: True)
             Should multi-polygons or multi-linestrings generated by clipping
             a geometry into discontinuous pieces be separated? Defaults to yes
             (``True``).
@@ -110,6 +117,19 @@ def tile_generator(self, src, tile_bounds, geom_type='Polygon',
             be retained within a tile's bounds to be included in the output.
             Defaults to ``0.0``, meaning that the contained portion of a
             clipped geometry will be included, no matter how small.
+        obj_id_col : str, optional (default: None)
+            If ``split_multi_geoms=True``, the name of a column that specifies
+            a unique identifier for each geometry (e.g. the ``"BuildingId"``
+            column in many SpaceNet datasets.) See
+            :func:`solaris.utils.geo.split_multi_geometries` for more.
+
+        Yields
+        ------
+        tile_gdf : :class:`geopandas.GeoDataFrame`
+            A tile geodataframe.
+        tb : list
+            A list with ``[left, bottom, right, top]`` coordinates for the
+            boundaries contained by `tile_gdf`.
         """
         self.src = _check_gdf_load(src)
         self.src_crs = _check_crs(self.src.crs)
@@ -122,6 +142,8 @@ def tile_generator(self, src, tile_bounds, geom_type='Polygon',
                                 geom_type)
             if self.src_crs != self.dest_crs:
                 tile_gdf = tile_gdf.to_crs(epsg=self.dest_crs)
+            if split_multi_geoms:
+                split_multi_geometries(tile_gdf, obj_id_col=obj_id_col)
             yield tile_gdf, tb
 
 

solaris/utils/core.py

@@ -9,6 +9,8 @@
 import skimage
 from fiona._err import CPLE_OpenFailedError
 from fiona.errors import DriverError
+from warnings import warn
+
 
 def _check_rasterio_im_load(im):
     """Check if `im` is already loaded in; if not, load it in."""
@@ -42,7 +44,7 @@ def _check_df_load(df):
     elif isinstance(df, pd.DataFrame):
         return df
     else:
-        raise ValueError("{} is not an accepted DataFrame format.".format(df))
+        raise ValueError(f"{df} is not an accepted DataFrame format.")
 
 
 def _check_gdf_load(gdf):
@@ -51,12 +53,14 @@ def _check_gdf_load(gdf):
         try:
             return gpd.read_file(gdf)
         except (DriverError, CPLE_OpenFailedError):
+            warn(f"GeoDataFrame couldn't be loaded: either {gdf} isn't a valid"
+                 " path or it isn't a valid vector file. Returning an empty"
+                 " GeoDataFrame.")
             return gpd.GeoDataFrame()
     elif isinstance(gdf, gpd.GeoDataFrame):
         return gdf
     else:
-        raise ValueError(
-            "{} is not an accepted GeoDataFrame format.".format(gdf))
+        raise ValueError(f"{gdf} is not an accepted GeoDataFrame format.")
 
 
 def _check_geom(geom):
@@ -85,6 +89,8 @@ def _check_crs(input_crs):
         out_crs = input_crs.to_epsg()
     elif isinstance(input_crs, int):
         out_crs = input_crs
+    elif input_crs is None:
+        out_crs = input_crs
     return out_crs
 
 def get_data_paths(path, infer=False):

solaris/utils/geo.py

@@ -522,8 +522,10 @@ def split_multi_geometries(gdf, obj_id_col=None, group_col=None,
     gdf2 = _check_gdf_load(gdf)
     # drop duplicate columns (happens if loading a csv with geopandas)
     gdf2 = gdf2.loc[:, ~gdf2.columns.duplicated()]
+    if len(gdf2) == 0:
+        return gdf2
     # check if the values in gdf2[geometry] are polygons; if strings, do loads
-    if isinstance(gdf2[geom_col][0], str):
+    if isinstance(gdf2[geom_col].iloc[0], str):
         gdf2[geom_col] = gdf2[geom_col].apply(loads)
     split_geoms_gdf = pd.concat(
         gdf2.apply(_split_multigeom_row, axis=1, geom_col=geom_col).tolist())