Add Region importer

Add a management command to import regions from ESRI Shapfiles.

There is also instructions on how to import some regions from publicly
available geometry data at Helsinki Open Data services.

parkings/importers/regions.py

@@ -0,0 +1,55 @@
+import sys
+
+from django.contrib.gis.gdal import DataSource
+from django.contrib.gis.utils import LayerMapping
+
+from ..models import Region
+
+
+class ShapeFileToRegionImporter(object):
+    """
+    Importer from ESRI Shapefiles to Region model in the database.
+    """
+    field_mapping = {
+        # Region model field -> Field in the file
+        'geom': 'MULTIPOLYGON',
+    }
+
+    def __init__(self, filename, encoding='utf-8',
+                 output_stream=sys.stderr, verbose=True):
+        self.filename = filename
+        self.encoding = encoding
+        self.data_source = DataSource(filename, encoding=encoding)
+        self.output_stream = output_stream
+        self.verbose = verbose
+
+    def set_field_mapping(self, mapping):
+        self.field_mapping = dict(self.field_mapping, **mapping)
+
+    def get_layer_names(self):
+        return [layer.name for layer in self.data_source]
+
+    def get_layer_fields(self, name):
+        layer = self.data_source[self._get_layer_index(name)]
+        return layer.fields
+
+    def import_from_layer(self, layer_name):
+        layer_mapping = LayerMapping(
+            model=Region,
+            data=self.filename,
+            mapping=self.field_mapping,
+            layer=self._get_layer_index(layer_name),
+            encoding=self.encoding)
+        silent = (self.output_stream is None)
+        layer_mapping.save(
+            strict=True,
+            stream=self.output_stream,
+            silent=silent,
+            verbose=(not silent and self.verbose))
+
+    def _get_layer_index(self, name):
+        layer_names = self.get_layer_names()
+        try:
+            return layer_names.index(name)
+        except ValueError:
+            raise ValueError('No such layer: {!r}'.format(name))

parkings/management/commands/import_regions.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+r"""
+Import regions from file.
+
+Instructions to import data
+===========================
+
+From geoserver.hel.fi
+---------------------
+
+ 1. Download feature data from http://geoserver.hel.fi/geoserver/web/
+
+    - Click "Layer Preview"
+    - Select "Helsinki_osa_alueet"
+    - Download as WFS / Shapefile
+
+ 2. Unzip the file::
+
+      Helsinki_osa_alueet.zip
+
+ 3. Run this import command to the Shapefile::
+
+      ./manage.py import_regions \
+           --verbosity 2 \
+           --encoding latin1 --name nimi_fi \
+           Helsinki_osa_alueet.shp Helsinki_osa_alueet
+
+From ptp.hel.fi/avoindata
+-------------------------
+
+ 1. Download feature data from http://ptp.hel.fi/avoindata/ link
+    "Helsingin piirialuejako vuosilta 1995-2016 (zip) 24.2.2017
+    Paikkatietohakemisto GeoPackage (ETRS-GK25 (EPSG:3879))", or use
+    direct link:
+
+    http://ptp.hel.fi/avoindata/aineistot/HKI-aluejako-1995-2016-gpkg.zip
+
+ 2. Unzip the file
+
+ 3. Install tools for Geographic data conversion::
+
+      sudo apt install gdal-bin  # Works on Ubuntu
+
+ 4. Convert the GeoPackage data to ESRI Shapefile format::
+
+      ogr2ogr piirialuejako.shp piirialuejako-1995-2016.gpkg
+
+ 5. Run this import command to the converted shp file::
+
+      ./manage.py import_regions piirialuejako.shp osa_alue_2016
+"""
+import argparse
+
+from django.core.management.base import BaseCommand
+
+from ...importers.regions import ShapeFileToRegionImporter
+
+
+class Command(BaseCommand):
+    help = __doc__.strip().splitlines()[0]
+
+    def create_parser(self, prog_name, subcommand):
+        parser = super().create_parser(prog_name, subcommand)
+        parser.epilog = '\n'.join(__doc__.strip().splitlines()[2:])
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        return parser
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            'filename', type=str,
+            help=("Path to the ESRI Shapefile (*.shp) to import from"))
+
+        parser.add_argument(
+            'layer_name', type=str,
+            help=("Name of the layer to import or \"LIST\" to get a list"))
+
+        parser.add_argument('--encoding', type=str, default='utf-8')
+        parser.add_argument('--name-field', type=str, default='Nimi')
+
+    def handle(self, filename, layer_name, encoding, name_field,
+               *args, **options):
+        verbosity = int(options['verbosity'])
+        importer = ShapeFileToRegionImporter(
+            filename,
+            encoding=encoding,
+            output_stream=(self.stdout if verbosity > 0 else None),
+            verbose=(verbosity >= 2))
+
+        importer.set_field_mapping({'name': name_field})
+
+        if layer_name == 'LIST':
+            for name in importer.get_layer_names():
+                self.stdout.write(name)
+                for field in importer.get_layer_fields(name):
+                    self.stdout.write(' - {}'.format(field))
+        else:
+            importer.import_from_layer(layer_name)

parkings/tests/test-features.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

parkings/tests/test_region_importer.py

@@ -0,0 +1,94 @@
+import os
+
+import pytest
+
+from parkings.importers.regions import ShapeFileToRegionImporter
+from parkings.management.commands import import_regions
+from parkings.models import Region
+
+from .utils import call_mgmt_cmd_with_output
+
+directory = os.path.abspath(os.path.dirname(__file__))
+
+shp_path = os.path.join(directory, 'test-features.shp')
+
+
+def test_get_layer_names():
+    importer = ShapeFileToRegionImporter(shp_path)
+    assert importer.get_layer_names() == ['test-features']
+
+
+def test_get_layer_fields():
+    importer = ShapeFileToRegionImporter(shp_path)
+    assert importer.get_layer_fields('test-features') == [
+        'Name', 'descriptio', 'timestamp', 'begin', 'end',
+        'altitudeMo', 'tessellate', 'extrude', 'visibility',
+        'drawOrder', 'icon', 'DisplayNam', 'year']
+
+
+def test_get_layer_fields_invalid_layer_name():
+    importer = ShapeFileToRegionImporter(shp_path)
+    with pytest.raises(ValueError) as excinfo:
+        importer.get_layer_fields('invalid-layer-name')
+    assert str(excinfo.value) == "No such layer: 'invalid-layer-name'"
+
+
+@pytest.mark.django_db
+def test_import():
+    assert Region.objects.count() == 0
+    importer = ShapeFileToRegionImporter(shp_path)
+    importer.set_field_mapping({'name': 'DisplayNam'})
+    importer.import_from_layer('test-features')
+    check_imported_regions()
+
+
+@pytest.mark.django_db
+def test_management_command():
+    call_the_command(shp_path, 'test-features', name_field='DisplayNam')
+    check_imported_regions()
+
+    (stdout, stderr) = call_the_command(shp_path, 'LIST')
+    assert stdout.splitlines() == [
+        'test-features',
+        ' - Name',
+        ' - descriptio',
+        ' - timestamp',
+        ' - begin',
+        ' - end',
+        ' - altitudeMo',
+        ' - tessellate',
+        ' - extrude',
+        ' - visibility',
+        ' - drawOrder',
+        ' - icon',
+        ' - DisplayNam',
+        ' - year']
+    assert stdout.endswith('\n')
+    assert stderr == ''
+
+
+def call_the_command(*args, **kwargs):
+    (result, stdout, stderr) = call_mgmt_cmd_with_output(
+        import_regions.Command, *args, **kwargs)
+    assert result is None
+    return (stdout, stderr)
+
+
+def check_imported_regions():
+    assert Region.objects.count() == 2
+    (reg1, reg2) = list(Region.objects.order_by('name'))
+    assert reg1.name == 'Feature 1 - Center'
+    assert reg2.name == 'Feature 2 - North'
+    assert reg1.geom.coords == (((
+        (25494876.99362251, 6677378.512999998),
+        (25494929.966569535, 6677389.664999997),
+        (25495159.757339746, 6677117.191999998),
+        (25494819.72517978, 6677425.432),
+        (25494876.99362251, 6677378.512999998),
+    ),),)
+    assert reg2.geom.coords == (((
+        (25494360.817638688, 6684192.751999998),
+        (25494493.262506243, 6684249.482999996),
+        (25494337.233162273, 6684151.803999996),
+        (25494360.817638688, 6684192.751999998)
+    ),),)

parkings/tests/utils.py

@@ -0,0 +1,13 @@
+import io
+
+from django.core import management
+
+
+def call_mgmt_cmd_with_output(command_cls, *args, **kwargs):
+    assert issubclass(command_cls, management.BaseCommand)
+    stdout = io.StringIO()
+    stderr = io.StringIO()
+    cmd = command_cls(stdout=stdout, stderr=stderr)
+    assert isinstance(cmd, management.BaseCommand)
+    result = management.call_command(cmd, *args, **kwargs)
+    return (result, stdout.getvalue(), stderr.getvalue())