From e5ddd9e9c46c80df8fcccc831167d7722afc3d9e Mon Sep 17 00:00:00 2001
From: Tuomas Suutari <tuomas.suutari@anders.fi>
Date: Fri, 9 Feb 2018 11:33:14 +0200
Subject: [PATCH] Add Region importer

Add a management command to import regions from ESRI Shapfiles.

There is also instructions on how to import some regions from publicly
available geometry data at Helsinki Open Data services.
---
 parkings/importers/regions.py                 |  55 ++++++++++
 .../management/commands/import_regions.py     |  97 ++++++++++++++++++
 parkings/tests/test-features.dbf              | Bin 0 -> 1399 bytes
 parkings/tests/test-features.prj              |   1 +
 parkings/tests/test-features.shp              | Bin 0 -> 356 bytes
 parkings/tests/test-features.shx              | Bin 0 -> 116 bytes
 parkings/tests/test_region_importer.py        |  94 +++++++++++++++++
 parkings/tests/utils.py                       |  13 +++
 8 files changed, 260 insertions(+)
 create mode 100644 parkings/importers/regions.py
 create mode 100755 parkings/management/commands/import_regions.py
 create mode 100644 parkings/tests/test-features.dbf
 create mode 100644 parkings/tests/test-features.prj
 create mode 100644 parkings/tests/test-features.shp
 create mode 100644 parkings/tests/test-features.shx
 create mode 100644 parkings/tests/test_region_importer.py
 create mode 100644 parkings/tests/utils.py

diff --git a/parkings/importers/regions.py b/parkings/importers/regions.py
new file mode 100644
index 00000000..450e6dce
--- /dev/null
+++ b/parkings/importers/regions.py
@@ -0,0 +1,55 @@
+import sys
+
+from django.contrib.gis.gdal import DataSource
+from django.contrib.gis.utils import LayerMapping
+
+from ..models import Region
+
+
+class ShapeFileToRegionImporter(object):
+    """
+    Importer from ESRI Shapefiles to Region model in the database.
+    """
+    field_mapping = {
+        # Region model field -> Field in the file
+        'geom': 'MULTIPOLYGON',
+    }
+
+    def __init__(self, filename, encoding='utf-8',
+                 output_stream=sys.stderr, verbose=True):
+        self.filename = filename
+        self.encoding = encoding
+        self.data_source = DataSource(filename, encoding=encoding)
+        self.output_stream = output_stream
+        self.verbose = verbose
+
+    def set_field_mapping(self, mapping):
+        self.field_mapping = dict(self.field_mapping, **mapping)
+
+    def get_layer_names(self):
+        return [layer.name for layer in self.data_source]
+
+    def get_layer_fields(self, name):
+        layer = self.data_source[self._get_layer_index(name)]
+        return layer.fields
+
+    def import_from_layer(self, layer_name):
+        layer_mapping = LayerMapping(
+            model=Region,
+            data=self.filename,
+            mapping=self.field_mapping,
+            layer=self._get_layer_index(layer_name),
+            encoding=self.encoding)
+        silent = (self.output_stream is None)
+        layer_mapping.save(
+            strict=True,
+            stream=self.output_stream,
+            silent=silent,
+            verbose=(not silent and self.verbose))
+
+    def _get_layer_index(self, name):
+        layer_names = self.get_layer_names()
+        try:
+            return layer_names.index(name)
+        except ValueError:
+            raise ValueError('No such layer: {!r}'.format(name))
diff --git a/parkings/management/commands/import_regions.py b/parkings/management/commands/import_regions.py
new file mode 100755
index 00000000..a8f6852f
--- /dev/null
+++ b/parkings/management/commands/import_regions.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+r"""
+Import regions from file.
+
+Instructions to import data
+===========================
+
+From geoserver.hel.fi
+---------------------
+
+ 1. Download feature data from http://geoserver.hel.fi/geoserver/web/
+
+    - Click "Layer Preview"
+    - Select "Helsinki_osa_alueet"
+    - Download as WFS / Shapefile
+
+ 2. Unzip the file::
+
+      Helsinki_osa_alueet.zip
+
+ 3. Run this import command to the Shapefile::
+
+      ./manage.py import_regions \
+           --verbosity 2 \
+           --encoding latin1 --name nimi_fi \
+           Helsinki_osa_alueet.shp Helsinki_osa_alueet
+
+From ptp.hel.fi/avoindata
+-------------------------
+
+ 1. Download feature data from http://ptp.hel.fi/avoindata/ link
+    "Helsingin piirialuejako vuosilta 1995-2016 (zip) 24.2.2017
+    Paikkatietohakemisto GeoPackage (ETRS-GK25 (EPSG:3879))", or use
+    direct link:
+
+    http://ptp.hel.fi/avoindata/aineistot/HKI-aluejako-1995-2016-gpkg.zip
+
+ 2. Unzip the file
+
+ 3. Install tools for Geographic data conversion::
+
+      sudo apt install gdal-bin  # Works on Ubuntu
+
+ 4. Convert the GeoPackage data to ESRI Shapefile format::
+
+      ogr2ogr piirialuejako.shp piirialuejako-1995-2016.gpkg
+
+ 5. Run this import command to the converted shp file::
+
+      ./manage.py import_regions piirialuejako.shp osa_alue_2016
+"""
+import argparse
+
+from django.core.management.base import BaseCommand
+
+from ...importers.regions import ShapeFileToRegionImporter
+
+
+class Command(BaseCommand):
+    help = __doc__.strip().splitlines()[0]
+
+    def create_parser(self, prog_name, subcommand):
+        parser = super().create_parser(prog_name, subcommand)
+        parser.epilog = '\n'.join(__doc__.strip().splitlines()[2:])
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        return parser
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            'filename', type=str,
+            help=("Path to the ESRI Shapefile (*.shp) to import from"))
+
+        parser.add_argument(
+            'layer_name', type=str,
+            help=("Name of the layer to import or \"LIST\" to get a list"))
+
+        parser.add_argument('--encoding', type=str, default='utf-8')
+        parser.add_argument('--name-field', type=str, default='Nimi')
+
+    def handle(self, filename, layer_name, encoding, name_field,
+               *args, **options):
+        verbosity = int(options['verbosity'])
+        importer = ShapeFileToRegionImporter(
+            filename,
+            encoding=encoding,
+            output_stream=(self.stdout if verbosity > 0 else None),
+            verbose=(verbosity >= 2))
+
+        importer.set_field_mapping({'name': name_field})
+
+        if layer_name == 'LIST':
+            for name in importer.get_layer_names():
+                self.stdout.write(name)
+                for field in importer.get_layer_fields(name):
+                    self.stdout.write(' - {}'.format(field))
+        else:
+            importer.import_from_layer(layer_name)
diff --git a/parkings/tests/test-features.dbf b/parkings/tests/test-features.dbf
new file mode 100644
index 0000000000000000000000000000000000000000..8ac5d52e072504ca59955f7c18ac6bab2644f446
GIT binary patch
literal 1399
zcmd5(%WlFj5M0Wks1iTGDQBceF(){KxRLhOBQki^uH-yqFKzRA{f_=j9f%QB&drfJ
zmq~UqvyNxHf1Eu4!1w59<meu+0J$u%8?IY^x&Hzb+M}VGse+)+RG?*9)Bu~-I{D*2
z;2V{fSy0DPc}Gx}nu$*ecYXWOTGU!HLf!^F`LvGli;c6lZ$bToEFDM_H(^~F`T1-V
znjSwLtM2#0_p>2eYuUJK4(km{7kx(XeADE=BkRFX=T3!BzI~cs;ni5YMad_FiAd69
wjU}Urkt|lLWsICbGkTiMhFW@CsNa!_YqN7di^aa$g{wBN48KPSUSyg01&#u;QUCw|

literal 0
HcmV?d00001

diff --git a/parkings/tests/test-features.prj b/parkings/tests/test-features.prj
new file mode 100644
index 00000000..a30c00a5
--- /dev/null
+++ b/parkings/tests/test-features.prj
@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]
\ No newline at end of file
diff --git a/parkings/tests/test-features.shp b/parkings/tests/test-features.shp
new file mode 100644
index 0000000000000000000000000000000000000000..4d3976a0de1ea4dd7b4df5d3c4d7037399473544
GIT binary patch
literal 356
zcmZQzQ0HR64mQ1DW?*0i%I%&NSbOfNg~NmH_9Yckeh$ud;<d|PS~z5=&-uSn$<G11
zb`()YAkP70)-hKeDbW|0X7O#((7X&Z8>A4V4g~5uL+?cZ)vnK4o^%u_XUVHnHXX<}
zJ9%^c2_$#G+yyfmW)`}Bppi^KY=P{j&8!K=y+A*(%=8s*e}?cA$Q>*|3<5Qg9pSHl
U{+GBa<v0_IT3GnN%tBWW0F<U?Z~y=R

literal 0
HcmV?d00001

diff --git a/parkings/tests/test-features.shx b/parkings/tests/test-features.shx
new file mode 100644
index 0000000000000000000000000000000000000000..67b3b33882690306286303b117c1b518ff524f1a
GIT binary patch
literal 116
zcmZQzQ0HR64y;}<Gcd3M<#x{stUdSC!r?)8`;rPNKL=+!@!I7tEgUk`=lox(<mZ50
PJBp|gkOxv%2E-NsMT!w1

literal 0
HcmV?d00001

diff --git a/parkings/tests/test_region_importer.py b/parkings/tests/test_region_importer.py
new file mode 100644
index 00000000..984f0476
--- /dev/null
+++ b/parkings/tests/test_region_importer.py
@@ -0,0 +1,94 @@
+import os
+
+import pytest
+
+from parkings.importers.regions import ShapeFileToRegionImporter
+from parkings.management.commands import import_regions
+from parkings.models import Region
+
+from .utils import call_mgmt_cmd_with_output
+
+directory = os.path.abspath(os.path.dirname(__file__))
+
+shp_path = os.path.join(directory, 'test-features.shp')
+
+
+def test_get_layer_names():
+    importer = ShapeFileToRegionImporter(shp_path)
+    assert importer.get_layer_names() == ['test-features']
+
+
+def test_get_layer_fields():
+    importer = ShapeFileToRegionImporter(shp_path)
+    assert importer.get_layer_fields('test-features') == [
+        'Name', 'descriptio', 'timestamp', 'begin', 'end',
+        'altitudeMo', 'tessellate', 'extrude', 'visibility',
+        'drawOrder', 'icon', 'DisplayNam', 'year']
+
+
+def test_get_layer_fields_invalid_layer_name():
+    importer = ShapeFileToRegionImporter(shp_path)
+    with pytest.raises(ValueError) as excinfo:
+        importer.get_layer_fields('invalid-layer-name')
+    assert str(excinfo.value) == "No such layer: 'invalid-layer-name'"
+
+
+@pytest.mark.django_db
+def test_import():
+    assert Region.objects.count() == 0
+    importer = ShapeFileToRegionImporter(shp_path)
+    importer.set_field_mapping({'name': 'DisplayNam'})
+    importer.import_from_layer('test-features')
+    check_imported_regions()
+
+
+@pytest.mark.django_db
+def test_management_command():
+    call_the_command(shp_path, 'test-features', name_field='DisplayNam')
+    check_imported_regions()
+
+    (stdout, stderr) = call_the_command(shp_path, 'LIST')
+    assert stdout.splitlines() == [
+        'test-features',
+        ' - Name',
+        ' - descriptio',
+        ' - timestamp',
+        ' - begin',
+        ' - end',
+        ' - altitudeMo',
+        ' - tessellate',
+        ' - extrude',
+        ' - visibility',
+        ' - drawOrder',
+        ' - icon',
+        ' - DisplayNam',
+        ' - year']
+    assert stdout.endswith('\n')
+    assert stderr == ''
+
+
+def call_the_command(*args, **kwargs):
+    (result, stdout, stderr) = call_mgmt_cmd_with_output(
+        import_regions.Command, *args, **kwargs)
+    assert result is None
+    return (stdout, stderr)
+
+
+def check_imported_regions():
+    assert Region.objects.count() == 2
+    (reg1, reg2) = list(Region.objects.order_by('name'))
+    assert reg1.name == 'Feature 1 - Center'
+    assert reg2.name == 'Feature 2 - North'
+    assert reg1.geom.coords == (((
+        (25494876.99362251, 6677378.512999998),
+        (25494929.966569535, 6677389.664999997),
+        (25495159.757339746, 6677117.191999998),
+        (25494819.72517978, 6677425.432),
+        (25494876.99362251, 6677378.512999998),
+    ),),)
+    assert reg2.geom.coords == (((
+        (25494360.817638688, 6684192.751999998),
+        (25494493.262506243, 6684249.482999996),
+        (25494337.233162273, 6684151.803999996),
+        (25494360.817638688, 6684192.751999998)
+    ),),)
diff --git a/parkings/tests/utils.py b/parkings/tests/utils.py
new file mode 100644
index 00000000..f4fd558e
--- /dev/null
+++ b/parkings/tests/utils.py
@@ -0,0 +1,13 @@
+import io
+
+from django.core import management
+
+
+def call_mgmt_cmd_with_output(command_cls, *args, **kwargs):
+    assert issubclass(command_cls, management.BaseCommand)
+    stdout = io.StringIO()
+    stderr = io.StringIO()
+    cmd = command_cls(stdout=stdout, stderr=stderr)
+    assert isinstance(cmd, management.BaseCommand)
+    result = management.call_command(cmd, *args, **kwargs)
+    return (result, stdout.getvalue(), stderr.getvalue())