diff --git a/parkings/importers/regions.py b/parkings/importers/regions.py new file mode 100644 index 00000000..450e6dce --- /dev/null +++ b/parkings/importers/regions.py @@ -0,0 +1,55 @@ +import sys + +from django.contrib.gis.gdal import DataSource +from django.contrib.gis.utils import LayerMapping + +from ..models import Region + + +class ShapeFileToRegionImporter(object): + """ + Importer from ESRI Shapefiles to Region model in the database. + """ + field_mapping = { + # Region model field -> Field in the file + 'geom': 'MULTIPOLYGON', + } + + def __init__(self, filename, encoding='utf-8', + output_stream=sys.stderr, verbose=True): + self.filename = filename + self.encoding = encoding + self.data_source = DataSource(filename, encoding=encoding) + self.output_stream = output_stream + self.verbose = verbose + + def set_field_mapping(self, mapping): + self.field_mapping = dict(self.field_mapping, **mapping) + + def get_layer_names(self): + return [layer.name for layer in self.data_source] + + def get_layer_fields(self, name): + layer = self.data_source[self._get_layer_index(name)] + return layer.fields + + def import_from_layer(self, layer_name): + layer_mapping = LayerMapping( + model=Region, + data=self.filename, + mapping=self.field_mapping, + layer=self._get_layer_index(layer_name), + encoding=self.encoding) + silent = (self.output_stream is None) + layer_mapping.save( + strict=True, + stream=self.output_stream, + silent=silent, + verbose=(not silent and self.verbose)) + + def _get_layer_index(self, name): + layer_names = self.get_layer_names() + try: + return layer_names.index(name) + except ValueError: + raise ValueError('No such layer: {!r}'.format(name)) diff --git a/parkings/management/commands/import_regions.py b/parkings/management/commands/import_regions.py new file mode 100755 index 00000000..a8f6852f --- /dev/null +++ b/parkings/management/commands/import_regions.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +r""" +Import regions from file. + +Instructions to import data +=========================== + +From geoserver.hel.fi +--------------------- + + 1. Download feature data from http://geoserver.hel.fi/geoserver/web/ + + - Click "Layer Preview" + - Select "Helsinki_osa_alueet" + - Download as WFS / Shapefile + + 2. Unzip the file:: + + Helsinki_osa_alueet.zip + + 3. Run this import command to the Shapefile:: + + ./manage.py import_regions \ + --verbosity 2 \ + --encoding latin1 --name nimi_fi \ + Helsinki_osa_alueet.shp Helsinki_osa_alueet + +From ptp.hel.fi/avoindata +------------------------- + + 1. Download feature data from http://ptp.hel.fi/avoindata/ link + "Helsingin piirialuejako vuosilta 1995-2016 (zip) 24.2.2017 + Paikkatietohakemisto GeoPackage (ETRS-GK25 (EPSG:3879))", or use + direct link: + + http://ptp.hel.fi/avoindata/aineistot/HKI-aluejako-1995-2016-gpkg.zip + + 2. Unzip the file + + 3. Install tools for Geographic data conversion:: + + sudo apt install gdal-bin # Works on Ubuntu + + 4. Convert the GeoPackage data to ESRI Shapefile format:: + + ogr2ogr piirialuejako.shp piirialuejako-1995-2016.gpkg + + 5. Run this import command to the converted shp file:: + + ./manage.py import_regions piirialuejako.shp osa_alue_2016 +""" +import argparse + +from django.core.management.base import BaseCommand + +from ...importers.regions import ShapeFileToRegionImporter + + +class Command(BaseCommand): + help = __doc__.strip().splitlines()[0] + + def create_parser(self, prog_name, subcommand): + parser = super().create_parser(prog_name, subcommand) + parser.epilog = '\n'.join(__doc__.strip().splitlines()[2:]) + parser.formatter_class = argparse.RawDescriptionHelpFormatter + return parser + + def add_arguments(self, parser): + parser.add_argument( + 'filename', type=str, + help=("Path to the ESRI Shapefile (*.shp) to import from")) + + parser.add_argument( + 'layer_name', type=str, + help=("Name of the layer to import or \"LIST\" to get a list")) + + parser.add_argument('--encoding', type=str, default='utf-8') + parser.add_argument('--name-field', type=str, default='Nimi') + + def handle(self, filename, layer_name, encoding, name_field, + *args, **options): + verbosity = int(options['verbosity']) + importer = ShapeFileToRegionImporter( + filename, + encoding=encoding, + output_stream=(self.stdout if verbosity > 0 else None), + verbose=(verbosity >= 2)) + + importer.set_field_mapping({'name': name_field}) + + if layer_name == 'LIST': + for name in importer.get_layer_names(): + self.stdout.write(name) + for field in importer.get_layer_fields(name): + self.stdout.write(' - {}'.format(field)) + else: + importer.import_from_layer(layer_name) diff --git a/parkings/tests/test-features.dbf b/parkings/tests/test-features.dbf new file mode 100644 index 00000000..8ac5d52e Binary files /dev/null and b/parkings/tests/test-features.dbf differ diff --git a/parkings/tests/test-features.prj b/parkings/tests/test-features.prj new file mode 100644 index 00000000..a30c00a5 --- /dev/null +++ b/parkings/tests/test-features.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] \ No newline at end of file diff --git a/parkings/tests/test-features.shp b/parkings/tests/test-features.shp new file mode 100644 index 00000000..4d3976a0 Binary files /dev/null and b/parkings/tests/test-features.shp differ diff --git a/parkings/tests/test-features.shx b/parkings/tests/test-features.shx new file mode 100644 index 00000000..67b3b338 Binary files /dev/null and b/parkings/tests/test-features.shx differ diff --git a/parkings/tests/test_region_importer.py b/parkings/tests/test_region_importer.py new file mode 100644 index 00000000..984f0476 --- /dev/null +++ b/parkings/tests/test_region_importer.py @@ -0,0 +1,94 @@ +import os + +import pytest + +from parkings.importers.regions import ShapeFileToRegionImporter +from parkings.management.commands import import_regions +from parkings.models import Region + +from .utils import call_mgmt_cmd_with_output + +directory = os.path.abspath(os.path.dirname(__file__)) + +shp_path = os.path.join(directory, 'test-features.shp') + + +def test_get_layer_names(): + importer = ShapeFileToRegionImporter(shp_path) + assert importer.get_layer_names() == ['test-features'] + + +def test_get_layer_fields(): + importer = ShapeFileToRegionImporter(shp_path) + assert importer.get_layer_fields('test-features') == [ + 'Name', 'descriptio', 'timestamp', 'begin', 'end', + 'altitudeMo', 'tessellate', 'extrude', 'visibility', + 'drawOrder', 'icon', 'DisplayNam', 'year'] + + +def test_get_layer_fields_invalid_layer_name(): + importer = ShapeFileToRegionImporter(shp_path) + with pytest.raises(ValueError) as excinfo: + importer.get_layer_fields('invalid-layer-name') + assert str(excinfo.value) == "No such layer: 'invalid-layer-name'" + + +@pytest.mark.django_db +def test_import(): + assert Region.objects.count() == 0 + importer = ShapeFileToRegionImporter(shp_path) + importer.set_field_mapping({'name': 'DisplayNam'}) + importer.import_from_layer('test-features') + check_imported_regions() + + +@pytest.mark.django_db +def test_management_command(): + call_the_command(shp_path, 'test-features', name_field='DisplayNam') + check_imported_regions() + + (stdout, stderr) = call_the_command(shp_path, 'LIST') + assert stdout.splitlines() == [ + 'test-features', + ' - Name', + ' - descriptio', + ' - timestamp', + ' - begin', + ' - end', + ' - altitudeMo', + ' - tessellate', + ' - extrude', + ' - visibility', + ' - drawOrder', + ' - icon', + ' - DisplayNam', + ' - year'] + assert stdout.endswith('\n') + assert stderr == '' + + +def call_the_command(*args, **kwargs): + (result, stdout, stderr) = call_mgmt_cmd_with_output( + import_regions.Command, *args, **kwargs) + assert result is None + return (stdout, stderr) + + +def check_imported_regions(): + assert Region.objects.count() == 2 + (reg1, reg2) = list(Region.objects.order_by('name')) + assert reg1.name == 'Feature 1 - Center' + assert reg2.name == 'Feature 2 - North' + assert reg1.geom.coords == ((( + (25494876.99362251, 6677378.512999998), + (25494929.966569535, 6677389.664999997), + (25495159.757339746, 6677117.191999998), + (25494819.72517978, 6677425.432), + (25494876.99362251, 6677378.512999998), + ),),) + assert reg2.geom.coords == ((( + (25494360.817638688, 6684192.751999998), + (25494493.262506243, 6684249.482999996), + (25494337.233162273, 6684151.803999996), + (25494360.817638688, 6684192.751999998) + ),),) diff --git a/parkings/tests/utils.py b/parkings/tests/utils.py new file mode 100644 index 00000000..f4fd558e --- /dev/null +++ b/parkings/tests/utils.py @@ -0,0 +1,13 @@ +import io + +from django.core import management + + +def call_mgmt_cmd_with_output(command_cls, *args, **kwargs): + assert issubclass(command_cls, management.BaseCommand) + stdout = io.StringIO() + stderr = io.StringIO() + cmd = command_cls(stdout=stdout, stderr=stderr) + assert isinstance(cmd, management.BaseCommand) + result = management.call_command(cmd, *args, **kwargs) + return (result, stdout.getvalue(), stderr.getvalue())