From db38644babe2381dcc6599bef4c2607c4558ca65 Mon Sep 17 00:00:00 2001 From: Tuomas Suutari Date: Wed, 20 Sep 2017 09:01:13 +0300 Subject: [PATCH 1/9] settings: Change PAGE_SIZE to 100 Dashboard loads more smoothly with a bit bigger page size. --- parkkihubi/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parkkihubi/settings.py b/parkkihubi/settings.py index 4703d3e5..cb0be44b 100644 --- a/parkkihubi/settings.py +++ b/parkkihubi/settings.py @@ -159,7 +159,7 @@ 'ALLOWED_VERSIONS': ('v1',), 'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',), 'EXCEPTION_HANDLER': 'parkings.exception_handler.parkings_exception_handler', - 'PAGE_SIZE': 20, + 'PAGE_SIZE': 100, 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', 'TEST_REQUEST_DEFAULT_FORMAT': 'json', } From f47f4acde1e8ca161216539094781c382cffe20d Mon Sep 17 00:00:00 2001 From: Tuomas Suutari Date: Tue, 26 Sep 2017 15:58:19 +0300 Subject: [PATCH 2/9] Configure CORS headers for the dashboard Add Cross-Origin Resources Sharing (CORS) HTTP headers to the responses so that web browsers allow access to the API endpoints from different domains too. --- parkkihubi/settings.py | 3 +++ requirements.in | 1 + requirements.txt | 1 + 3 files changed, 5 insertions(+) diff --git a/parkkihubi/settings.py b/parkkihubi/settings.py index cb0be44b..49327707 100644 --- a/parkkihubi/settings.py +++ b/parkkihubi/settings.py @@ -71,6 +71,7 @@ 'django.contrib.staticfiles', 'django.contrib.gis', 'raven.contrib.django.raven_compat', + 'corsheaders', 'rest_framework', 'rest_framework.authtoken', 'rest_framework_gis', @@ -88,6 +89,7 @@ ############## MIDDLEWARE = [ 'django.contrib.sessions.middleware.SessionMiddleware', + 'corsheaders.middleware.CorsMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'parkkihubi.middleware.AdminTimezoneMiddleware', @@ -164,6 +166,7 @@ 'TEST_REQUEST_DEFAULT_FORMAT': 'json', } +CORS_ORIGIN_ALLOW_ALL = True ############## # Parkkihubi # diff --git a/requirements.in b/requirements.in index 7a35fe74..19bbef0c 100644 --- a/requirements.in +++ b/requirements.in @@ -23,3 +23,4 @@ owslib # Misc pytz +django-cors-headers diff --git a/requirements.txt b/requirements.txt index 28f1ba8c..89f0c18c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ certifi==2018.1.18 # via requests chardet==3.0.4 # via requests django==1.11.9 +django-cors-headers==2.1.0 django-environ==0.4.4 django-filter==1.1.0 djangorestframework==3.7.7 From 25b0cdb61d2d41459f7f0188981599332cfe16e5 Mon Sep 17 00:00:00 2001 From: Tuomas Suutari Date: Fri, 9 Feb 2018 11:32:59 +0200 Subject: [PATCH 3/9] Add Region model with migrations The dashboard will need also some bigger areas than the current "parking areas". Let's call these regions. Add a new model Region with name, geometry and capacity_estimate fields. Add a region field to Parking model so that it's fast to find parkings of a region. The capacity estimates of the regions will be calculated based on the capacity estimates of the parking areas. Add some methods to ParkingAreaQuerySet to help calculating these estimates. --- parkings/admin.py | 7 +- parkings/migrations/0017_region.py | 48 ++++++++++++ parkings/migrations/0018_parking_region.py | 29 ++++++++ parkings/models/__init__.py | 2 + parkings/models/parking.py | 24 ++++-- parkings/models/parking_area.py | 86 ++++++++++++++++++++++ parkings/models/region.py | 60 +++++++++++++++ parkings/tests/test_parking_area_model.py | 22 ++++++ parkings/tests/test_region_model.py | 73 ++++++++++++++++++ 9 files changed, 345 insertions(+), 6 deletions(-) create mode 100644 parkings/migrations/0017_region.py create mode 100644 parkings/migrations/0018_parking_region.py create mode 100644 parkings/models/region.py create mode 100644 parkings/tests/test_parking_area_model.py create mode 100644 parkings/tests/test_region_model.py diff --git a/parkings/admin.py b/parkings/admin.py index 4810466d..0cb8ae01 100644 --- a/parkings/admin.py +++ b/parkings/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin from django.contrib.gis.admin import OSMGeoAdmin -from .models import Operator, Parking, ParkingArea, ParkingTerminal +from .models import Operator, Parking, ParkingArea, ParkingTerminal, Region class OperatorAdmin(admin.ModelAdmin): @@ -18,6 +18,11 @@ class ParkingAdmin(OSMGeoAdmin): ordering = ('-created_at',) +@admin.register(Region) +class RegionAdmin(OSMGeoAdmin): + ordering = ('name',) + + class ParkingAreaAdmin(OSMGeoAdmin): ordering = ('origin_id',) diff --git a/parkings/migrations/0017_region.py b/parkings/migrations/0017_region.py new file mode 100644 index 00000000..4439771c --- /dev/null +++ b/parkings/migrations/0017_region.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-09-20 02:22 +from __future__ import unicode_literals + +import uuid + +import django.contrib.gis.db.models.fields +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parkings', '0016_normalized_reg_num_noblank'), + ] + + operations = [ + migrations.CreateModel( + name='Region', + fields=[ + ('created_at', models.DateTimeField( + auto_now_add=True, verbose_name='time created')), + ('modified_at', models.DateTimeField( + auto_now=True, verbose_name='time modified')), + ('id', models.UUIDField( + default=uuid.uuid4, editable=False, + primary_key=True, serialize=False)), + ('name', models.CharField( + blank=True, max_length=200, verbose_name='name')), + ('geom', django.contrib.gis.db.models.fields.MultiPolygonField( + srid=3879, verbose_name='geometry')), + ], + options={ + 'verbose_name_plural': 'regions', + 'verbose_name': 'region', + }, + ), + migrations.AddField( + model_name='parkingarea', + name='region', + field=models.ForeignKey( + blank=True, null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='areas', + to='parkings.Region', verbose_name='region'), + ), + ] diff --git a/parkings/migrations/0018_parking_region.py b/parkings/migrations/0018_parking_region.py new file mode 100644 index 00000000..3111a548 --- /dev/null +++ b/parkings/migrations/0018_parking_region.py @@ -0,0 +1,29 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('parkings', '0017_region'), + ] + + operations = [ + migrations.RemoveField( + model_name='parkingarea', + name='region', ), + migrations.AddField( + model_name='parking', + name='region', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='parkings', + to='parkings.Region', + verbose_name='region')), + migrations.AddField( + model_name='region', + name='capacity_estimate', + field=models.PositiveIntegerField( + blank=True, null=True, verbose_name='capacity estimate')), + ] diff --git a/parkings/models/__init__.py b/parkings/models/__init__.py index 9b0fddc7..a2ba0c57 100644 --- a/parkings/models/__init__.py +++ b/parkings/models/__init__.py @@ -2,6 +2,7 @@ from .parking import Parking, ParkingQuerySet from .parking_area import ParkingArea from .parking_terminal import ParkingTerminal +from .region import Region __all__ = [ 'Operator', @@ -9,4 +10,5 @@ 'ParkingArea', 'ParkingTerminal', 'ParkingQuerySet', + 'Region', ] diff --git a/parkings/models/parking.py b/parkings/models/parking.py index 42411062..6eacb892 100644 --- a/parkings/models/parking.py +++ b/parkings/models/parking.py @@ -9,6 +9,7 @@ from parkings.models.parking_area import ParkingArea from .parking_terminal import ParkingTerminal +from .region import Region Q = models.Q @@ -44,6 +45,10 @@ class Parking(TimestampedModelMixin, UUIDPrimaryKeyMixin): VALID = 'valid' NOT_VALID = 'not_valid' + region = models.ForeignKey( + Region, null=True, blank=True, on_delete=models.SET_NULL, + related_name='parkings', verbose_name=_("region"), + ) parking_area = models.ForeignKey( ParkingArea, on_delete=models.SET_NULL, verbose_name=_("parking area"), related_name='parkings', null=True, blank=True, @@ -106,6 +111,11 @@ def get_state(self): return Parking.VALID + def get_region(self): + if not self.location: + return None + return Region.objects.filter(geom__intersects=self.location).first() + def get_closest_area(self, max_distance=50): if self.location: location = self.location @@ -117,7 +127,7 @@ def get_closest_area(self, max_distance=50): ).filter(distance__lte=max_distance).order_by('distance').first() return closest_area - def save(self, *args, **kwargs): + def save(self, update_fields=None, *args, **kwargs): if not self.terminal and self.terminal_number: self.terminal = ParkingTerminal.objects.filter( number=_try_cast_int(self.terminal_number)).first() @@ -125,12 +135,16 @@ def save(self, *args, **kwargs): if self.terminal and not self.location: self.location = self.terminal.location - self.parking_area = self.get_closest_area() + if update_fields is None or 'region' in update_fields: + self.region = self.get_region() + if update_fields is None or 'parking_area' in update_fields: + self.parking_area = self.get_closest_area() - self.normalized_reg_num = ( - self.normalize_reg_num(self.registration_number)) + if update_fields is None or 'normalized_reg_num' in update_fields: + self.normalized_reg_num = ( + self.normalize_reg_num(self.registration_number)) - super(Parking, self).save(*args, **kwargs) + super(Parking, self).save(update_fields=update_fields, *args, **kwargs) @classmethod def normalize_reg_num(cls, registration_number): diff --git a/parkings/models/parking_area.py b/parkings/models/parking_area.py index c096dd02..f6ca46d5 100644 --- a/parkings/models/parking_area.py +++ b/parkings/models/parking_area.py @@ -1,8 +1,68 @@ from django.contrib.gis.db import models +from django.contrib.gis.db.models.functions import Area +from django.db.models import Func, Sum from django.utils.translation import ugettext_lazy as _ from parkings.models.mixins import TimestampedModelMixin, UUIDPrimaryKeyMixin +# Estimated number of parking spots per square meter (m^2) +PARKING_SPOTS_PER_SQ_M = 0.07328 + + +class Round(Func): + """ + Function for rounding in SQL using Django ORM. + """ + function = 'ROUND' + + +class ParkingAreaQuerySet(models.QuerySet): + def inside_region(self, region): + """ + Filter to parking areas which are inside given region. + + :type region: .region.Region + :rtype: ParkingAreaQuerySet + """ + return self.filter(geom__coveredby=region.geom) + + def intersecting_region(self, region): + """ + Filter to parking areas which intersect with given region. + + :type region: .region.Region + :rtype: ParkingAreaQuerySet + """ + return self.filter(geom__intersects=region.geom) + + @property + def total_estimated_capacity(self): + """ + Estimated number of parking spots in this queryset. + + This is the sum of `parking_area.estimated_capacity` of each + `parking_area` in this queryset. The value is calculated in the + database to make it faster compared to calculating the sum in + Python. + + :rtype: int + """ + defined_qs = self.exclude(capacity_estimate=None) + undefined_qs = self.filter(capacity_estimate=None) + total_defined = defined_qs.sum_of_capacity_estimates or 0 + total_by_area = undefined_qs.estimate_total_capacity_by_areas() + return total_defined + total_by_area + + @property + def sum_of_capacity_estimates(self): + return self.aggregate(val=Sum('capacity_estimate'))['val'] + + def estimate_total_capacity_by_areas(self): + spots = ( + self.annotate(spots=Round(PARKING_SPOTS_PER_SQ_M * Area('geom'))) + .aggregate(val=Sum('spots')))['val'] + return int(spots.sq_m) if spots else 0 + class ParkingArea(TimestampedModelMixin, UUIDPrimaryKeyMixin): # This is for whatever ID the external system that this parking lot was @@ -34,9 +94,35 @@ class ParkingArea(TimestampedModelMixin, UUIDPrimaryKeyMixin): blank=True, ) + objects = ParkingAreaQuerySet.as_manager() + class Meta: verbose_name = _('parking area') verbose_name_plural = _('parking areas') def __str__(self): return 'Parking Area %s' % str(self.origin_id) + + @property + def estimated_capacity(self): + """ + Estimated number of parking spots in this parking area. + + If there is a capacity estimate specified for this area in the + `capacity_estimate` field, return it, otherwise return a + capacity estimate based on the area (m^2) of this parking area. + + :rtype: int + """ + return ( + self.capacity_estimate if self.capacity_estimate is not None + else self.estimate_capacity_by_area()) + + def estimate_capacity_by_area(self): + """ + Estimate number of parking spots in this parking area by m^2. + + :rtype: int + """ + assert self.geom.srs.units == (1.0, 'metre') + return int(round(self.geom.area * PARKING_SPOTS_PER_SQ_M)) diff --git a/parkings/models/region.py b/parkings/models/region.py new file mode 100644 index 00000000..7f35ed5b --- /dev/null +++ b/parkings/models/region.py @@ -0,0 +1,60 @@ +from django.contrib.gis.db import models as gis_models +from django.contrib.gis.db.models.functions import Intersection +from django.db import models +from django.db.models import Case, Count, Q, When +from django.utils import timezone +from django.utils.translation import ugettext_lazy as _ + +from .mixins import TimestampedModelMixin, UUIDPrimaryKeyMixin +from .parking_area import ParkingArea + + +class RegionQuerySet(models.QuerySet): + def with_parking_count(self, at_time=None): + time = at_time if at_time else timezone.now() + valid_parkings_q = ( + Q(parkings__time_start__lte=time) & + (Q(parkings__time_end__gte=time) | Q(parkings__time_end=None))) + return self.annotate( + parking_count=Count(Case(When(valid_parkings_q, then=1)))) + + +class Region(TimestampedModelMixin, UUIDPrimaryKeyMixin): + name = models.CharField(max_length=200, blank=True, verbose_name=_("name")) + geom = gis_models.MultiPolygonField(srid=3879, verbose_name=_("geometry")) + capacity_estimate = models.PositiveIntegerField( + blank=True, null=True, + verbose_name=_("capacity estimate"), + ) + + objects = RegionQuerySet.as_manager() + + class Meta: + verbose_name = _("region") + verbose_name_plural = _("regions") + + def __str__(self): + return self.name or str(_("Unnamed region")) + + def save(self, *args, **kwargs): + self.capacity_estimate = self.calculate_capacity_estimate() + super().save(*args, **kwargs) + + def calculate_capacity_estimate(self): + """ + Calculate capacity estimate of this region from parking areas. + + :rtype: int + """ + areas = ParkingArea.objects.all() + covered_areas = areas.inside_region(self) + partial_areas = ( + areas.intersecting_region(self).exclude(pk__in=covered_areas) + .annotate(intsect=Intersection('geom', self.geom))) + sum_covered = covered_areas.total_estimated_capacity + sum_partials = sum( + # Scale the estimated capacity according to ratio of area of + # the intersection and total area + int(round(x.estimated_capacity * x.intsect.area / x.geom.area)) + for x in partial_areas) + return sum_covered + sum_partials diff --git a/parkings/tests/test_parking_area_model.py b/parkings/tests/test_parking_area_model.py new file mode 100644 index 00000000..9622e311 --- /dev/null +++ b/parkings/tests/test_parking_area_model.py @@ -0,0 +1,22 @@ +import pytest + +from parkings.models import ParkingArea + + +def test_str(): + assert str(ParkingArea(origin_id='TEST_ID')) == 'Parking Area TEST_ID' + + +@pytest.mark.django_db +def test_estimated_capacity(parking_area): + parking_area.capacity_estimate = 123 + assert parking_area.estimated_capacity == 123 + parking_area.capacity_estimate = None + by_area = parking_area.estimate_capacity_by_area() + assert parking_area.estimated_capacity == by_area + + +@pytest.mark.django_db +def test_estimate_capacity_by_area(parking_area): + assert parking_area.estimate_capacity_by_area() == int( + round(parking_area.geom.area * 0.07328)) diff --git a/parkings/tests/test_region_model.py b/parkings/tests/test_region_model.py new file mode 100644 index 00000000..82b73216 --- /dev/null +++ b/parkings/tests/test_region_model.py @@ -0,0 +1,73 @@ +import pytest +from django.contrib.gis.geos import MultiPolygon, Polygon + +from parkings.models import ParkingArea, Region + + +def test_str(): + assert str(Region(name='Foobba')) == 'Foobba' + assert str(Region(name=None)) == 'Unnamed region' + assert str(Region(name='')) == 'Unnamed region' + assert isinstance(str(Region()), str) + + +@pytest.mark.django_db +def test_calculate_capacity_estimate(): + """ + Test Region capacity estimate calculation. + + Construct the following geometries: + + - reg = Region to estimate + - pa1 = Parking area 1, capacity=3, inside reg + - pa2 = Parking area 2, capacity=4, inside reg + - pa3 = Parking area 3, capacity=15, 1/3 of area inside reg + - pa4 = Parking area 4, capacity=20, outside reg + + Total capacity should be 3 + 4 + (1/3)*15 = 12. + + Same as a graph:: + + +-------------------------------+ + | reg | + | +--------+ +---------+ +---------------------+ +-------... + | | pa1 | | pa2 | | | pa3 | | pa4 + | +--------+ +---------+ +---------------------+ +-------... + | capacity=3 capacity=4 | capacity=15 capacity=20 + | | + +-------------------------------+ + """ + def rect(topleft, width, height): + """ + Construct a rectangle as multipolygon. + """ + (ax, ay) = topleft # left = ax, top = ay + (bx, by) = (ax + width, ay + height) # right = bx, bottom = by + return MultiPolygon(Polygon( + [(ax, ay), (ax, by), (bx, by), (bx, ay), (ax, ay)])) + + reg = Region(geom=rect((0, 0), 100, 14)) + pa1 = ParkingArea(geom=rect((2, 2), 20, 10), capacity_estimate=3) + pa2 = ParkingArea(geom=rect((30, 2), 20, 10), capacity_estimate=4) + pa3 = ParkingArea(geom=rect((60, 2), 120, 10), capacity_estimate=15) + pa4 = ParkingArea(geom=rect((200, 2), 40, 10), capacity_estimate=20) + + assert reg.geom.area == 1400 + assert pa1.geom.area == 200 + assert pa2.geom.area == 200 + assert pa3.geom.area == 1200 + assert pa4.geom.area == 400 + + assert reg.geom.intersection(pa1.geom).area == pa1.geom.area + assert reg.geom.intersection(pa2.geom).area == pa2.geom.area + assert reg.geom.intersection(pa3.geom).area == pa3.geom.area / 3 + assert reg.geom.intersection(pa4.geom).area == 0 + + # Save them to the database + reg.save() + for (n, pa) in enumerate([pa1, pa2, pa3, pa4]): + pa.origin_id = 'PA-{}'.format(n) + pa.save() + + # And finally, check the result of the calculation is correct + assert reg.calculate_capacity_estimate() == 12 From e5ddd9e9c46c80df8fcccc831167d7722afc3d9e Mon Sep 17 00:00:00 2001 From: Tuomas Suutari Date: Fri, 9 Feb 2018 11:33:14 +0200 Subject: [PATCH 4/9] Add Region importer Add a management command to import regions from ESRI Shapfiles. There is also instructions on how to import some regions from publicly available geometry data at Helsinki Open Data services. --- parkings/importers/regions.py | 55 ++++++++++ .../management/commands/import_regions.py | 97 ++++++++++++++++++ parkings/tests/test-features.dbf | Bin 0 -> 1399 bytes parkings/tests/test-features.prj | 1 + parkings/tests/test-features.shp | Bin 0 -> 356 bytes parkings/tests/test-features.shx | Bin 0 -> 116 bytes parkings/tests/test_region_importer.py | 94 +++++++++++++++++ parkings/tests/utils.py | 13 +++ 8 files changed, 260 insertions(+) create mode 100644 parkings/importers/regions.py create mode 100755 parkings/management/commands/import_regions.py create mode 100644 parkings/tests/test-features.dbf create mode 100644 parkings/tests/test-features.prj create mode 100644 parkings/tests/test-features.shp create mode 100644 parkings/tests/test-features.shx create mode 100644 parkings/tests/test_region_importer.py create mode 100644 parkings/tests/utils.py diff --git a/parkings/importers/regions.py b/parkings/importers/regions.py new file mode 100644 index 00000000..450e6dce --- /dev/null +++ b/parkings/importers/regions.py @@ -0,0 +1,55 @@ +import sys + +from django.contrib.gis.gdal import DataSource +from django.contrib.gis.utils import LayerMapping + +from ..models import Region + + +class ShapeFileToRegionImporter(object): + """ + Importer from ESRI Shapefiles to Region model in the database. + """ + field_mapping = { + # Region model field -> Field in the file + 'geom': 'MULTIPOLYGON', + } + + def __init__(self, filename, encoding='utf-8', + output_stream=sys.stderr, verbose=True): + self.filename = filename + self.encoding = encoding + self.data_source = DataSource(filename, encoding=encoding) + self.output_stream = output_stream + self.verbose = verbose + + def set_field_mapping(self, mapping): + self.field_mapping = dict(self.field_mapping, **mapping) + + def get_layer_names(self): + return [layer.name for layer in self.data_source] + + def get_layer_fields(self, name): + layer = self.data_source[self._get_layer_index(name)] + return layer.fields + + def import_from_layer(self, layer_name): + layer_mapping = LayerMapping( + model=Region, + data=self.filename, + mapping=self.field_mapping, + layer=self._get_layer_index(layer_name), + encoding=self.encoding) + silent = (self.output_stream is None) + layer_mapping.save( + strict=True, + stream=self.output_stream, + silent=silent, + verbose=(not silent and self.verbose)) + + def _get_layer_index(self, name): + layer_names = self.get_layer_names() + try: + return layer_names.index(name) + except ValueError: + raise ValueError('No such layer: {!r}'.format(name)) diff --git a/parkings/management/commands/import_regions.py b/parkings/management/commands/import_regions.py new file mode 100755 index 00000000..a8f6852f --- /dev/null +++ b/parkings/management/commands/import_regions.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +r""" +Import regions from file. + +Instructions to import data +=========================== + +From geoserver.hel.fi +--------------------- + + 1. Download feature data from http://geoserver.hel.fi/geoserver/web/ + + - Click "Layer Preview" + - Select "Helsinki_osa_alueet" + - Download as WFS / Shapefile + + 2. Unzip the file:: + + Helsinki_osa_alueet.zip + + 3. Run this import command to the Shapefile:: + + ./manage.py import_regions \ + --verbosity 2 \ + --encoding latin1 --name nimi_fi \ + Helsinki_osa_alueet.shp Helsinki_osa_alueet + +From ptp.hel.fi/avoindata +------------------------- + + 1. Download feature data from http://ptp.hel.fi/avoindata/ link + "Helsingin piirialuejako vuosilta 1995-2016 (zip) 24.2.2017 + Paikkatietohakemisto GeoPackage (ETRS-GK25 (EPSG:3879))", or use + direct link: + + http://ptp.hel.fi/avoindata/aineistot/HKI-aluejako-1995-2016-gpkg.zip + + 2. Unzip the file + + 3. Install tools for Geographic data conversion:: + + sudo apt install gdal-bin # Works on Ubuntu + + 4. Convert the GeoPackage data to ESRI Shapefile format:: + + ogr2ogr piirialuejako.shp piirialuejako-1995-2016.gpkg + + 5. Run this import command to the converted shp file:: + + ./manage.py import_regions piirialuejako.shp osa_alue_2016 +""" +import argparse + +from django.core.management.base import BaseCommand + +from ...importers.regions import ShapeFileToRegionImporter + + +class Command(BaseCommand): + help = __doc__.strip().splitlines()[0] + + def create_parser(self, prog_name, subcommand): + parser = super().create_parser(prog_name, subcommand) + parser.epilog = '\n'.join(__doc__.strip().splitlines()[2:]) + parser.formatter_class = argparse.RawDescriptionHelpFormatter + return parser + + def add_arguments(self, parser): + parser.add_argument( + 'filename', type=str, + help=("Path to the ESRI Shapefile (*.shp) to import from")) + + parser.add_argument( + 'layer_name', type=str, + help=("Name of the layer to import or \"LIST\" to get a list")) + + parser.add_argument('--encoding', type=str, default='utf-8') + parser.add_argument('--name-field', type=str, default='Nimi') + + def handle(self, filename, layer_name, encoding, name_field, + *args, **options): + verbosity = int(options['verbosity']) + importer = ShapeFileToRegionImporter( + filename, + encoding=encoding, + output_stream=(self.stdout if verbosity > 0 else None), + verbose=(verbosity >= 2)) + + importer.set_field_mapping({'name': name_field}) + + if layer_name == 'LIST': + for name in importer.get_layer_names(): + self.stdout.write(name) + for field in importer.get_layer_fields(name): + self.stdout.write(' - {}'.format(field)) + else: + importer.import_from_layer(layer_name) diff --git a/parkings/tests/test-features.dbf b/parkings/tests/test-features.dbf new file mode 100644 index 0000000000000000000000000000000000000000..8ac5d52e072504ca59955f7c18ac6bab2644f446 GIT binary patch literal 1399 zcmd5(%WlFj5M0Wks1iTGDQBceF(){KxRLhOBQki^uH-yqFKzRA{f_=j9f%QB&drfJ zmq~UqvyNxHf1Eu4!1w592eYuUJK4(km{7kx(XeADE=BkRFX=T3!BzI~cs;ni5YMad_FiAd69 wjU}Urkt|lLWsICbGkTiMhFW@CsNa!_YqN7di^aa$g{wBN48KPSUSyg01&#u;QUCw| literal 0 HcmV?d00001 diff --git a/parkings/tests/test-features.prj b/parkings/tests/test-features.prj new file mode 100644 index 00000000..a30c00a5 --- /dev/null +++ b/parkings/tests/test-features.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] \ No newline at end of file diff --git a/parkings/tests/test-features.shp b/parkings/tests/test-features.shp new file mode 100644 index 0000000000000000000000000000000000000000..4d3976a0de1ea4dd7b4df5d3c4d7037399473544 GIT binary patch literal 356 zcmZQzQ0HR64mQ1DW?*0i%I%&NSbOfNg~NmH_9Yckeh$ud;A4V4g~5uL+?cZ)vnK4o^%u_XUVHnHXX<} zJ9%^c2_$#G+yyfmW)`}Bppi^KY=P{j&8!K=y+A*(%=8s*e}?cA$Q>*|3<5Qg9pSHl U{+GBa Date: Fri, 9 Feb 2018 11:33:22 +0200 Subject: [PATCH 5/9] Add command for filling regions of parkings New parkings with a location will get the region field filled on their `save` method, but the existing parkings will have to be processed separately. Add a management command for that task. The command divides the parkings to time intervals so that each time interval has about 20000 parkings. Then each time interval is processed separately in a transaction. This should (a) limit the amount of resources the command consumes and (b) make it easy to abort the command, if it takes too much resources, and then continue later where it left off. --- .../commands/fill_parking_regions.py | 71 ++++++++++++++++ .../test_fill_parking_regions_command.py | 80 +++++++++++++++++++ parkings/tests/utils.py | 36 +++++++++ 3 files changed, 187 insertions(+) create mode 100755 parkings/management/commands/fill_parking_regions.py create mode 100644 parkings/tests/test_fill_parking_regions_command.py diff --git a/parkings/management/commands/fill_parking_regions.py b/parkings/management/commands/fill_parking_regions.py new file mode 100755 index 00000000..465f816f --- /dev/null +++ b/parkings/management/commands/fill_parking_regions.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +""" +Fill regions to Parking objects. +""" +import datetime + +from django.core.management.base import BaseCommand +from django.db import transaction + +from ...models import Parking, Region + + +class Command(BaseCommand): + help = __doc__.strip().splitlines()[0] + + def add_arguments(self, parser): + parser.add_argument( + 'block_size_target', type=int, nargs='?', default=20000, + help=( + "Block size target, " + "i.e. the number of parkings to process at time")) + + def handle(self, block_size_target, *args, **options): + verbosity = int(options['verbosity']) + silent = (verbosity == 0) + show_info = (self._print_and_flush if not silent else self._null_print) + + regions = Region.objects.all() + parkings = ( + Parking.objects + .exclude(location=None) + .filter(region=None) + .order_by('created_at')) + count = parkings.count() + + if not count: + show_info("Nothing to do") + return + + block_count = int(max(count / block_size_target, 1)) + start = parkings.first().created_at.replace(microsecond=0, second=0) + end = parkings.last().created_at + block_seconds = int((end - start).total_seconds() / block_count) + 1 + block_span = datetime.timedelta(seconds=block_seconds) + + for block_num in range(block_count): + block_start = start + (block_num * block_span) + block_end = start + ((block_num + 1) * block_span) + block = parkings.filter( + created_at__gte=block_start, + created_at__lt=block_end) + block_size = block.count() + + show_info( + "Processing block {:5d}/{:5d}, size {:6d}, {}--{}".format( + block_num + 1, block_count, block_size, + block_start, block_end), ending='') + + with transaction.atomic(): + for (n, region) in enumerate(regions): + if n % 10 == 0: + show_info('.', ending='') + in_region = block.filter(location__intersects=region.geom) + in_region.update(region=region) + show_info('', ending='\n') # Print end of line + + def _print_and_flush(self, *args, ending='\n'): + self.stdout.write(*args, ending=ending) + + def _null_print(self, *args, ending='\n'): + pass diff --git a/parkings/tests/test_fill_parking_regions_command.py b/parkings/tests/test_fill_parking_regions_command.py new file mode 100644 index 00000000..8cb3c506 --- /dev/null +++ b/parkings/tests/test_fill_parking_regions_command.py @@ -0,0 +1,80 @@ +import re + +import pytest +from dateutil.parser import parse as parse_date + +from parkings.management.commands import fill_parking_regions + +from .utils import ( + call_mgmt_cmd_with_output, create_parkings_and_regions, intersects, + intersects_with_any) + + +@pytest.mark.django_db +def test_fill_parking_regions_mgmt_cmd(): + (parkings, regions) = create_parkings_and_regions() + + # Clear the regions + for parking in parkings: + # First save without a location to prevent region being autofilled + old_location = parking.location + parking.location = None + parking.region = None + parking.save() + # Then fill the location back, but save only location + parking.location = old_location + parking.save(update_fields=['location']) + + assert all(parking.region is None for parking in parkings) + + # Call the command with output streams attached + target_block_size = 5 + (stdout, stderr) = call_the_command(target_block_size) + + # Check the results + for parking in parkings: + parking.refresh_from_db() + if parking.location and intersects_with_any(parking.location, regions): + assert parking.region is not None + assert intersects(parking.location, parking.region) + else: + assert parking.region is None + + # Check the outputted lines + block_count = len(parkings) // target_block_size + for (n, line) in enumerate(stdout.splitlines(), 1): + match = re.match( + '^Processing block +(\d+)/ *(\d+), size +(\d+), ([^.]*)(\.+)$', + line) + assert match, 'Invalid output line {}: {!r}'.format(n, line) + assert match.group(1) == str(n) + assert match.group(2) == str(block_count) + assert 0 <= int(match.group(3)) <= len(parkings) + (start_str, end_str) = match.group(4).split('--') + block_start = parse_date(start_str) + block_end = parse_date(end_str) + assert block_start <= block_end + assert len(match.group(5)) == len(regions) // 10 + assert stderr == '' + + # Check that the command doesn't do anything if all parkings with a + # location have region filled + for parking in parkings: + if not parking.region: + parking.location = None + parking.save() + (stdout, stderr) = call_the_command(target_block_size) + assert stdout == 'Nothing to do\n' + assert stderr == '' + + # And finally check that it doesn't print anything when verbosity=0 + (stdout, stderr) = call_the_command(target_block_size, verbosity=0) + assert stdout == '' + assert stderr == '' + + +def call_the_command(*args, **kwargs): + (result, stdout, stderr) = call_mgmt_cmd_with_output( + fill_parking_regions.Command, *args, **kwargs) + assert result is None + return (stdout, stderr) diff --git a/parkings/tests/utils.py b/parkings/tests/utils.py index f4fd558e..fefd154f 100644 --- a/parkings/tests/utils.py +++ b/parkings/tests/utils.py @@ -2,6 +2,8 @@ from django.core import management +from parkings.factories import ParkingFactory, RegionFactory + def call_mgmt_cmd_with_output(command_cls, *args, **kwargs): assert issubclass(command_cls, management.BaseCommand) @@ -11,3 +13,37 @@ def call_mgmt_cmd_with_output(command_cls, *args, **kwargs): assert isinstance(cmd, management.BaseCommand) result = management.call_command(cmd, *args, **kwargs) return (result, stdout.getvalue(), stderr.getvalue()) + + +def create_parkings_and_regions(parking_count=100, region_count=20): + regions = RegionFactory.create_batch(region_count) + parkings = ParkingFactory.create_batch(parking_count) + + centroids = [region.geom.centroid for region in regions] + touching_points = [p for p in centroids if intersects_with_any(p, regions)] + + # Make sure that some of the parkings are inside the regions + for (point, parking) in zip(touching_points, parkings): + parking.location = point + parking.save() + + for parking in parkings: # pragma: no cover + if intersects_with_any(parking.location, regions): + assert parking.region + assert intersects(parking.location, parking.region) + else: + assert parking.region is None + + return (parkings, regions) + + +def intersects_with_any(point, regions): + assert regions + p = point.transform(regions[0].geom.srid, clone=True) + assert all(x.geom.srid == p.srid for x in regions) + return any(p.intersects(x.geom) for x in regions) + + +def intersects(point, region): + geom = region.geom + return point.transform(geom.srid, clone=True).intersects(geom) From 0c3042570c3a502c5c01ac5f6e4e319e15736a23 Mon Sep 17 00:00:00 2001 From: Tuomas Suutari Date: Fri, 9 Feb 2018 11:33:43 +0200 Subject: [PATCH 6/9] Reconfigure API authentication classes Configure the ApiKeyAuthentication via DEFAULT_AUTHENTICATION_CLASSES setting and only set permission classes per API views. This allows having SessionAuthentication and BasicAuthentication in the development environment (in DEBUG mode), which helps the development. --- parkings/api/enforcement/operator.py | 4 +--- parkings/api/enforcement/valid_parking.py | 4 +--- parkings/api/operator/parking.py | 4 +--- parkings/api/public/parking_area.py | 3 ++- parkings/api/public/parking_area_statistics.py | 3 ++- parkings/api/public/urls.py | 14 ++++++++++++-- parkkihubi/settings.py | 14 +++++++++++++- 7 files changed, 32 insertions(+), 14 deletions(-) diff --git a/parkings/api/enforcement/operator.py b/parkings/api/enforcement/operator.py index b495e37c..f83c36f1 100644 --- a/parkings/api/enforcement/operator.py +++ b/parkings/api/enforcement/operator.py @@ -1,6 +1,5 @@ from rest_framework import permissions, serializers, viewsets -from ...authentication import ApiKeyAuthentication from ...models import Operator @@ -16,7 +15,6 @@ class Meta: class OperatorViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.IsAdminUser] queryset = Operator.objects.order_by('name') serializer_class = OperatorSerializer - authentication_classes = [ApiKeyAuthentication] - permission_classes = [permissions.IsAdminUser] diff --git a/parkings/api/enforcement/valid_parking.py b/parkings/api/enforcement/valid_parking.py index b2179360..1042bf6e 100644 --- a/parkings/api/enforcement/valid_parking.py +++ b/parkings/api/enforcement/valid_parking.py @@ -6,7 +6,6 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import permissions, serializers, viewsets -from ...authentication import ApiKeyAuthentication from ...models import Parking @@ -90,8 +89,7 @@ def get_time_old_parkings_visible(default=datetime.timedelta(minutes=15)): class ValidParkingViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.IsAdminUser] queryset = Parking.objects.order_by('-time_end') serializer_class = ValidParkingSerializer filter_class = ValidParkingFilter - authentication_classes = [ApiKeyAuthentication] - permission_classes = [permissions.IsAdminUser] diff --git a/parkings/api/operator/parking.py b/parkings/api/operator/parking.py index 6d32c705..4ddae566 100644 --- a/parkings/api/operator/parking.py +++ b/parkings/api/operator/parking.py @@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import mixins, permissions, serializers, viewsets -from parkings.authentication import ApiKeyAuthentication from parkings.models import Operator, Parking from ..common import ParkingException @@ -87,10 +86,9 @@ def has_object_permission(self, request, view, obj): class OperatorAPIParkingViewSet(mixins.CreateModelMixin, mixins.UpdateModelMixin, mixins.DestroyModelMixin, viewsets.GenericViewSet): + permission_classes = [OperatorAPIParkingPermission] queryset = Parking.objects.order_by('time_start') serializer_class = OperatorAPIParkingSerializer - authentication_classes = (ApiKeyAuthentication,) - permission_classes = (OperatorAPIParkingPermission,) def perform_create(self, serializer): serializer.save(operator=self.request.user.operator) diff --git a/parkings/api/public/parking_area.py b/parkings/api/public/parking_area.py index fe9c5a8f..6ca43eab 100644 --- a/parkings/api/public/parking_area.py +++ b/parkings/api/public/parking_area.py @@ -1,4 +1,4 @@ -from rest_framework import viewsets +from rest_framework import permissions, viewsets from rest_framework_gis.pagination import GeoJsonPagination from rest_framework_gis.serializers import ( GeoFeatureModelSerializer, GeometrySerializerMethodField) @@ -24,6 +24,7 @@ class Meta: class PublicAPIParkingAreaViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.AllowAny] queryset = ParkingArea.objects.order_by('origin_id') serializer_class = ParkingAreaSerializer pagination_class = GeoJsonPagination diff --git a/parkings/api/public/parking_area_statistics.py b/parkings/api/public/parking_area_statistics.py index 889ed34f..9a6a9d36 100644 --- a/parkings/api/public/parking_area_statistics.py +++ b/parkings/api/public/parking_area_statistics.py @@ -1,6 +1,6 @@ from django.db.models import Case, Count, Q, When from django.utils import timezone -from rest_framework import serializers, viewsets +from rest_framework import permissions, serializers, viewsets from parkings.models import ParkingArea from parkings.pagination import PublicAPIPagination @@ -33,6 +33,7 @@ class Meta: class PublicAPIParkingAreaStatisticsViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.AllowAny] queryset = ParkingArea.objects.all() serializer_class = ParkingAreaStatisticsSerializer pagination_class = PublicAPIPagination diff --git a/parkings/api/public/urls.py b/parkings/api/public/urls.py index 2f5c8eb5..00e8ff9d 100644 --- a/parkings/api/public/urls.py +++ b/parkings/api/public/urls.py @@ -1,10 +1,20 @@ from django.conf.urls import include, url -from rest_framework.routers import DefaultRouter +from rest_framework import permissions +from rest_framework.routers import APIRootView, DefaultRouter from .parking_area import PublicAPIParkingAreaViewSet from .parking_area_statistics import PublicAPIParkingAreaStatisticsViewSet -router = DefaultRouter() + +class PublicApiRootView(APIRootView): + permission_classes = [permissions.AllowAny] + + +class Router(DefaultRouter): + APIRootView = PublicApiRootView + + +router = Router() router.register(r'parking_area', PublicAPIParkingAreaViewSet, base_name='parkingarea') router.register(r'parking_area_statistics', PublicAPIParkingAreaStatisticsViewSet, base_name='parkingareastatistics') diff --git a/parkkihubi/settings.py b/parkkihubi/settings.py index 49327707..3dddd2fe 100644 --- a/parkkihubi/settings.py +++ b/parkkihubi/settings.py @@ -18,6 +18,7 @@ # Django core settings # ######################## DEBUG = env.bool('DEBUG', default=False) +TIER = env.str('TIER', default='dev') SECRET_KEY = env.str('SECRET_KEY', default=('' if not DEBUG else 'xxx')) ALLOWED_HOSTS = ['*'] @@ -80,7 +81,7 @@ 'parkings', ] -if DEBUG: +if DEBUG and TIER == 'dev': # shell_plus and other goodies INSTALLED_APPS.append("django_extensions") @@ -157,6 +158,17 @@ # Django REST Framework # ######################### REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': [ + # Make nothing accessible to non-admins by default. Viewsets + # should specify permission_classes to override permissions. + 'rest_framework.permissions.IsAdminUser', + ], + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'parkings.authentication.ApiKeyAuthentication', + ] + ([ # Following two are only for DEBUG mode in dev environment: + 'rest_framework.authentication.SessionAuthentication', + 'rest_framework.authentication.BasicAuthentication', + ] if (DEBUG and TIER == 'dev') else []), 'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.NamespaceVersioning', 'ALLOWED_VERSIONS': ('v1',), 'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',), From 5839c11e7b86f121310613589d90dc5923447a3d Mon Sep 17 00:00:00 2001 From: Tuomas Suutari Date: Fri, 9 Feb 2018 11:34:12 +0200 Subject: [PATCH 7/9] Add API for regions and their parking statistics Add a new API namespace "monitoring" which will be used by the Dashboard. Then add an API endpoint to that namespace for getting basic geometry and metadata information about Regions and another endpoint for getting the parking counts for each region at specified time. --- README.md | 1 + parkings/api/monitoring/permissions.py | 15 ++++ parkings/api/monitoring/region.py | 54 ++++++++++++ parkings/api/monitoring/region_statistics.py | 37 ++++++++ parkings/api/monitoring/urls.py | 14 +++ .../api/public/parking_area_statistics.py | 4 +- parkings/api/utils.py | 26 ++++++ parkings/factories/__init__.py | 12 +++ parkings/factories/region.py | 15 ++++ parkings/pagination.py | 2 +- parkings/tests/api/conftest.py | 9 ++ parkings/tests/api/monitoring/__init__.py | 0 .../tests/api/monitoring/test_permissions.py | 38 ++++++++ parkings/tests/api/monitoring/test_region.py | 58 +++++++++++++ .../api/monitoring/test_region_statistics.py | 86 +++++++++++++++++++ parkings/tests/api/test_utils.py | 39 +++++++++ parkings/tests/conftest.py | 4 +- parkkihubi/settings.py | 3 + parkkihubi/urls.py | 6 ++ 19 files changed, 419 insertions(+), 4 deletions(-) create mode 100644 parkings/api/monitoring/permissions.py create mode 100644 parkings/api/monitoring/region.py create mode 100644 parkings/api/monitoring/region_statistics.py create mode 100644 parkings/api/monitoring/urls.py create mode 100644 parkings/api/utils.py create mode 100644 parkings/factories/region.py create mode 100644 parkings/tests/api/monitoring/__init__.py create mode 100644 parkings/tests/api/monitoring/test_permissions.py create mode 100644 parkings/tests/api/monitoring/test_region.py create mode 100644 parkings/tests/api/monitoring/test_region_statistics.py create mode 100644 parkings/tests/api/test_utils.py diff --git a/README.md b/README.md index bdbbe01e..80a666f0 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Create a basic file for development as follows #### Parkkihubi settings - `PARKKIHUBI_PUBLIC_API_ENABLED` default `True` +- `PARKKIHUBI_MONITORING_API_ENABLED` default `True` - `PARKKIHUBI_OPERATOR_API_ENABLED` default `True` - `PARKKIHUBI_ENFORCEMENT_API_ENABLED` default `True` diff --git a/parkings/api/monitoring/permissions.py b/parkings/api/monitoring/permissions.py new file mode 100644 index 00000000..6d823dea --- /dev/null +++ b/parkings/api/monitoring/permissions.py @@ -0,0 +1,15 @@ +from django.conf import settings +from rest_framework import permissions + + +class MonitoringApiPermission(permissions.IsAuthenticated): + def has_permission(self, request, view): + if not super().has_permission(request, view): + return False + + user_groups = request.user.groups + group_name = getattr(settings, 'MONITORING_GROUP_NAME', 'monitoring') + + is_in_monitoring_group = (user_groups.filter(name=group_name).exists()) + + return is_in_monitoring_group diff --git a/parkings/api/monitoring/region.py b/parkings/api/monitoring/region.py new file mode 100644 index 00000000..e7a74ec1 --- /dev/null +++ b/parkings/api/monitoring/region.py @@ -0,0 +1,54 @@ +import rest_framework_gis.pagination as gis_pagination +import rest_framework_gis.serializers as gis_serializers +from rest_framework import serializers, viewsets + +from ...models import ParkingArea, Region +from ..common import WGS84InBBoxFilter +from .permissions import MonitoringApiPermission + +WGS84_SRID = 4326 + +# Square meters in square kilometer +M2_PER_KM2 = 1000000.0 + + +class RegionSerializer(gis_serializers.GeoFeatureModelSerializer): + wgs84_geometry = gis_serializers.GeometrySerializerMethodField() + area_km2 = serializers.SerializerMethodField() + spots_per_km2 = serializers.SerializerMethodField() + parking_areas = serializers.SerializerMethodField() + + def get_wgs84_geometry(self, instance): + return instance.geom.transform(WGS84_SRID, clone=True) + + def get_area_km2(self, instance): + return instance.geom.area / M2_PER_KM2 + + def get_spots_per_km2(self, instance): + return M2_PER_KM2 * instance.capacity_estimate / instance.geom.area + + def get_parking_areas(self, instance): + parking_areas = ParkingArea.objects.intersecting_region(instance) + return [x.pk for x in parking_areas] + + class Meta: + model = Region + geo_field = 'wgs84_geometry' + fields = [ + 'id', + 'name', + 'capacity_estimate', + 'area_km2', + 'spots_per_km2', + 'parking_areas', + ] + + +class RegionViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [MonitoringApiPermission] + queryset = Region.objects.all().order_by('id') + serializer_class = RegionSerializer + pagination_class = gis_pagination.GeoJsonPagination + bbox_filter_field = 'geom' + filter_backends = [WGS84InBBoxFilter] + bbox_filter_include_overlapping = True diff --git a/parkings/api/monitoring/region_statistics.py b/parkings/api/monitoring/region_statistics.py new file mode 100644 index 00000000..e45022cb --- /dev/null +++ b/parkings/api/monitoring/region_statistics.py @@ -0,0 +1,37 @@ +from rest_framework import serializers, viewsets + +from ...models import Region +from ...pagination import Pagination +from ..common import WGS84InBBoxFilter +from ..utils import parse_timestamp_or_now +from .permissions import MonitoringApiPermission + + +class RegionStatisticsSerializer(serializers.ModelSerializer): + parking_count = serializers.IntegerField(read_only=True) + + class Meta: + model = Region + fields = ( + 'id', + 'parking_count', + ) + + +class RegionStatisticsViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [MonitoringApiPermission] + queryset = Region.objects.all() + serializer_class = RegionStatisticsSerializer + pagination_class = Pagination + bbox_filter_field = 'geom' + filter_backends = [WGS84InBBoxFilter] + bbox_filter_include_overlapping = True + + def get_queryset(self): + time = parse_timestamp_or_now(self.request.query_params.get('time')) + return ( + super().get_queryset() + .with_parking_count(time) + .values('id', 'parking_count') + .order_by('id') + .filter(parking_count__gt=0)) diff --git a/parkings/api/monitoring/urls.py b/parkings/api/monitoring/urls.py new file mode 100644 index 00000000..2fd4c849 --- /dev/null +++ b/parkings/api/monitoring/urls.py @@ -0,0 +1,14 @@ +from django.conf.urls import include, url +from rest_framework.routers import DefaultRouter + +from .region import RegionViewSet +from .region_statistics import RegionStatisticsViewSet + +router = DefaultRouter() +router.register(r'region', RegionViewSet, base_name='region') +router.register(r'region_statistics', RegionStatisticsViewSet, + base_name='regionstatistics') + +urlpatterns = [ + url(r'^', include(router.urls, namespace='v1')), +] diff --git a/parkings/api/public/parking_area_statistics.py b/parkings/api/public/parking_area_statistics.py index 9a6a9d36..c9fe2bb0 100644 --- a/parkings/api/public/parking_area_statistics.py +++ b/parkings/api/public/parking_area_statistics.py @@ -3,7 +3,7 @@ from rest_framework import permissions, serializers, viewsets from parkings.models import ParkingArea -from parkings.pagination import PublicAPIPagination +from parkings.pagination import Pagination from ..common import WGS84InBBoxFilter @@ -36,7 +36,7 @@ class PublicAPIParkingAreaStatisticsViewSet(viewsets.ReadOnlyModelViewSet): permission_classes = [permissions.AllowAny] queryset = ParkingArea.objects.all() serializer_class = ParkingAreaStatisticsSerializer - pagination_class = PublicAPIPagination + pagination_class = Pagination bbox_filter_field = 'geom' filter_backends = (WGS84InBBoxFilter,) bbox_filter_include_overlapping = True diff --git a/parkings/api/utils.py b/parkings/api/utils.py new file mode 100644 index 00000000..24f2cc49 --- /dev/null +++ b/parkings/api/utils.py @@ -0,0 +1,26 @@ +import dateutil.parser +from django.utils import timezone +from rest_framework.exceptions import ValidationError + + +def parse_timestamp_or_now(timestamp_string): + """ + Parse given timestamp string or return current time. + + If the timestamp string is falsy, return current time, otherwise try + to parse the string and return the parsed value. + + :type timestamp_string: str + :rtype: datetime.datetime + :raises rest_framework.exceptions.ValidationError: on parse error + """ + if not timestamp_string: + return timezone.now() + return parse_timestamp(timestamp_string) + + +def parse_timestamp(datetime_string): + try: + return dateutil.parser.parse(datetime_string) + except (ValueError, OverflowError): + raise ValidationError('Invalid timestamp: {}'.format(datetime_string)) diff --git a/parkings/factories/__init__.py b/parkings/factories/__init__.py index 5521e8ec..d4b3891e 100644 --- a/parkings/factories/__init__.py +++ b/parkings/factories/__init__.py @@ -1,4 +1,16 @@ from .operator import OperatorFactory # noqa from .parking import HistoryParkingFactory, ParkingFactory # noqa from .parking_area import ParkingAreaFactory # noqa +from .region import RegionFactory from .user import AdminUserFactory, StaffUserFactory, UserFactory # noqa + +__all__ = [ + 'AdminUserFactory', + 'HistoryParkingFactory', + 'OperatorFactory', + 'ParkingAreaFactory', + 'ParkingFactory', + 'RegionFactory', + 'StaffUserFactory', + 'UserFactory', +] diff --git a/parkings/factories/region.py b/parkings/factories/region.py new file mode 100644 index 00000000..85c293c5 --- /dev/null +++ b/parkings/factories/region.py @@ -0,0 +1,15 @@ +import factory + +from parkings.models import Region + +from .faker import fake +from .parking_area import generate_multi_polygon + + +class RegionFactory(factory.django.DjangoModelFactory): + class Meta: + model = Region + + geom = factory.LazyFunction(generate_multi_polygon) + capacity_estimate = fake.random.randint(0, 500) + name = factory.LazyFunction(fake.city) diff --git a/parkings/pagination.py b/parkings/pagination.py index 26aefbd2..67a60745 100644 --- a/parkings/pagination.py +++ b/parkings/pagination.py @@ -1,5 +1,5 @@ from rest_framework.pagination import PageNumberPagination -class PublicAPIPagination(PageNumberPagination): +class Pagination(PageNumberPagination): page_size_query_param = 'page_size' diff --git a/parkings/tests/api/conftest.py b/parkings/tests/api/conftest.py index 63b709c8..fe32cfdd 100644 --- a/parkings/tests/api/conftest.py +++ b/parkings/tests/api/conftest.py @@ -14,6 +14,15 @@ def api_client(): return APIClient() +@pytest.fixture +def monitoring_api_client(user_factory): + api_client = APIClient() + user = user_factory() + user.groups.get_or_create(name='monitoring') + api_client.force_authenticate(user) + return api_client + + @pytest.fixture def user_api_client(user_factory): api_client = APIClient() diff --git a/parkings/tests/api/monitoring/__init__.py b/parkings/tests/api/monitoring/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/parkings/tests/api/monitoring/test_permissions.py b/parkings/tests/api/monitoring/test_permissions.py new file mode 100644 index 00000000..360a34fe --- /dev/null +++ b/parkings/tests/api/monitoring/test_permissions.py @@ -0,0 +1,38 @@ +import pytest +from django.contrib.auth.models import AnonymousUser, User + +from parkings.api.monitoring.permissions import MonitoringApiPermission + + +@pytest.mark.django_db +def test_monitoring_api_permission_anonymous(rf): + request = rf.get('/') + request.user = None + perm = MonitoringApiPermission() + assert perm.has_permission(request, None) is False + request.user = AnonymousUser() + assert perm.has_permission(request, None) is False + + +@pytest.mark.django_db +def test_monitoring_api_permission_not_in_group(rf, user): + request = rf.get('/') + request.user = user + perm = MonitoringApiPermission() + assert perm.has_permission(request, None) is False + + +@pytest.mark.django_db +@pytest.mark.parametrize('group,allowed', [ + ('monitoring', True), ('othergrp', False)]) +def test_monitoring_api_permission_by_group(group, allowed, rf): + user = User.objects.create_user('dummy-user') + user.groups.create(name=group) + try: + request = rf.get('/') + request.user = user + perm = MonitoringApiPermission() + assert perm.has_permission(request, None) is allowed + finally: + user.groups.all().delete() + user.delete() diff --git a/parkings/tests/api/monitoring/test_region.py b/parkings/tests/api/monitoring/test_region.py new file mode 100644 index 00000000..7de288f8 --- /dev/null +++ b/parkings/tests/api/monitoring/test_region.py @@ -0,0 +1,58 @@ +import json + +from django.urls import reverse +from rest_framework import status + +WGS84_SRID = 4326 + + +list_url = reverse('monitoring:v1:region-list') + + +def test_get_regions_empty(monitoring_api_client): + result = monitoring_api_client.get(list_url) + assert result.data == { + 'type': 'FeatureCollection', + 'features': [], + 'count': 0, + 'next': None, + 'previous': None, + } + assert result.status_code == status.HTTP_200_OK + + +def test_get_regions_with_data(monitoring_api_client, region, parking_area): + parking_area.geom = region.geom + parking_area.save() + region.save() # Update capacity_estimate + + result = monitoring_api_client.get(list_url) + features = result.data.pop('features', None) + assert result.data == { + 'type': 'FeatureCollection', + 'count': 1, + 'next': None, + 'previous': None, + } + assert result.status_code == status.HTTP_200_OK + assert isinstance(features, list) + assert len(features) == 1 + geometry = features[0].pop('geometry', None) + properties = features[0].pop('properties', None) + assert features[0] == {'id': str(region.id), 'type': 'Feature'} + km2 = region.geom.area / 1000000.0 + assert properties == { + 'name': region.name, + 'capacity_estimate': region.capacity_estimate, + 'area_km2': km2, + 'spots_per_km2': region.capacity_estimate / km2, + 'parking_areas': [parking_area.id], + } + coordinates = geometry.pop('coordinates', None) + assert geometry == {'type': 'MultiPolygon'} + wgs84_geom = region.geom.transform(WGS84_SRID, clone=True) + assert coordinates == tuples_to_lists(wgs84_geom.coords) + + +def tuples_to_lists(tuples_of_tuples): + return json.loads(json.dumps(tuples_of_tuples)) diff --git a/parkings/tests/api/monitoring/test_region_statistics.py b/parkings/tests/api/monitoring/test_region_statistics.py new file mode 100644 index 00000000..fd08e242 --- /dev/null +++ b/parkings/tests/api/monitoring/test_region_statistics.py @@ -0,0 +1,86 @@ +from collections import OrderedDict + +import pytz +from django.urls import reverse +from rest_framework import status + +from parkings.factories.faker import fake + +from ...utils import create_parkings_and_regions, intersects + +list_url = reverse('monitoring:v1:regionstatistics-list') + + +def test_empty(monitoring_api_client): + result = monitoring_api_client.get(list_url) + assert result.data == { + 'count': 0, + 'next': None, + 'previous': None, + 'results': [], + } + assert result.status_code == status.HTTP_200_OK + + +def test_with_single_parking(monitoring_api_client, region, parking): + point_in_region = region.geom.centroid + parking.location = point_in_region + parking.save() + assert intersects(point_in_region, region) + + result = monitoring_api_client.get(list_url) + assert result.data == { + 'count': 1, + 'next': None, + 'previous': None, + 'results': [ + OrderedDict([ + ('id', str(region.id)), + ('parking_count', 1)]), + ] + } + assert result.status_code == status.HTTP_200_OK + + +def test_with_many_parkings_and_specified_time(monitoring_api_client): + (parkings, regions) = create_parkings_and_regions( + parking_count=20, region_count=5) + + # Pick a point in time that is in between the parking times + earliest_end_time = min(x.time_end for x in parkings) + latest_end_time = max(x.time_end for x in parkings) + time = fake.date_time_between(earliest_end_time, latest_end_time, + tzinfo=pytz.utc) + + # Calculate some lookup containers + valid_parkings_at_time = [ + parking for parking in parkings + if parking.time_start <= time and (parking.time_end is None + or parking.time_end >= time)] + regions_with_valid_parkings = { + parking.region for parking in valid_parkings_at_time + if parking.region + } + regions_by_id = {str(region.id): region for region in regions} + + # Do the API call + api_result = monitoring_api_client.get(list_url, {'time': str(time)}) + + # Check the results + results = api_result.data.pop('results', None) + assert api_result.data == { + 'count': len(regions_with_valid_parkings), + 'next': None, # No paging, since parking count < page size + 'previous': None, + } + assert len(results) == len(regions_with_valid_parkings) + for result in results: + assert isinstance(result, OrderedDict) + assert set(result.keys()) == {'id', 'parking_count'} + assert result['id'] in regions_by_id + region = regions_by_id[result['id']] + expected_count = sum( + 1 for parking in valid_parkings_at_time + if parking.region == region) + assert result['parking_count'] == expected_count + assert api_result.status_code == status.HTTP_200_OK diff --git a/parkings/tests/api/test_utils.py b/parkings/tests/api/test_utils.py new file mode 100644 index 00000000..aa7829b2 --- /dev/null +++ b/parkings/tests/api/test_utils.py @@ -0,0 +1,39 @@ +import datetime + +import pytest +from freezegun import freeze_time +from rest_framework.exceptions import ValidationError as DrfValidationError + +from parkings.api.utils import parse_timestamp, parse_timestamp_or_now + + +def test_parse_timestamp_or_now_with_val(): + parsed = parse_timestamp_or_now('2000-01-01T20:01:05+0200') + assert isinstance(parsed, datetime.datetime) + assert str(parsed) == '2000-01-01 20:01:05+02:00' + + +@freeze_time('2000-02-29T10:15:22Z') +def test_parse_timestamp_or_now_with_empty(): + parsed = parse_timestamp_or_now('') + assert isinstance(parsed, datetime.datetime) + assert str(parsed) == '2000-02-29 10:15:22+00:00' + + +@freeze_time('2000-02-29T10:15:22Z') +def test_parse_timestamp_or_now_with_none(): + parsed = parse_timestamp_or_now(None) + assert isinstance(parsed, datetime.datetime) + assert str(parsed) == '2000-02-29 10:15:22+00:00' + + +def test_parse_timestamp_with_valid_data(): + parsed = parse_timestamp('2000-02-29T10:15:22-12:00') + assert isinstance(parsed, datetime.datetime) + assert str(parsed) == '2000-02-29 10:15:22-12:00' + + +def test_parse_timestamp_with_invalid_data(): + with pytest.raises(DrfValidationError) as excinfo: + parse_timestamp('foobar') + assert str(excinfo.value) == "['Invalid timestamp: foobar']" diff --git a/parkings/tests/conftest.py b/parkings/tests/conftest.py index c676bc46..4eefb9cc 100644 --- a/parkings/tests/conftest.py +++ b/parkings/tests/conftest.py @@ -3,7 +3,8 @@ from parkings.factories import ( AdminUserFactory, HistoryParkingFactory, OperatorFactory, - ParkingAreaFactory, ParkingFactory, StaffUserFactory, UserFactory) + ParkingAreaFactory, ParkingFactory, RegionFactory, StaffUserFactory, + UserFactory) register(OperatorFactory) register(ParkingFactory, 'parking') @@ -12,6 +13,7 @@ register(StaffUserFactory, 'staff_user') register(UserFactory) register(ParkingAreaFactory) +register(RegionFactory) @pytest.fixture(autouse=True) diff --git a/parkkihubi/settings.py b/parkkihubi/settings.py index 3dddd2fe..47242753 100644 --- a/parkkihubi/settings.py +++ b/parkkihubi/settings.py @@ -183,9 +183,12 @@ ############## # Parkkihubi # ############## +MONITORING_GROUP_NAME = 'monitoring' PARKKIHUBI_TIME_PARKINGS_EDITABLE = timedelta(minutes=2) PARKKIHUBI_TIME_OLD_PARKINGS_VISIBLE = timedelta(minutes=15) PARKKIHUBI_PUBLIC_API_ENABLED = env.bool('PARKKIHUBI_PUBLIC_API_ENABLED', True) +PARKKIHUBI_MONITORING_API_ENABLED = env.bool( + 'PARKKIHUBI_MONITORING_API_ENABLED', True) PARKKIHUBI_OPERATOR_API_ENABLED = env.bool('PARKKIHUBI_OPERATOR_API_ENABLED', True) PARKKIHUBI_ENFORCEMENT_API_ENABLED = ( env.bool('PARKKIHUBI_ENFORCEMENT_API_ENABLED', True)) diff --git a/parkkihubi/urls.py b/parkkihubi/urls.py index 45fd7d7a..a62a91eb 100644 --- a/parkkihubi/urls.py +++ b/parkkihubi/urls.py @@ -3,6 +3,7 @@ from django.contrib import admin from parkings.api.enforcement import urls as enforcement_urls +from parkings.api.monitoring import urls as monitoring_urls from parkings.api.operator import urls as operator_urls from parkings.api.public import urls as public_urls @@ -11,6 +12,11 @@ if getattr(settings, 'PARKKIHUBI_PUBLIC_API_ENABLED', False): urlpatterns.append(url(r'^public/v1/', include(public_urls, namespace='public'))) +if getattr(settings, 'PARKKIHUBI_MONITORING_API_ENABLED', False): + urlpatterns.append( + url(r'^monitoring/v1/', + include(monitoring_urls, namespace='monitoring'))) + if getattr(settings, 'PARKKIHUBI_OPERATOR_API_ENABLED', False): urlpatterns.append(url(r'^operator/v1/', include(operator_urls, namespace='operator'))) From a0431f1bfd7065b0b2cd5a5554bf45c0a992925d Mon Sep 17 00:00:00 2001 From: Tuomas Suutari Date: Fri, 9 Feb 2018 11:35:12 +0200 Subject: [PATCH 8/9] API endpoints for JWT authentication --- parkings/api/auth/urls.py | 10 ++++++++++ parkkihubi/settings.py | 20 ++++++++++++++++++++ parkkihubi/urls.py | 5 ++++- requirements.in | 1 + requirements.txt | 3 +++ 5 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 parkings/api/auth/urls.py diff --git a/parkings/api/auth/urls.py b/parkings/api/auth/urls.py new file mode 100644 index 00000000..f1126559 --- /dev/null +++ b/parkings/api/auth/urls.py @@ -0,0 +1,10 @@ +import drf_jwt_2fa.urls +from django.conf.urls import include, url + +v1_urlpatterns = [ + url(r'^', include(drf_jwt_2fa.urls, namespace='auth')), +] + +urlpatterns = [ + url(r'^', include(v1_urlpatterns, namespace='v1')), +] diff --git a/parkkihubi/settings.py b/parkkihubi/settings.py index 47242753..914892e1 100644 --- a/parkkihubi/settings.py +++ b/parkkihubi/settings.py @@ -153,6 +153,7 @@ vars().update(env.email_url( default=('consolemail://' if DEBUG else 'smtp://localhost:25') )) +DEFAULT_FROM_EMAIL = 'no-reply.parkkihubi@fiupparkp01.anders.fi' ######################### # Django REST Framework # @@ -165,6 +166,7 @@ ], 'DEFAULT_AUTHENTICATION_CLASSES': [ 'parkings.authentication.ApiKeyAuthentication', + 'drf_jwt_2fa.authentication.Jwt2faAuthentication', ] + ([ # Following two are only for DEBUG mode in dev environment: 'rest_framework.authentication.SessionAuthentication', 'rest_framework.authentication.BasicAuthentication', @@ -178,6 +180,24 @@ 'TEST_REQUEST_DEFAULT_FORMAT': 'json', } +JWT_AUTH = { + 'JWT_EXPIRATION_DELTA': timedelta(minutes=30), + 'JWT_ALLOW_REFRESH': True, + 'JWT_REFRESH_EXPIRATION_DELTA': timedelta(days=7), +} + +JWT2FA_AUTH = { + 'CODE_TOKEN_THROTTLE_RATE': '5/15m', + 'AUTH_TOKEN_RETRY_WAIT_TIME': timedelta(seconds=10), + 'EMAIL_SENDER_SUBJECT_OVERRIDE': '{code} - Varmennuskoodisi', + 'EMAIL_SENDER_BODY_OVERRIDE': ( + 'Hei!\n' + '\n' + 'Varmennuskoodisi kirjautumista varten on: {code}\n' + '\n' + 't. Parkkihubi'), +} + CORS_ORIGIN_ALLOW_ALL = True ############## diff --git a/parkkihubi/urls.py b/parkkihubi/urls.py index a62a91eb..bb6c4f44 100644 --- a/parkkihubi/urls.py +++ b/parkkihubi/urls.py @@ -2,12 +2,15 @@ from django.conf.urls import include, url from django.contrib import admin +from parkings.api.auth import urls as auth_urls from parkings.api.enforcement import urls as enforcement_urls from parkings.api.monitoring import urls as monitoring_urls from parkings.api.operator import urls as operator_urls from parkings.api.public import urls as public_urls -urlpatterns = [] +urlpatterns = [ + url(r'^auth/v1/', include(auth_urls, namespace='auth')), +] if getattr(settings, 'PARKKIHUBI_PUBLIC_API_ENABLED', False): urlpatterns.append(url(r'^public/v1/', include(public_urls, namespace='public'))) diff --git a/requirements.in b/requirements.in index 19bbef0c..12534c41 100644 --- a/requirements.in +++ b/requirements.in @@ -14,6 +14,7 @@ psycopg2 djangorestframework djangorestframework-gis django-filter +drf-jwt-2fa # XML parsing lxml diff --git a/requirements.txt b/requirements.txt index 89f0c18c..2c4e0b4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,10 +10,13 @@ django-environ==0.4.4 django-filter==1.1.0 djangorestframework==3.7.7 djangorestframework-gis==0.12 +djangorestframework-jwt==1.11.0 # via drf-jwt-2fa +drf-jwt-2fa==0.3.0 idna==2.6 # via requests lxml==4.1.1 owslib==0.16.0 psycopg2==2.7.3.2 +pyjwt==1.5.3 # via djangorestframework-jwt pyproj==1.9.5.1 # via owslib python-dateutil==2.6.1 # via owslib pytz==2017.3 From 771ef01064bc67f6b5906c0681d9fc3b52debd9e Mon Sep 17 00:00:00 2001 From: Tuomas Suutari Date: Fri, 9 Feb 2018 11:35:34 +0200 Subject: [PATCH 9/9] Improve coverage in tests * Remove unused checking function. * Add a "pragma: no cover" comment to a testing helper * Reorganize id finding constuction that created some unnecessary uncovered code branches * Convert couple `elif`s to `else`s with an assert --- parkings/tests/api/enforcement/test_operator.py | 3 ++- parkings/tests/api/enforcement/test_valid_parking.py | 3 ++- parkings/tests/api/operator/test_parking.py | 7 ------- .../tests/api/public/test_parking_area_statistics.py | 11 +++++++---- parkings/tests/api/utils.py | 2 +- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/parkings/tests/api/enforcement/test_operator.py b/parkings/tests/api/enforcement/test_operator.py index 7fa11bf9..e21a1ba3 100644 --- a/parkings/tests/api/enforcement/test_operator.py +++ b/parkings/tests/api/enforcement/test_operator.py @@ -14,7 +14,8 @@ def get_url(kind, operator): if kind == 'list': return list_url - elif kind == 'detail': + else: + assert kind == 'detail' return reverse('enforcement:v1:operator-detail', kwargs={'pk': operator.pk}) diff --git a/parkings/tests/api/enforcement/test_valid_parking.py b/parkings/tests/api/enforcement/test_valid_parking.py index 28658792..8164c36a 100644 --- a/parkings/tests/api/enforcement/test_valid_parking.py +++ b/parkings/tests/api/enforcement/test_valid_parking.py @@ -25,7 +25,8 @@ def get_url(kind, parking): return list_url elif kind == 'list_by_reg_num': return list_url_for(parking.registration_number) - elif kind == 'detail': + else: + assert kind == 'detail' return reverse('enforcement:v1:valid_parking-detail', kwargs={'pk': parking.pk}) diff --git a/parkings/tests/api/operator/test_parking.py b/parkings/tests/api/operator/test_parking.py index 5e0fbb67..8f7505ac 100644 --- a/parkings/tests/api/operator/test_parking.py +++ b/parkings/tests/api/operator/test_parking.py @@ -41,13 +41,6 @@ def updated_parking_data(): } -def check_parking_data_keys(parking_data): - assert set(parking_data.keys()) == { - 'id', 'created_at', 'modified_at', 'location', 'registration_number', 'time_start', 'time_end', 'zone', - 'status', - } - - def check_parking_data_matches_parking_object(parking_data, parking_obj): """ Check that a parking data dict and an actual Parking object match. diff --git a/parkings/tests/api/public/test_parking_area_statistics.py b/parkings/tests/api/public/test_parking_area_statistics.py index ecc19c78..99f35978 100644 --- a/parkings/tests/api/public/test_parking_area_statistics.py +++ b/parkings/tests/api/public/test_parking_area_statistics.py @@ -45,10 +45,13 @@ def test_get_list_check_data(api_client, parking_factory, parking_area_factory, results = get(api_client, list_url)['results'] assert len(results) == 4 - stats_data_1 = next(result for result in results if result['id'] == str(parking_area_1.id)) - stats_data_2 = next(result for result in results if result['id'] == str(parking_area_2.id)) - stats_data_3 = next(result for result in results if result['id'] == str(parking_area_3.id)) - stats_data_4 = next(result for result in results if result['id'] == str(parking_area_4.id)) + def find_by_obj_id(obj, iterable): + return [x for x in iterable if x['id'] == str(obj.id)][0] + + stats_data_1 = find_by_obj_id(parking_area_1, results) + stats_data_2 = find_by_obj_id(parking_area_2, results) + stats_data_3 = find_by_obj_id(parking_area_3, results) + stats_data_4 = find_by_obj_id(parking_area_4, results) assert stats_data_1.keys() == {'id', 'current_parking_count'} assert stats_data_1['current_parking_count'] == 4 diff --git a/parkings/tests/api/utils.py b/parkings/tests/api/utils.py index 01f7dcb4..23c2c63b 100644 --- a/parkings/tests/api/utils.py +++ b/parkings/tests/api/utils.py @@ -72,7 +72,7 @@ def check_required_fields(api_client, url, expected_required_fields, detail_endp required_fields = set() for (field, errors) in response_data.items(): - if 'This field is required.' in repr(errors): + if 'This field is required.' in repr(errors): # pragma: no cover required_fields.add(field) assert required_fields == expected_required_fields, '%s != %s' % (required_fields, expected_required_fields)