diff --git a/README.md b/README.md index bdbbe01e..80a666f0 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Create a basic file for development as follows #### Parkkihubi settings - `PARKKIHUBI_PUBLIC_API_ENABLED` default `True` +- `PARKKIHUBI_MONITORING_API_ENABLED` default `True` - `PARKKIHUBI_OPERATOR_API_ENABLED` default `True` - `PARKKIHUBI_ENFORCEMENT_API_ENABLED` default `True` diff --git a/parkings/admin.py b/parkings/admin.py index 4810466d..0cb8ae01 100644 --- a/parkings/admin.py +++ b/parkings/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin from django.contrib.gis.admin import OSMGeoAdmin -from .models import Operator, Parking, ParkingArea, ParkingTerminal +from .models import Operator, Parking, ParkingArea, ParkingTerminal, Region class OperatorAdmin(admin.ModelAdmin): @@ -18,6 +18,11 @@ class ParkingAdmin(OSMGeoAdmin): ordering = ('-created_at',) +@admin.register(Region) +class RegionAdmin(OSMGeoAdmin): + ordering = ('name',) + + class ParkingAreaAdmin(OSMGeoAdmin): ordering = ('origin_id',) diff --git a/parkings/api/auth/urls.py b/parkings/api/auth/urls.py new file mode 100644 index 00000000..f1126559 --- /dev/null +++ b/parkings/api/auth/urls.py @@ -0,0 +1,10 @@ +import drf_jwt_2fa.urls +from django.conf.urls import include, url + +v1_urlpatterns = [ + url(r'^', include(drf_jwt_2fa.urls, namespace='auth')), +] + +urlpatterns = [ + url(r'^', include(v1_urlpatterns, namespace='v1')), +] diff --git a/parkings/api/enforcement/operator.py b/parkings/api/enforcement/operator.py index b495e37c..f83c36f1 100644 --- a/parkings/api/enforcement/operator.py +++ b/parkings/api/enforcement/operator.py @@ -1,6 +1,5 @@ from rest_framework import permissions, serializers, viewsets -from ...authentication import ApiKeyAuthentication from ...models import Operator @@ -16,7 +15,6 @@ class Meta: class OperatorViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.IsAdminUser] queryset = Operator.objects.order_by('name') serializer_class = OperatorSerializer - authentication_classes = [ApiKeyAuthentication] - permission_classes = [permissions.IsAdminUser] diff --git a/parkings/api/enforcement/valid_parking.py b/parkings/api/enforcement/valid_parking.py index b2179360..1042bf6e 100644 --- a/parkings/api/enforcement/valid_parking.py +++ b/parkings/api/enforcement/valid_parking.py @@ -6,7 +6,6 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import permissions, serializers, viewsets -from ...authentication import ApiKeyAuthentication from ...models import Parking @@ -90,8 +89,7 @@ def get_time_old_parkings_visible(default=datetime.timedelta(minutes=15)): class ValidParkingViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.IsAdminUser] queryset = Parking.objects.order_by('-time_end') serializer_class = ValidParkingSerializer filter_class = ValidParkingFilter - authentication_classes = [ApiKeyAuthentication] - permission_classes = [permissions.IsAdminUser] diff --git a/parkings/api/monitoring/permissions.py b/parkings/api/monitoring/permissions.py new file mode 100644 index 00000000..6d823dea --- /dev/null +++ b/parkings/api/monitoring/permissions.py @@ -0,0 +1,15 @@ +from django.conf import settings +from rest_framework import permissions + + +class MonitoringApiPermission(permissions.IsAuthenticated): + def has_permission(self, request, view): + if not super().has_permission(request, view): + return False + + user_groups = request.user.groups + group_name = getattr(settings, 'MONITORING_GROUP_NAME', 'monitoring') + + is_in_monitoring_group = (user_groups.filter(name=group_name).exists()) + + return is_in_monitoring_group diff --git a/parkings/api/monitoring/region.py b/parkings/api/monitoring/region.py new file mode 100644 index 00000000..e7a74ec1 --- /dev/null +++ b/parkings/api/monitoring/region.py @@ -0,0 +1,54 @@ +import rest_framework_gis.pagination as gis_pagination +import rest_framework_gis.serializers as gis_serializers +from rest_framework import serializers, viewsets + +from ...models import ParkingArea, Region +from ..common import WGS84InBBoxFilter +from .permissions import MonitoringApiPermission + +WGS84_SRID = 4326 + +# Square meters in square kilometer +M2_PER_KM2 = 1000000.0 + + +class RegionSerializer(gis_serializers.GeoFeatureModelSerializer): + wgs84_geometry = gis_serializers.GeometrySerializerMethodField() + area_km2 = serializers.SerializerMethodField() + spots_per_km2 = serializers.SerializerMethodField() + parking_areas = serializers.SerializerMethodField() + + def get_wgs84_geometry(self, instance): + return instance.geom.transform(WGS84_SRID, clone=True) + + def get_area_km2(self, instance): + return instance.geom.area / M2_PER_KM2 + + def get_spots_per_km2(self, instance): + return M2_PER_KM2 * instance.capacity_estimate / instance.geom.area + + def get_parking_areas(self, instance): + parking_areas = ParkingArea.objects.intersecting_region(instance) + return [x.pk for x in parking_areas] + + class Meta: + model = Region + geo_field = 'wgs84_geometry' + fields = [ + 'id', + 'name', + 'capacity_estimate', + 'area_km2', + 'spots_per_km2', + 'parking_areas', + ] + + +class RegionViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [MonitoringApiPermission] + queryset = Region.objects.all().order_by('id') + serializer_class = RegionSerializer + pagination_class = gis_pagination.GeoJsonPagination + bbox_filter_field = 'geom' + filter_backends = [WGS84InBBoxFilter] + bbox_filter_include_overlapping = True diff --git a/parkings/api/monitoring/region_statistics.py b/parkings/api/monitoring/region_statistics.py new file mode 100644 index 00000000..e45022cb --- /dev/null +++ b/parkings/api/monitoring/region_statistics.py @@ -0,0 +1,37 @@ +from rest_framework import serializers, viewsets + +from ...models import Region +from ...pagination import Pagination +from ..common import WGS84InBBoxFilter +from ..utils import parse_timestamp_or_now +from .permissions import MonitoringApiPermission + + +class RegionStatisticsSerializer(serializers.ModelSerializer): + parking_count = serializers.IntegerField(read_only=True) + + class Meta: + model = Region + fields = ( + 'id', + 'parking_count', + ) + + +class RegionStatisticsViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [MonitoringApiPermission] + queryset = Region.objects.all() + serializer_class = RegionStatisticsSerializer + pagination_class = Pagination + bbox_filter_field = 'geom' + filter_backends = [WGS84InBBoxFilter] + bbox_filter_include_overlapping = True + + def get_queryset(self): + time = parse_timestamp_or_now(self.request.query_params.get('time')) + return ( + super().get_queryset() + .with_parking_count(time) + .values('id', 'parking_count') + .order_by('id') + .filter(parking_count__gt=0)) diff --git a/parkings/api/monitoring/urls.py b/parkings/api/monitoring/urls.py new file mode 100644 index 00000000..2fd4c849 --- /dev/null +++ b/parkings/api/monitoring/urls.py @@ -0,0 +1,14 @@ +from django.conf.urls import include, url +from rest_framework.routers import DefaultRouter + +from .region import RegionViewSet +from .region_statistics import RegionStatisticsViewSet + +router = DefaultRouter() +router.register(r'region', RegionViewSet, base_name='region') +router.register(r'region_statistics', RegionStatisticsViewSet, + base_name='regionstatistics') + +urlpatterns = [ + url(r'^', include(router.urls, namespace='v1')), +] diff --git a/parkings/api/operator/parking.py b/parkings/api/operator/parking.py index 6d32c705..4ddae566 100644 --- a/parkings/api/operator/parking.py +++ b/parkings/api/operator/parking.py @@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import mixins, permissions, serializers, viewsets -from parkings.authentication import ApiKeyAuthentication from parkings.models import Operator, Parking from ..common import ParkingException @@ -87,10 +86,9 @@ def has_object_permission(self, request, view, obj): class OperatorAPIParkingViewSet(mixins.CreateModelMixin, mixins.UpdateModelMixin, mixins.DestroyModelMixin, viewsets.GenericViewSet): + permission_classes = [OperatorAPIParkingPermission] queryset = Parking.objects.order_by('time_start') serializer_class = OperatorAPIParkingSerializer - authentication_classes = (ApiKeyAuthentication,) - permission_classes = (OperatorAPIParkingPermission,) def perform_create(self, serializer): serializer.save(operator=self.request.user.operator) diff --git a/parkings/api/public/parking_area.py b/parkings/api/public/parking_area.py index fe9c5a8f..6ca43eab 100644 --- a/parkings/api/public/parking_area.py +++ b/parkings/api/public/parking_area.py @@ -1,4 +1,4 @@ -from rest_framework import viewsets +from rest_framework import permissions, viewsets from rest_framework_gis.pagination import GeoJsonPagination from rest_framework_gis.serializers import ( GeoFeatureModelSerializer, GeometrySerializerMethodField) @@ -24,6 +24,7 @@ class Meta: class PublicAPIParkingAreaViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.AllowAny] queryset = ParkingArea.objects.order_by('origin_id') serializer_class = ParkingAreaSerializer pagination_class = GeoJsonPagination diff --git a/parkings/api/public/parking_area_statistics.py b/parkings/api/public/parking_area_statistics.py index 889ed34f..c9fe2bb0 100644 --- a/parkings/api/public/parking_area_statistics.py +++ b/parkings/api/public/parking_area_statistics.py @@ -1,9 +1,9 @@ from django.db.models import Case, Count, Q, When from django.utils import timezone -from rest_framework import serializers, viewsets +from rest_framework import permissions, serializers, viewsets from parkings.models import ParkingArea -from parkings.pagination import PublicAPIPagination +from parkings.pagination import Pagination from ..common import WGS84InBBoxFilter @@ -33,9 +33,10 @@ class Meta: class PublicAPIParkingAreaStatisticsViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.AllowAny] queryset = ParkingArea.objects.all() serializer_class = ParkingAreaStatisticsSerializer - pagination_class = PublicAPIPagination + pagination_class = Pagination bbox_filter_field = 'geom' filter_backends = (WGS84InBBoxFilter,) bbox_filter_include_overlapping = True diff --git a/parkings/api/public/urls.py b/parkings/api/public/urls.py index 2f5c8eb5..00e8ff9d 100644 --- a/parkings/api/public/urls.py +++ b/parkings/api/public/urls.py @@ -1,10 +1,20 @@ from django.conf.urls import include, url -from rest_framework.routers import DefaultRouter +from rest_framework import permissions +from rest_framework.routers import APIRootView, DefaultRouter from .parking_area import PublicAPIParkingAreaViewSet from .parking_area_statistics import PublicAPIParkingAreaStatisticsViewSet -router = DefaultRouter() + +class PublicApiRootView(APIRootView): + permission_classes = [permissions.AllowAny] + + +class Router(DefaultRouter): + APIRootView = PublicApiRootView + + +router = Router() router.register(r'parking_area', PublicAPIParkingAreaViewSet, base_name='parkingarea') router.register(r'parking_area_statistics', PublicAPIParkingAreaStatisticsViewSet, base_name='parkingareastatistics') diff --git a/parkings/api/utils.py b/parkings/api/utils.py new file mode 100644 index 00000000..24f2cc49 --- /dev/null +++ b/parkings/api/utils.py @@ -0,0 +1,26 @@ +import dateutil.parser +from django.utils import timezone +from rest_framework.exceptions import ValidationError + + +def parse_timestamp_or_now(timestamp_string): + """ + Parse given timestamp string or return current time. + + If the timestamp string is falsy, return current time, otherwise try + to parse the string and return the parsed value. + + :type timestamp_string: str + :rtype: datetime.datetime + :raises rest_framework.exceptions.ValidationError: on parse error + """ + if not timestamp_string: + return timezone.now() + return parse_timestamp(timestamp_string) + + +def parse_timestamp(datetime_string): + try: + return dateutil.parser.parse(datetime_string) + except (ValueError, OverflowError): + raise ValidationError('Invalid timestamp: {}'.format(datetime_string)) diff --git a/parkings/factories/__init__.py b/parkings/factories/__init__.py index 5521e8ec..d4b3891e 100644 --- a/parkings/factories/__init__.py +++ b/parkings/factories/__init__.py @@ -1,4 +1,16 @@ from .operator import OperatorFactory # noqa from .parking import HistoryParkingFactory, ParkingFactory # noqa from .parking_area import ParkingAreaFactory # noqa +from .region import RegionFactory from .user import AdminUserFactory, StaffUserFactory, UserFactory # noqa + +__all__ = [ + 'AdminUserFactory', + 'HistoryParkingFactory', + 'OperatorFactory', + 'ParkingAreaFactory', + 'ParkingFactory', + 'RegionFactory', + 'StaffUserFactory', + 'UserFactory', +] diff --git a/parkings/factories/region.py b/parkings/factories/region.py new file mode 100644 index 00000000..85c293c5 --- /dev/null +++ b/parkings/factories/region.py @@ -0,0 +1,15 @@ +import factory + +from parkings.models import Region + +from .faker import fake +from .parking_area import generate_multi_polygon + + +class RegionFactory(factory.django.DjangoModelFactory): + class Meta: + model = Region + + geom = factory.LazyFunction(generate_multi_polygon) + capacity_estimate = fake.random.randint(0, 500) + name = factory.LazyFunction(fake.city) diff --git a/parkings/importers/regions.py b/parkings/importers/regions.py new file mode 100644 index 00000000..450e6dce --- /dev/null +++ b/parkings/importers/regions.py @@ -0,0 +1,55 @@ +import sys + +from django.contrib.gis.gdal import DataSource +from django.contrib.gis.utils import LayerMapping + +from ..models import Region + + +class ShapeFileToRegionImporter(object): + """ + Importer from ESRI Shapefiles to Region model in the database. + """ + field_mapping = { + # Region model field -> Field in the file + 'geom': 'MULTIPOLYGON', + } + + def __init__(self, filename, encoding='utf-8', + output_stream=sys.stderr, verbose=True): + self.filename = filename + self.encoding = encoding + self.data_source = DataSource(filename, encoding=encoding) + self.output_stream = output_stream + self.verbose = verbose + + def set_field_mapping(self, mapping): + self.field_mapping = dict(self.field_mapping, **mapping) + + def get_layer_names(self): + return [layer.name for layer in self.data_source] + + def get_layer_fields(self, name): + layer = self.data_source[self._get_layer_index(name)] + return layer.fields + + def import_from_layer(self, layer_name): + layer_mapping = LayerMapping( + model=Region, + data=self.filename, + mapping=self.field_mapping, + layer=self._get_layer_index(layer_name), + encoding=self.encoding) + silent = (self.output_stream is None) + layer_mapping.save( + strict=True, + stream=self.output_stream, + silent=silent, + verbose=(not silent and self.verbose)) + + def _get_layer_index(self, name): + layer_names = self.get_layer_names() + try: + return layer_names.index(name) + except ValueError: + raise ValueError('No such layer: {!r}'.format(name)) diff --git a/parkings/management/commands/fill_parking_regions.py b/parkings/management/commands/fill_parking_regions.py new file mode 100755 index 00000000..465f816f --- /dev/null +++ b/parkings/management/commands/fill_parking_regions.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +""" +Fill regions to Parking objects. +""" +import datetime + +from django.core.management.base import BaseCommand +from django.db import transaction + +from ...models import Parking, Region + + +class Command(BaseCommand): + help = __doc__.strip().splitlines()[0] + + def add_arguments(self, parser): + parser.add_argument( + 'block_size_target', type=int, nargs='?', default=20000, + help=( + "Block size target, " + "i.e. the number of parkings to process at time")) + + def handle(self, block_size_target, *args, **options): + verbosity = int(options['verbosity']) + silent = (verbosity == 0) + show_info = (self._print_and_flush if not silent else self._null_print) + + regions = Region.objects.all() + parkings = ( + Parking.objects + .exclude(location=None) + .filter(region=None) + .order_by('created_at')) + count = parkings.count() + + if not count: + show_info("Nothing to do") + return + + block_count = int(max(count / block_size_target, 1)) + start = parkings.first().created_at.replace(microsecond=0, second=0) + end = parkings.last().created_at + block_seconds = int((end - start).total_seconds() / block_count) + 1 + block_span = datetime.timedelta(seconds=block_seconds) + + for block_num in range(block_count): + block_start = start + (block_num * block_span) + block_end = start + ((block_num + 1) * block_span) + block = parkings.filter( + created_at__gte=block_start, + created_at__lt=block_end) + block_size = block.count() + + show_info( + "Processing block {:5d}/{:5d}, size {:6d}, {}--{}".format( + block_num + 1, block_count, block_size, + block_start, block_end), ending='') + + with transaction.atomic(): + for (n, region) in enumerate(regions): + if n % 10 == 0: + show_info('.', ending='') + in_region = block.filter(location__intersects=region.geom) + in_region.update(region=region) + show_info('', ending='\n') # Print end of line + + def _print_and_flush(self, *args, ending='\n'): + self.stdout.write(*args, ending=ending) + + def _null_print(self, *args, ending='\n'): + pass diff --git a/parkings/management/commands/import_regions.py b/parkings/management/commands/import_regions.py new file mode 100755 index 00000000..a8f6852f --- /dev/null +++ b/parkings/management/commands/import_regions.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +r""" +Import regions from file. + +Instructions to import data +=========================== + +From geoserver.hel.fi +--------------------- + + 1. Download feature data from http://geoserver.hel.fi/geoserver/web/ + + - Click "Layer Preview" + - Select "Helsinki_osa_alueet" + - Download as WFS / Shapefile + + 2. Unzip the file:: + + Helsinki_osa_alueet.zip + + 3. Run this import command to the Shapefile:: + + ./manage.py import_regions \ + --verbosity 2 \ + --encoding latin1 --name nimi_fi \ + Helsinki_osa_alueet.shp Helsinki_osa_alueet + +From ptp.hel.fi/avoindata +------------------------- + + 1. Download feature data from http://ptp.hel.fi/avoindata/ link + "Helsingin piirialuejako vuosilta 1995-2016 (zip) 24.2.2017 + Paikkatietohakemisto GeoPackage (ETRS-GK25 (EPSG:3879))", or use + direct link: + + http://ptp.hel.fi/avoindata/aineistot/HKI-aluejako-1995-2016-gpkg.zip + + 2. Unzip the file + + 3. Install tools for Geographic data conversion:: + + sudo apt install gdal-bin # Works on Ubuntu + + 4. Convert the GeoPackage data to ESRI Shapefile format:: + + ogr2ogr piirialuejako.shp piirialuejako-1995-2016.gpkg + + 5. Run this import command to the converted shp file:: + + ./manage.py import_regions piirialuejako.shp osa_alue_2016 +""" +import argparse + +from django.core.management.base import BaseCommand + +from ...importers.regions import ShapeFileToRegionImporter + + +class Command(BaseCommand): + help = __doc__.strip().splitlines()[0] + + def create_parser(self, prog_name, subcommand): + parser = super().create_parser(prog_name, subcommand) + parser.epilog = '\n'.join(__doc__.strip().splitlines()[2:]) + parser.formatter_class = argparse.RawDescriptionHelpFormatter + return parser + + def add_arguments(self, parser): + parser.add_argument( + 'filename', type=str, + help=("Path to the ESRI Shapefile (*.shp) to import from")) + + parser.add_argument( + 'layer_name', type=str, + help=("Name of the layer to import or \"LIST\" to get a list")) + + parser.add_argument('--encoding', type=str, default='utf-8') + parser.add_argument('--name-field', type=str, default='Nimi') + + def handle(self, filename, layer_name, encoding, name_field, + *args, **options): + verbosity = int(options['verbosity']) + importer = ShapeFileToRegionImporter( + filename, + encoding=encoding, + output_stream=(self.stdout if verbosity > 0 else None), + verbose=(verbosity >= 2)) + + importer.set_field_mapping({'name': name_field}) + + if layer_name == 'LIST': + for name in importer.get_layer_names(): + self.stdout.write(name) + for field in importer.get_layer_fields(name): + self.stdout.write(' - {}'.format(field)) + else: + importer.import_from_layer(layer_name) diff --git a/parkings/migrations/0017_region.py b/parkings/migrations/0017_region.py new file mode 100644 index 00000000..4439771c --- /dev/null +++ b/parkings/migrations/0017_region.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-09-20 02:22 +from __future__ import unicode_literals + +import uuid + +import django.contrib.gis.db.models.fields +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parkings', '0016_normalized_reg_num_noblank'), + ] + + operations = [ + migrations.CreateModel( + name='Region', + fields=[ + ('created_at', models.DateTimeField( + auto_now_add=True, verbose_name='time created')), + ('modified_at', models.DateTimeField( + auto_now=True, verbose_name='time modified')), + ('id', models.UUIDField( + default=uuid.uuid4, editable=False, + primary_key=True, serialize=False)), + ('name', models.CharField( + blank=True, max_length=200, verbose_name='name')), + ('geom', django.contrib.gis.db.models.fields.MultiPolygonField( + srid=3879, verbose_name='geometry')), + ], + options={ + 'verbose_name_plural': 'regions', + 'verbose_name': 'region', + }, + ), + migrations.AddField( + model_name='parkingarea', + name='region', + field=models.ForeignKey( + blank=True, null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='areas', + to='parkings.Region', verbose_name='region'), + ), + ] diff --git a/parkings/migrations/0018_parking_region.py b/parkings/migrations/0018_parking_region.py new file mode 100644 index 00000000..3111a548 --- /dev/null +++ b/parkings/migrations/0018_parking_region.py @@ -0,0 +1,29 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('parkings', '0017_region'), + ] + + operations = [ + migrations.RemoveField( + model_name='parkingarea', + name='region', ), + migrations.AddField( + model_name='parking', + name='region', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='parkings', + to='parkings.Region', + verbose_name='region')), + migrations.AddField( + model_name='region', + name='capacity_estimate', + field=models.PositiveIntegerField( + blank=True, null=True, verbose_name='capacity estimate')), + ] diff --git a/parkings/models/__init__.py b/parkings/models/__init__.py index 9b0fddc7..a2ba0c57 100644 --- a/parkings/models/__init__.py +++ b/parkings/models/__init__.py @@ -2,6 +2,7 @@ from .parking import Parking, ParkingQuerySet from .parking_area import ParkingArea from .parking_terminal import ParkingTerminal +from .region import Region __all__ = [ 'Operator', @@ -9,4 +10,5 @@ 'ParkingArea', 'ParkingTerminal', 'ParkingQuerySet', + 'Region', ] diff --git a/parkings/models/parking.py b/parkings/models/parking.py index 42411062..6eacb892 100644 --- a/parkings/models/parking.py +++ b/parkings/models/parking.py @@ -9,6 +9,7 @@ from parkings.models.parking_area import ParkingArea from .parking_terminal import ParkingTerminal +from .region import Region Q = models.Q @@ -44,6 +45,10 @@ class Parking(TimestampedModelMixin, UUIDPrimaryKeyMixin): VALID = 'valid' NOT_VALID = 'not_valid' + region = models.ForeignKey( + Region, null=True, blank=True, on_delete=models.SET_NULL, + related_name='parkings', verbose_name=_("region"), + ) parking_area = models.ForeignKey( ParkingArea, on_delete=models.SET_NULL, verbose_name=_("parking area"), related_name='parkings', null=True, blank=True, @@ -106,6 +111,11 @@ def get_state(self): return Parking.VALID + def get_region(self): + if not self.location: + return None + return Region.objects.filter(geom__intersects=self.location).first() + def get_closest_area(self, max_distance=50): if self.location: location = self.location @@ -117,7 +127,7 @@ def get_closest_area(self, max_distance=50): ).filter(distance__lte=max_distance).order_by('distance').first() return closest_area - def save(self, *args, **kwargs): + def save(self, update_fields=None, *args, **kwargs): if not self.terminal and self.terminal_number: self.terminal = ParkingTerminal.objects.filter( number=_try_cast_int(self.terminal_number)).first() @@ -125,12 +135,16 @@ def save(self, *args, **kwargs): if self.terminal and not self.location: self.location = self.terminal.location - self.parking_area = self.get_closest_area() + if update_fields is None or 'region' in update_fields: + self.region = self.get_region() + if update_fields is None or 'parking_area' in update_fields: + self.parking_area = self.get_closest_area() - self.normalized_reg_num = ( - self.normalize_reg_num(self.registration_number)) + if update_fields is None or 'normalized_reg_num' in update_fields: + self.normalized_reg_num = ( + self.normalize_reg_num(self.registration_number)) - super(Parking, self).save(*args, **kwargs) + super(Parking, self).save(update_fields=update_fields, *args, **kwargs) @classmethod def normalize_reg_num(cls, registration_number): diff --git a/parkings/models/parking_area.py b/parkings/models/parking_area.py index c096dd02..f6ca46d5 100644 --- a/parkings/models/parking_area.py +++ b/parkings/models/parking_area.py @@ -1,8 +1,68 @@ from django.contrib.gis.db import models +from django.contrib.gis.db.models.functions import Area +from django.db.models import Func, Sum from django.utils.translation import ugettext_lazy as _ from parkings.models.mixins import TimestampedModelMixin, UUIDPrimaryKeyMixin +# Estimated number of parking spots per square meter (m^2) +PARKING_SPOTS_PER_SQ_M = 0.07328 + + +class Round(Func): + """ + Function for rounding in SQL using Django ORM. + """ + function = 'ROUND' + + +class ParkingAreaQuerySet(models.QuerySet): + def inside_region(self, region): + """ + Filter to parking areas which are inside given region. + + :type region: .region.Region + :rtype: ParkingAreaQuerySet + """ + return self.filter(geom__coveredby=region.geom) + + def intersecting_region(self, region): + """ + Filter to parking areas which intersect with given region. + + :type region: .region.Region + :rtype: ParkingAreaQuerySet + """ + return self.filter(geom__intersects=region.geom) + + @property + def total_estimated_capacity(self): + """ + Estimated number of parking spots in this queryset. + + This is the sum of `parking_area.estimated_capacity` of each + `parking_area` in this queryset. The value is calculated in the + database to make it faster compared to calculating the sum in + Python. + + :rtype: int + """ + defined_qs = self.exclude(capacity_estimate=None) + undefined_qs = self.filter(capacity_estimate=None) + total_defined = defined_qs.sum_of_capacity_estimates or 0 + total_by_area = undefined_qs.estimate_total_capacity_by_areas() + return total_defined + total_by_area + + @property + def sum_of_capacity_estimates(self): + return self.aggregate(val=Sum('capacity_estimate'))['val'] + + def estimate_total_capacity_by_areas(self): + spots = ( + self.annotate(spots=Round(PARKING_SPOTS_PER_SQ_M * Area('geom'))) + .aggregate(val=Sum('spots')))['val'] + return int(spots.sq_m) if spots else 0 + class ParkingArea(TimestampedModelMixin, UUIDPrimaryKeyMixin): # This is for whatever ID the external system that this parking lot was @@ -34,9 +94,35 @@ class ParkingArea(TimestampedModelMixin, UUIDPrimaryKeyMixin): blank=True, ) + objects = ParkingAreaQuerySet.as_manager() + class Meta: verbose_name = _('parking area') verbose_name_plural = _('parking areas') def __str__(self): return 'Parking Area %s' % str(self.origin_id) + + @property + def estimated_capacity(self): + """ + Estimated number of parking spots in this parking area. + + If there is a capacity estimate specified for this area in the + `capacity_estimate` field, return it, otherwise return a + capacity estimate based on the area (m^2) of this parking area. + + :rtype: int + """ + return ( + self.capacity_estimate if self.capacity_estimate is not None + else self.estimate_capacity_by_area()) + + def estimate_capacity_by_area(self): + """ + Estimate number of parking spots in this parking area by m^2. + + :rtype: int + """ + assert self.geom.srs.units == (1.0, 'metre') + return int(round(self.geom.area * PARKING_SPOTS_PER_SQ_M)) diff --git a/parkings/models/region.py b/parkings/models/region.py new file mode 100644 index 00000000..7f35ed5b --- /dev/null +++ b/parkings/models/region.py @@ -0,0 +1,60 @@ +from django.contrib.gis.db import models as gis_models +from django.contrib.gis.db.models.functions import Intersection +from django.db import models +from django.db.models import Case, Count, Q, When +from django.utils import timezone +from django.utils.translation import ugettext_lazy as _ + +from .mixins import TimestampedModelMixin, UUIDPrimaryKeyMixin +from .parking_area import ParkingArea + + +class RegionQuerySet(models.QuerySet): + def with_parking_count(self, at_time=None): + time = at_time if at_time else timezone.now() + valid_parkings_q = ( + Q(parkings__time_start__lte=time) & + (Q(parkings__time_end__gte=time) | Q(parkings__time_end=None))) + return self.annotate( + parking_count=Count(Case(When(valid_parkings_q, then=1)))) + + +class Region(TimestampedModelMixin, UUIDPrimaryKeyMixin): + name = models.CharField(max_length=200, blank=True, verbose_name=_("name")) + geom = gis_models.MultiPolygonField(srid=3879, verbose_name=_("geometry")) + capacity_estimate = models.PositiveIntegerField( + blank=True, null=True, + verbose_name=_("capacity estimate"), + ) + + objects = RegionQuerySet.as_manager() + + class Meta: + verbose_name = _("region") + verbose_name_plural = _("regions") + + def __str__(self): + return self.name or str(_("Unnamed region")) + + def save(self, *args, **kwargs): + self.capacity_estimate = self.calculate_capacity_estimate() + super().save(*args, **kwargs) + + def calculate_capacity_estimate(self): + """ + Calculate capacity estimate of this region from parking areas. + + :rtype: int + """ + areas = ParkingArea.objects.all() + covered_areas = areas.inside_region(self) + partial_areas = ( + areas.intersecting_region(self).exclude(pk__in=covered_areas) + .annotate(intsect=Intersection('geom', self.geom))) + sum_covered = covered_areas.total_estimated_capacity + sum_partials = sum( + # Scale the estimated capacity according to ratio of area of + # the intersection and total area + int(round(x.estimated_capacity * x.intsect.area / x.geom.area)) + for x in partial_areas) + return sum_covered + sum_partials diff --git a/parkings/pagination.py b/parkings/pagination.py index 26aefbd2..67a60745 100644 --- a/parkings/pagination.py +++ b/parkings/pagination.py @@ -1,5 +1,5 @@ from rest_framework.pagination import PageNumberPagination -class PublicAPIPagination(PageNumberPagination): +class Pagination(PageNumberPagination): page_size_query_param = 'page_size' diff --git a/parkings/tests/api/conftest.py b/parkings/tests/api/conftest.py index 63b709c8..fe32cfdd 100644 --- a/parkings/tests/api/conftest.py +++ b/parkings/tests/api/conftest.py @@ -14,6 +14,15 @@ def api_client(): return APIClient() +@pytest.fixture +def monitoring_api_client(user_factory): + api_client = APIClient() + user = user_factory() + user.groups.get_or_create(name='monitoring') + api_client.force_authenticate(user) + return api_client + + @pytest.fixture def user_api_client(user_factory): api_client = APIClient() diff --git a/parkings/tests/api/enforcement/test_operator.py b/parkings/tests/api/enforcement/test_operator.py index 7fa11bf9..e21a1ba3 100644 --- a/parkings/tests/api/enforcement/test_operator.py +++ b/parkings/tests/api/enforcement/test_operator.py @@ -14,7 +14,8 @@ def get_url(kind, operator): if kind == 'list': return list_url - elif kind == 'detail': + else: + assert kind == 'detail' return reverse('enforcement:v1:operator-detail', kwargs={'pk': operator.pk}) diff --git a/parkings/tests/api/enforcement/test_valid_parking.py b/parkings/tests/api/enforcement/test_valid_parking.py index 28658792..8164c36a 100644 --- a/parkings/tests/api/enforcement/test_valid_parking.py +++ b/parkings/tests/api/enforcement/test_valid_parking.py @@ -25,7 +25,8 @@ def get_url(kind, parking): return list_url elif kind == 'list_by_reg_num': return list_url_for(parking.registration_number) - elif kind == 'detail': + else: + assert kind == 'detail' return reverse('enforcement:v1:valid_parking-detail', kwargs={'pk': parking.pk}) diff --git a/parkings/tests/api/monitoring/__init__.py b/parkings/tests/api/monitoring/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/parkings/tests/api/monitoring/test_permissions.py b/parkings/tests/api/monitoring/test_permissions.py new file mode 100644 index 00000000..360a34fe --- /dev/null +++ b/parkings/tests/api/monitoring/test_permissions.py @@ -0,0 +1,38 @@ +import pytest +from django.contrib.auth.models import AnonymousUser, User + +from parkings.api.monitoring.permissions import MonitoringApiPermission + + +@pytest.mark.django_db +def test_monitoring_api_permission_anonymous(rf): + request = rf.get('/') + request.user = None + perm = MonitoringApiPermission() + assert perm.has_permission(request, None) is False + request.user = AnonymousUser() + assert perm.has_permission(request, None) is False + + +@pytest.mark.django_db +def test_monitoring_api_permission_not_in_group(rf, user): + request = rf.get('/') + request.user = user + perm = MonitoringApiPermission() + assert perm.has_permission(request, None) is False + + +@pytest.mark.django_db +@pytest.mark.parametrize('group,allowed', [ + ('monitoring', True), ('othergrp', False)]) +def test_monitoring_api_permission_by_group(group, allowed, rf): + user = User.objects.create_user('dummy-user') + user.groups.create(name=group) + try: + request = rf.get('/') + request.user = user + perm = MonitoringApiPermission() + assert perm.has_permission(request, None) is allowed + finally: + user.groups.all().delete() + user.delete() diff --git a/parkings/tests/api/monitoring/test_region.py b/parkings/tests/api/monitoring/test_region.py new file mode 100644 index 00000000..7de288f8 --- /dev/null +++ b/parkings/tests/api/monitoring/test_region.py @@ -0,0 +1,58 @@ +import json + +from django.urls import reverse +from rest_framework import status + +WGS84_SRID = 4326 + + +list_url = reverse('monitoring:v1:region-list') + + +def test_get_regions_empty(monitoring_api_client): + result = monitoring_api_client.get(list_url) + assert result.data == { + 'type': 'FeatureCollection', + 'features': [], + 'count': 0, + 'next': None, + 'previous': None, + } + assert result.status_code == status.HTTP_200_OK + + +def test_get_regions_with_data(monitoring_api_client, region, parking_area): + parking_area.geom = region.geom + parking_area.save() + region.save() # Update capacity_estimate + + result = monitoring_api_client.get(list_url) + features = result.data.pop('features', None) + assert result.data == { + 'type': 'FeatureCollection', + 'count': 1, + 'next': None, + 'previous': None, + } + assert result.status_code == status.HTTP_200_OK + assert isinstance(features, list) + assert len(features) == 1 + geometry = features[0].pop('geometry', None) + properties = features[0].pop('properties', None) + assert features[0] == {'id': str(region.id), 'type': 'Feature'} + km2 = region.geom.area / 1000000.0 + assert properties == { + 'name': region.name, + 'capacity_estimate': region.capacity_estimate, + 'area_km2': km2, + 'spots_per_km2': region.capacity_estimate / km2, + 'parking_areas': [parking_area.id], + } + coordinates = geometry.pop('coordinates', None) + assert geometry == {'type': 'MultiPolygon'} + wgs84_geom = region.geom.transform(WGS84_SRID, clone=True) + assert coordinates == tuples_to_lists(wgs84_geom.coords) + + +def tuples_to_lists(tuples_of_tuples): + return json.loads(json.dumps(tuples_of_tuples)) diff --git a/parkings/tests/api/monitoring/test_region_statistics.py b/parkings/tests/api/monitoring/test_region_statistics.py new file mode 100644 index 00000000..fd08e242 --- /dev/null +++ b/parkings/tests/api/monitoring/test_region_statistics.py @@ -0,0 +1,86 @@ +from collections import OrderedDict + +import pytz +from django.urls import reverse +from rest_framework import status + +from parkings.factories.faker import fake + +from ...utils import create_parkings_and_regions, intersects + +list_url = reverse('monitoring:v1:regionstatistics-list') + + +def test_empty(monitoring_api_client): + result = monitoring_api_client.get(list_url) + assert result.data == { + 'count': 0, + 'next': None, + 'previous': None, + 'results': [], + } + assert result.status_code == status.HTTP_200_OK + + +def test_with_single_parking(monitoring_api_client, region, parking): + point_in_region = region.geom.centroid + parking.location = point_in_region + parking.save() + assert intersects(point_in_region, region) + + result = monitoring_api_client.get(list_url) + assert result.data == { + 'count': 1, + 'next': None, + 'previous': None, + 'results': [ + OrderedDict([ + ('id', str(region.id)), + ('parking_count', 1)]), + ] + } + assert result.status_code == status.HTTP_200_OK + + +def test_with_many_parkings_and_specified_time(monitoring_api_client): + (parkings, regions) = create_parkings_and_regions( + parking_count=20, region_count=5) + + # Pick a point in time that is in between the parking times + earliest_end_time = min(x.time_end for x in parkings) + latest_end_time = max(x.time_end for x in parkings) + time = fake.date_time_between(earliest_end_time, latest_end_time, + tzinfo=pytz.utc) + + # Calculate some lookup containers + valid_parkings_at_time = [ + parking for parking in parkings + if parking.time_start <= time and (parking.time_end is None + or parking.time_end >= time)] + regions_with_valid_parkings = { + parking.region for parking in valid_parkings_at_time + if parking.region + } + regions_by_id = {str(region.id): region for region in regions} + + # Do the API call + api_result = monitoring_api_client.get(list_url, {'time': str(time)}) + + # Check the results + results = api_result.data.pop('results', None) + assert api_result.data == { + 'count': len(regions_with_valid_parkings), + 'next': None, # No paging, since parking count < page size + 'previous': None, + } + assert len(results) == len(regions_with_valid_parkings) + for result in results: + assert isinstance(result, OrderedDict) + assert set(result.keys()) == {'id', 'parking_count'} + assert result['id'] in regions_by_id + region = regions_by_id[result['id']] + expected_count = sum( + 1 for parking in valid_parkings_at_time + if parking.region == region) + assert result['parking_count'] == expected_count + assert api_result.status_code == status.HTTP_200_OK diff --git a/parkings/tests/api/operator/test_parking.py b/parkings/tests/api/operator/test_parking.py index 5e0fbb67..8f7505ac 100644 --- a/parkings/tests/api/operator/test_parking.py +++ b/parkings/tests/api/operator/test_parking.py @@ -41,13 +41,6 @@ def updated_parking_data(): } -def check_parking_data_keys(parking_data): - assert set(parking_data.keys()) == { - 'id', 'created_at', 'modified_at', 'location', 'registration_number', 'time_start', 'time_end', 'zone', - 'status', - } - - def check_parking_data_matches_parking_object(parking_data, parking_obj): """ Check that a parking data dict and an actual Parking object match. diff --git a/parkings/tests/api/public/test_parking_area_statistics.py b/parkings/tests/api/public/test_parking_area_statistics.py index ecc19c78..99f35978 100644 --- a/parkings/tests/api/public/test_parking_area_statistics.py +++ b/parkings/tests/api/public/test_parking_area_statistics.py @@ -45,10 +45,13 @@ def test_get_list_check_data(api_client, parking_factory, parking_area_factory, results = get(api_client, list_url)['results'] assert len(results) == 4 - stats_data_1 = next(result for result in results if result['id'] == str(parking_area_1.id)) - stats_data_2 = next(result for result in results if result['id'] == str(parking_area_2.id)) - stats_data_3 = next(result for result in results if result['id'] == str(parking_area_3.id)) - stats_data_4 = next(result for result in results if result['id'] == str(parking_area_4.id)) + def find_by_obj_id(obj, iterable): + return [x for x in iterable if x['id'] == str(obj.id)][0] + + stats_data_1 = find_by_obj_id(parking_area_1, results) + stats_data_2 = find_by_obj_id(parking_area_2, results) + stats_data_3 = find_by_obj_id(parking_area_3, results) + stats_data_4 = find_by_obj_id(parking_area_4, results) assert stats_data_1.keys() == {'id', 'current_parking_count'} assert stats_data_1['current_parking_count'] == 4 diff --git a/parkings/tests/api/test_utils.py b/parkings/tests/api/test_utils.py new file mode 100644 index 00000000..aa7829b2 --- /dev/null +++ b/parkings/tests/api/test_utils.py @@ -0,0 +1,39 @@ +import datetime + +import pytest +from freezegun import freeze_time +from rest_framework.exceptions import ValidationError as DrfValidationError + +from parkings.api.utils import parse_timestamp, parse_timestamp_or_now + + +def test_parse_timestamp_or_now_with_val(): + parsed = parse_timestamp_or_now('2000-01-01T20:01:05+0200') + assert isinstance(parsed, datetime.datetime) + assert str(parsed) == '2000-01-01 20:01:05+02:00' + + +@freeze_time('2000-02-29T10:15:22Z') +def test_parse_timestamp_or_now_with_empty(): + parsed = parse_timestamp_or_now('') + assert isinstance(parsed, datetime.datetime) + assert str(parsed) == '2000-02-29 10:15:22+00:00' + + +@freeze_time('2000-02-29T10:15:22Z') +def test_parse_timestamp_or_now_with_none(): + parsed = parse_timestamp_or_now(None) + assert isinstance(parsed, datetime.datetime) + assert str(parsed) == '2000-02-29 10:15:22+00:00' + + +def test_parse_timestamp_with_valid_data(): + parsed = parse_timestamp('2000-02-29T10:15:22-12:00') + assert isinstance(parsed, datetime.datetime) + assert str(parsed) == '2000-02-29 10:15:22-12:00' + + +def test_parse_timestamp_with_invalid_data(): + with pytest.raises(DrfValidationError) as excinfo: + parse_timestamp('foobar') + assert str(excinfo.value) == "['Invalid timestamp: foobar']" diff --git a/parkings/tests/api/utils.py b/parkings/tests/api/utils.py index 01f7dcb4..23c2c63b 100644 --- a/parkings/tests/api/utils.py +++ b/parkings/tests/api/utils.py @@ -72,7 +72,7 @@ def check_required_fields(api_client, url, expected_required_fields, detail_endp required_fields = set() for (field, errors) in response_data.items(): - if 'This field is required.' in repr(errors): + if 'This field is required.' in repr(errors): # pragma: no cover required_fields.add(field) assert required_fields == expected_required_fields, '%s != %s' % (required_fields, expected_required_fields) diff --git a/parkings/tests/conftest.py b/parkings/tests/conftest.py index c676bc46..4eefb9cc 100644 --- a/parkings/tests/conftest.py +++ b/parkings/tests/conftest.py @@ -3,7 +3,8 @@ from parkings.factories import ( AdminUserFactory, HistoryParkingFactory, OperatorFactory, - ParkingAreaFactory, ParkingFactory, StaffUserFactory, UserFactory) + ParkingAreaFactory, ParkingFactory, RegionFactory, StaffUserFactory, + UserFactory) register(OperatorFactory) register(ParkingFactory, 'parking') @@ -12,6 +13,7 @@ register(StaffUserFactory, 'staff_user') register(UserFactory) register(ParkingAreaFactory) +register(RegionFactory) @pytest.fixture(autouse=True) diff --git a/parkings/tests/test-features.dbf b/parkings/tests/test-features.dbf new file mode 100644 index 00000000..8ac5d52e Binary files /dev/null and b/parkings/tests/test-features.dbf differ diff --git a/parkings/tests/test-features.prj b/parkings/tests/test-features.prj new file mode 100644 index 00000000..a30c00a5 --- /dev/null +++ b/parkings/tests/test-features.prj @@ -0,0 +1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] \ No newline at end of file diff --git a/parkings/tests/test-features.shp b/parkings/tests/test-features.shp new file mode 100644 index 00000000..4d3976a0 Binary files /dev/null and b/parkings/tests/test-features.shp differ diff --git a/parkings/tests/test-features.shx b/parkings/tests/test-features.shx new file mode 100644 index 00000000..67b3b338 Binary files /dev/null and b/parkings/tests/test-features.shx differ diff --git a/parkings/tests/test_fill_parking_regions_command.py b/parkings/tests/test_fill_parking_regions_command.py new file mode 100644 index 00000000..8cb3c506 --- /dev/null +++ b/parkings/tests/test_fill_parking_regions_command.py @@ -0,0 +1,80 @@ +import re + +import pytest +from dateutil.parser import parse as parse_date + +from parkings.management.commands import fill_parking_regions + +from .utils import ( + call_mgmt_cmd_with_output, create_parkings_and_regions, intersects, + intersects_with_any) + + +@pytest.mark.django_db +def test_fill_parking_regions_mgmt_cmd(): + (parkings, regions) = create_parkings_and_regions() + + # Clear the regions + for parking in parkings: + # First save without a location to prevent region being autofilled + old_location = parking.location + parking.location = None + parking.region = None + parking.save() + # Then fill the location back, but save only location + parking.location = old_location + parking.save(update_fields=['location']) + + assert all(parking.region is None for parking in parkings) + + # Call the command with output streams attached + target_block_size = 5 + (stdout, stderr) = call_the_command(target_block_size) + + # Check the results + for parking in parkings: + parking.refresh_from_db() + if parking.location and intersects_with_any(parking.location, regions): + assert parking.region is not None + assert intersects(parking.location, parking.region) + else: + assert parking.region is None + + # Check the outputted lines + block_count = len(parkings) // target_block_size + for (n, line) in enumerate(stdout.splitlines(), 1): + match = re.match( + '^Processing block +(\d+)/ *(\d+), size +(\d+), ([^.]*)(\.+)$', + line) + assert match, 'Invalid output line {}: {!r}'.format(n, line) + assert match.group(1) == str(n) + assert match.group(2) == str(block_count) + assert 0 <= int(match.group(3)) <= len(parkings) + (start_str, end_str) = match.group(4).split('--') + block_start = parse_date(start_str) + block_end = parse_date(end_str) + assert block_start <= block_end + assert len(match.group(5)) == len(regions) // 10 + assert stderr == '' + + # Check that the command doesn't do anything if all parkings with a + # location have region filled + for parking in parkings: + if not parking.region: + parking.location = None + parking.save() + (stdout, stderr) = call_the_command(target_block_size) + assert stdout == 'Nothing to do\n' + assert stderr == '' + + # And finally check that it doesn't print anything when verbosity=0 + (stdout, stderr) = call_the_command(target_block_size, verbosity=0) + assert stdout == '' + assert stderr == '' + + +def call_the_command(*args, **kwargs): + (result, stdout, stderr) = call_mgmt_cmd_with_output( + fill_parking_regions.Command, *args, **kwargs) + assert result is None + return (stdout, stderr) diff --git a/parkings/tests/test_parking_area_model.py b/parkings/tests/test_parking_area_model.py new file mode 100644 index 00000000..9622e311 --- /dev/null +++ b/parkings/tests/test_parking_area_model.py @@ -0,0 +1,22 @@ +import pytest + +from parkings.models import ParkingArea + + +def test_str(): + assert str(ParkingArea(origin_id='TEST_ID')) == 'Parking Area TEST_ID' + + +@pytest.mark.django_db +def test_estimated_capacity(parking_area): + parking_area.capacity_estimate = 123 + assert parking_area.estimated_capacity == 123 + parking_area.capacity_estimate = None + by_area = parking_area.estimate_capacity_by_area() + assert parking_area.estimated_capacity == by_area + + +@pytest.mark.django_db +def test_estimate_capacity_by_area(parking_area): + assert parking_area.estimate_capacity_by_area() == int( + round(parking_area.geom.area * 0.07328)) diff --git a/parkings/tests/test_region_importer.py b/parkings/tests/test_region_importer.py new file mode 100644 index 00000000..984f0476 --- /dev/null +++ b/parkings/tests/test_region_importer.py @@ -0,0 +1,94 @@ +import os + +import pytest + +from parkings.importers.regions import ShapeFileToRegionImporter +from parkings.management.commands import import_regions +from parkings.models import Region + +from .utils import call_mgmt_cmd_with_output + +directory = os.path.abspath(os.path.dirname(__file__)) + +shp_path = os.path.join(directory, 'test-features.shp') + + +def test_get_layer_names(): + importer = ShapeFileToRegionImporter(shp_path) + assert importer.get_layer_names() == ['test-features'] + + +def test_get_layer_fields(): + importer = ShapeFileToRegionImporter(shp_path) + assert importer.get_layer_fields('test-features') == [ + 'Name', 'descriptio', 'timestamp', 'begin', 'end', + 'altitudeMo', 'tessellate', 'extrude', 'visibility', + 'drawOrder', 'icon', 'DisplayNam', 'year'] + + +def test_get_layer_fields_invalid_layer_name(): + importer = ShapeFileToRegionImporter(shp_path) + with pytest.raises(ValueError) as excinfo: + importer.get_layer_fields('invalid-layer-name') + assert str(excinfo.value) == "No such layer: 'invalid-layer-name'" + + +@pytest.mark.django_db +def test_import(): + assert Region.objects.count() == 0 + importer = ShapeFileToRegionImporter(shp_path) + importer.set_field_mapping({'name': 'DisplayNam'}) + importer.import_from_layer('test-features') + check_imported_regions() + + +@pytest.mark.django_db +def test_management_command(): + call_the_command(shp_path, 'test-features', name_field='DisplayNam') + check_imported_regions() + + (stdout, stderr) = call_the_command(shp_path, 'LIST') + assert stdout.splitlines() == [ + 'test-features', + ' - Name', + ' - descriptio', + ' - timestamp', + ' - begin', + ' - end', + ' - altitudeMo', + ' - tessellate', + ' - extrude', + ' - visibility', + ' - drawOrder', + ' - icon', + ' - DisplayNam', + ' - year'] + assert stdout.endswith('\n') + assert stderr == '' + + +def call_the_command(*args, **kwargs): + (result, stdout, stderr) = call_mgmt_cmd_with_output( + import_regions.Command, *args, **kwargs) + assert result is None + return (stdout, stderr) + + +def check_imported_regions(): + assert Region.objects.count() == 2 + (reg1, reg2) = list(Region.objects.order_by('name')) + assert reg1.name == 'Feature 1 - Center' + assert reg2.name == 'Feature 2 - North' + assert reg1.geom.coords == ((( + (25494876.99362251, 6677378.512999998), + (25494929.966569535, 6677389.664999997), + (25495159.757339746, 6677117.191999998), + (25494819.72517978, 6677425.432), + (25494876.99362251, 6677378.512999998), + ),),) + assert reg2.geom.coords == ((( + (25494360.817638688, 6684192.751999998), + (25494493.262506243, 6684249.482999996), + (25494337.233162273, 6684151.803999996), + (25494360.817638688, 6684192.751999998) + ),),) diff --git a/parkings/tests/test_region_model.py b/parkings/tests/test_region_model.py new file mode 100644 index 00000000..82b73216 --- /dev/null +++ b/parkings/tests/test_region_model.py @@ -0,0 +1,73 @@ +import pytest +from django.contrib.gis.geos import MultiPolygon, Polygon + +from parkings.models import ParkingArea, Region + + +def test_str(): + assert str(Region(name='Foobba')) == 'Foobba' + assert str(Region(name=None)) == 'Unnamed region' + assert str(Region(name='')) == 'Unnamed region' + assert isinstance(str(Region()), str) + + +@pytest.mark.django_db +def test_calculate_capacity_estimate(): + """ + Test Region capacity estimate calculation. + + Construct the following geometries: + + - reg = Region to estimate + - pa1 = Parking area 1, capacity=3, inside reg + - pa2 = Parking area 2, capacity=4, inside reg + - pa3 = Parking area 3, capacity=15, 1/3 of area inside reg + - pa4 = Parking area 4, capacity=20, outside reg + + Total capacity should be 3 + 4 + (1/3)*15 = 12. + + Same as a graph:: + + +-------------------------------+ + | reg | + | +--------+ +---------+ +---------------------+ +-------... + | | pa1 | | pa2 | | | pa3 | | pa4 + | +--------+ +---------+ +---------------------+ +-------... + | capacity=3 capacity=4 | capacity=15 capacity=20 + | | + +-------------------------------+ + """ + def rect(topleft, width, height): + """ + Construct a rectangle as multipolygon. + """ + (ax, ay) = topleft # left = ax, top = ay + (bx, by) = (ax + width, ay + height) # right = bx, bottom = by + return MultiPolygon(Polygon( + [(ax, ay), (ax, by), (bx, by), (bx, ay), (ax, ay)])) + + reg = Region(geom=rect((0, 0), 100, 14)) + pa1 = ParkingArea(geom=rect((2, 2), 20, 10), capacity_estimate=3) + pa2 = ParkingArea(geom=rect((30, 2), 20, 10), capacity_estimate=4) + pa3 = ParkingArea(geom=rect((60, 2), 120, 10), capacity_estimate=15) + pa4 = ParkingArea(geom=rect((200, 2), 40, 10), capacity_estimate=20) + + assert reg.geom.area == 1400 + assert pa1.geom.area == 200 + assert pa2.geom.area == 200 + assert pa3.geom.area == 1200 + assert pa4.geom.area == 400 + + assert reg.geom.intersection(pa1.geom).area == pa1.geom.area + assert reg.geom.intersection(pa2.geom).area == pa2.geom.area + assert reg.geom.intersection(pa3.geom).area == pa3.geom.area / 3 + assert reg.geom.intersection(pa4.geom).area == 0 + + # Save them to the database + reg.save() + for (n, pa) in enumerate([pa1, pa2, pa3, pa4]): + pa.origin_id = 'PA-{}'.format(n) + pa.save() + + # And finally, check the result of the calculation is correct + assert reg.calculate_capacity_estimate() == 12 diff --git a/parkings/tests/utils.py b/parkings/tests/utils.py new file mode 100644 index 00000000..fefd154f --- /dev/null +++ b/parkings/tests/utils.py @@ -0,0 +1,49 @@ +import io + +from django.core import management + +from parkings.factories import ParkingFactory, RegionFactory + + +def call_mgmt_cmd_with_output(command_cls, *args, **kwargs): + assert issubclass(command_cls, management.BaseCommand) + stdout = io.StringIO() + stderr = io.StringIO() + cmd = command_cls(stdout=stdout, stderr=stderr) + assert isinstance(cmd, management.BaseCommand) + result = management.call_command(cmd, *args, **kwargs) + return (result, stdout.getvalue(), stderr.getvalue()) + + +def create_parkings_and_regions(parking_count=100, region_count=20): + regions = RegionFactory.create_batch(region_count) + parkings = ParkingFactory.create_batch(parking_count) + + centroids = [region.geom.centroid for region in regions] + touching_points = [p for p in centroids if intersects_with_any(p, regions)] + + # Make sure that some of the parkings are inside the regions + for (point, parking) in zip(touching_points, parkings): + parking.location = point + parking.save() + + for parking in parkings: # pragma: no cover + if intersects_with_any(parking.location, regions): + assert parking.region + assert intersects(parking.location, parking.region) + else: + assert parking.region is None + + return (parkings, regions) + + +def intersects_with_any(point, regions): + assert regions + p = point.transform(regions[0].geom.srid, clone=True) + assert all(x.geom.srid == p.srid for x in regions) + return any(p.intersects(x.geom) for x in regions) + + +def intersects(point, region): + geom = region.geom + return point.transform(geom.srid, clone=True).intersects(geom) diff --git a/parkkihubi/settings.py b/parkkihubi/settings.py index 4703d3e5..914892e1 100644 --- a/parkkihubi/settings.py +++ b/parkkihubi/settings.py @@ -18,6 +18,7 @@ # Django core settings # ######################## DEBUG = env.bool('DEBUG', default=False) +TIER = env.str('TIER', default='dev') SECRET_KEY = env.str('SECRET_KEY', default=('' if not DEBUG else 'xxx')) ALLOWED_HOSTS = ['*'] @@ -71,6 +72,7 @@ 'django.contrib.staticfiles', 'django.contrib.gis', 'raven.contrib.django.raven_compat', + 'corsheaders', 'rest_framework', 'rest_framework.authtoken', 'rest_framework_gis', @@ -79,7 +81,7 @@ 'parkings', ] -if DEBUG: +if DEBUG and TIER == 'dev': # shell_plus and other goodies INSTALLED_APPS.append("django_extensions") @@ -88,6 +90,7 @@ ############## MIDDLEWARE = [ 'django.contrib.sessions.middleware.SessionMiddleware', + 'corsheaders.middleware.CorsMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'parkkihubi.middleware.AdminTimezoneMiddleware', @@ -150,27 +153,62 @@ vars().update(env.email_url( default=('consolemail://' if DEBUG else 'smtp://localhost:25') )) +DEFAULT_FROM_EMAIL = 'no-reply.parkkihubi@fiupparkp01.anders.fi' ######################### # Django REST Framework # ######################### REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': [ + # Make nothing accessible to non-admins by default. Viewsets + # should specify permission_classes to override permissions. + 'rest_framework.permissions.IsAdminUser', + ], + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'parkings.authentication.ApiKeyAuthentication', + 'drf_jwt_2fa.authentication.Jwt2faAuthentication', + ] + ([ # Following two are only for DEBUG mode in dev environment: + 'rest_framework.authentication.SessionAuthentication', + 'rest_framework.authentication.BasicAuthentication', + ] if (DEBUG and TIER == 'dev') else []), 'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.NamespaceVersioning', 'ALLOWED_VERSIONS': ('v1',), 'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',), 'EXCEPTION_HANDLER': 'parkings.exception_handler.parkings_exception_handler', - 'PAGE_SIZE': 20, + 'PAGE_SIZE': 100, 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', 'TEST_REQUEST_DEFAULT_FORMAT': 'json', } +JWT_AUTH = { + 'JWT_EXPIRATION_DELTA': timedelta(minutes=30), + 'JWT_ALLOW_REFRESH': True, + 'JWT_REFRESH_EXPIRATION_DELTA': timedelta(days=7), +} + +JWT2FA_AUTH = { + 'CODE_TOKEN_THROTTLE_RATE': '5/15m', + 'AUTH_TOKEN_RETRY_WAIT_TIME': timedelta(seconds=10), + 'EMAIL_SENDER_SUBJECT_OVERRIDE': '{code} - Varmennuskoodisi', + 'EMAIL_SENDER_BODY_OVERRIDE': ( + 'Hei!\n' + '\n' + 'Varmennuskoodisi kirjautumista varten on: {code}\n' + '\n' + 't. Parkkihubi'), +} + +CORS_ORIGIN_ALLOW_ALL = True ############## # Parkkihubi # ############## +MONITORING_GROUP_NAME = 'monitoring' PARKKIHUBI_TIME_PARKINGS_EDITABLE = timedelta(minutes=2) PARKKIHUBI_TIME_OLD_PARKINGS_VISIBLE = timedelta(minutes=15) PARKKIHUBI_PUBLIC_API_ENABLED = env.bool('PARKKIHUBI_PUBLIC_API_ENABLED', True) +PARKKIHUBI_MONITORING_API_ENABLED = env.bool( + 'PARKKIHUBI_MONITORING_API_ENABLED', True) PARKKIHUBI_OPERATOR_API_ENABLED = env.bool('PARKKIHUBI_OPERATOR_API_ENABLED', True) PARKKIHUBI_ENFORCEMENT_API_ENABLED = ( env.bool('PARKKIHUBI_ENFORCEMENT_API_ENABLED', True)) diff --git a/parkkihubi/urls.py b/parkkihubi/urls.py index 45fd7d7a..bb6c4f44 100644 --- a/parkkihubi/urls.py +++ b/parkkihubi/urls.py @@ -2,15 +2,24 @@ from django.conf.urls import include, url from django.contrib import admin +from parkings.api.auth import urls as auth_urls from parkings.api.enforcement import urls as enforcement_urls +from parkings.api.monitoring import urls as monitoring_urls from parkings.api.operator import urls as operator_urls from parkings.api.public import urls as public_urls -urlpatterns = [] +urlpatterns = [ + url(r'^auth/v1/', include(auth_urls, namespace='auth')), +] if getattr(settings, 'PARKKIHUBI_PUBLIC_API_ENABLED', False): urlpatterns.append(url(r'^public/v1/', include(public_urls, namespace='public'))) +if getattr(settings, 'PARKKIHUBI_MONITORING_API_ENABLED', False): + urlpatterns.append( + url(r'^monitoring/v1/', + include(monitoring_urls, namespace='monitoring'))) + if getattr(settings, 'PARKKIHUBI_OPERATOR_API_ENABLED', False): urlpatterns.append(url(r'^operator/v1/', include(operator_urls, namespace='operator'))) diff --git a/requirements.in b/requirements.in index 7a35fe74..12534c41 100644 --- a/requirements.in +++ b/requirements.in @@ -14,6 +14,7 @@ psycopg2 djangorestframework djangorestframework-gis django-filter +drf-jwt-2fa # XML parsing lxml @@ -23,3 +24,4 @@ owslib # Misc pytz +django-cors-headers diff --git a/requirements.txt b/requirements.txt index 28f1ba8c..2c4e0b4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,14 +5,18 @@ certifi==2018.1.18 # via requests chardet==3.0.4 # via requests django==1.11.9 +django-cors-headers==2.1.0 django-environ==0.4.4 django-filter==1.1.0 djangorestframework==3.7.7 djangorestframework-gis==0.12 +djangorestframework-jwt==1.11.0 # via drf-jwt-2fa +drf-jwt-2fa==0.3.0 idna==2.6 # via requests lxml==4.1.1 owslib==0.16.0 psycopg2==2.7.3.2 +pyjwt==1.5.3 # via djangorestframework-jwt pyproj==1.9.5.1 # via owslib python-dateutil==2.6.1 # via owslib pytz==2017.3