diff --git a/pyproject.toml b/pyproject.toml index e9f8813d..273810c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,3 +19,7 @@ target-version = "py310" [tool.ruff.format] quote-style = "single" + +[tool.ruff.lint.isort] +force-sort-within-sections = true # Sort by name, don't cluster "from" vs "import" +combine-as-imports = true # Combines "as" imports on the same line diff --git a/uvdat/core/admin.py b/uvdat/core/admin.py index dec0fda8..22506892 100644 --- a/uvdat/core/admin.py +++ b/uvdat/core/admin.py @@ -11,8 +11,8 @@ RasterMapLayer, SimulationResult, SourceRegion, + VectorFeature, VectorMapLayer, - VectorTile, ) @@ -53,14 +53,8 @@ def get_dataset_name(self, obj): return obj.file_item.dataset.name -class VectorTileAdmin(admin.ModelAdmin): - list_display = ['id', 'get_fileitem_name', 'get_map_layer_index', 'x', 'y', 'z'] - - def get_fileitem_name(self, obj): - return obj.map_layer.file_item.name - - def get_map_layer_index(self, obj): - return obj.map_layer.index +class VectorFeatureAdmin(admin.ModelAdmin): + list_display = ['id', 'map_layer'] class SourceRegionAdmin(admin.ModelAdmin): @@ -107,7 +101,7 @@ class SimulationResultAdmin(admin.ModelAdmin): admin.site.register(Chart, ChartAdmin) admin.site.register(RasterMapLayer, RasterMapLayerAdmin) admin.site.register(VectorMapLayer, VectorMapLayerAdmin) -admin.site.register(VectorTile, VectorTileAdmin) +admin.site.register(VectorFeature, VectorFeatureAdmin) admin.site.register(SourceRegion, SourceRegionAdmin) admin.site.register(DerivedRegion, DerivedRegionAdmin) admin.site.register(NetworkNode, NetworkNodeAdmin) diff --git a/uvdat/core/migrations/0002_vector_features.py b/uvdat/core/migrations/0002_vector_features.py new file mode 100644 index 00000000..d60ae19e --- /dev/null +++ b/uvdat/core/migrations/0002_vector_features.py @@ -0,0 +1,36 @@ +# Generated by Django 4.1 on 2024-06-24 18:24 + +import django.contrib.gis.db.models.fields +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('core', '0001_models_redesign'), + ] + + operations = [ + migrations.CreateModel( + name='VectorFeature', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID' + ), + ), + ('geometry', django.contrib.gis.db.models.fields.GeometryField(srid=4326)), + ('properties', models.JSONField()), + ( + 'map_layer', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to='core.vectormaplayer' + ), + ), + ], + ), + migrations.DeleteModel( + name='VectorTile', + ), + ] diff --git a/uvdat/core/models/__init__.py b/uvdat/core/models/__init__.py index 908b9769..1e5a3232 100644 --- a/uvdat/core/models/__init__.py +++ b/uvdat/core/models/__init__.py @@ -2,7 +2,7 @@ from .context import Context from .dataset import Dataset from .file_item import FileItem -from .map_layers import RasterMapLayer, VectorMapLayer, VectorTile +from .map_layers import RasterMapLayer, VectorFeature, VectorMapLayer from .networks import NetworkEdge, NetworkNode from .regions import DerivedRegion, SourceRegion from .simulations import SimulationResult @@ -14,7 +14,7 @@ FileItem, RasterMapLayer, VectorMapLayer, - VectorTile, + VectorFeature, SourceRegion, DerivedRegion, NetworkEdge, diff --git a/uvdat/core/models/dataset.py b/uvdat/core/models/dataset.py index 8374ceaf..81e0f8bd 100644 --- a/uvdat/core/models/dataset.py +++ b/uvdat/core/models/dataset.py @@ -89,43 +89,3 @@ def get_map_layers(self): return VectorMapLayer.objects.filter(file_item__dataset=self) raise NotImplementedError(f'Dataset Type {self.dataset_type}') - - def get_map_layer_tile_extents(self): - """ - Return the extents of all vector map layers of this dataset. - - Returns `None` if the dataset is not a vector dataset. - """ - if self.dataset_type != self.DatasetType.VECTOR: - return None - - from uvdat.core.models import VectorMapLayer, VectorTile - - # Retrieve all layers - layer_ids = VectorMapLayer.objects.filter(file_item__dataset=self).values_list( - 'id', flat=True - ) - - # Return x/y extents by layer id and z depth - vals = ( - VectorTile.objects.filter(map_layer_id__in=layer_ids) - .values('map_layer_id', 'z') - .annotate( - min_x=models.Min('x'), - min_y=models.Min('y'), - max_x=models.Max('x'), - max_y=models.Max('y'), - ) - .order_by('map_layer_id') - ) - - # Deconstruct query into response format - layers = {} - for entry in vals: - map_layer_id = entry.pop('map_layer_id') - if map_layer_id not in layers: - layers[map_layer_id] = {} - - layers[map_layer_id][entry.pop('z')] = entry - - return layers diff --git a/uvdat/core/models/map_layers.py b/uvdat/core/models/map_layers.py index 1d2d0356..f90ba7a2 100644 --- a/uvdat/core/models/map_layers.py +++ b/uvdat/core/models/map_layers.py @@ -2,6 +2,7 @@ from pathlib import Path import tempfile +from django.contrib.gis.db import models as geomodels from django.core.files.base import ContentFile from django.db import models from django_extensions.db.models import TimeStampedModel @@ -58,41 +59,8 @@ def read_geojson_data(self) -> dict: """Read and load the data from geojson_file into a dict.""" return json.load(self.geojson_file.open()) - def get_tile_extents(self): - """Return a dict that maps z tile values to the x/y extent at that depth.""" - return { - entry.pop('z'): entry - for entry in ( - VectorTile.objects.filter(map_layer=self) - .values('z') - .annotate( - min_x=models.Min('x'), - min_y=models.Min('y'), - max_x=models.Max('x'), - max_y=models.Max('y'), - ) - .order_by() - ) - } - - -class VectorTile(models.Model): - EMPTY_TILE_DATA = { - 'type': 'FeatureCollection', - 'features': [], - } +class VectorFeature(models.Model): map_layer = models.ForeignKey(VectorMapLayer, on_delete=models.CASCADE) - geojson_data = models.JSONField(blank=True, null=True) - x = models.IntegerField(default=0) - y = models.IntegerField(default=0) - z = models.IntegerField(default=0) - - class Meta: - constraints = [ - # Ensure that a full index only ever resolves to one record - models.UniqueConstraint( - name='unique-map-layer-index', fields=['map_layer', 'z', 'x', 'y'] - ) - ] - indexes = [models.Index(fields=('z', 'x', 'y'), name='vectortile-coordinates-index')] + geometry = geomodels.GeometryField() + properties = models.JSONField() diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index 05eb7a3b..a354f775 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -29,13 +29,8 @@ def map_layers(self, request, **kwargs): if dataset.dataset_type == Dataset.DatasetType.RASTER: serializer = uvdat_serializers.RasterMapLayerSerializer(map_layers, many=True) elif dataset.dataset_type == Dataset.DatasetType.VECTOR: - # Inject tile extents - extents = dataset.get_map_layer_tile_extents() - for layer in map_layers: - layer.tile_extents = extents.pop(layer.id) - # Set serializer - serializer = uvdat_serializers.ExtendedVectorMapLayerSerializer(map_layers, many=True) + serializer = uvdat_serializers.VectorMapLayerSerializer(map_layers, many=True) else: raise NotImplementedError(f'Dataset Type {dataset.dataset_type}') diff --git a/uvdat/core/rest/map_layers.py b/uvdat/core/rest/map_layers.py index 9eb27961..a9ed6289 100644 --- a/uvdat/core/rest/map_layers.py +++ b/uvdat/core/rest/map_layers.py @@ -1,5 +1,6 @@ import json +from django.db import connection from django.http import HttpResponse from django_large_image.rest import LargeImageFileDetailMixin from rest_framework.decorators import action @@ -7,13 +8,70 @@ from rest_framework.viewsets import ModelViewSet from uvdat.core.models import RasterMapLayer, VectorMapLayer -from uvdat.core.models.map_layers import VectorTile from uvdat.core.rest.serializers import ( RasterMapLayerSerializer, VectorMapLayerDetailSerializer, VectorMapLayerSerializer, ) +VECTOR_TILE_SQL = """ +WITH +tilenv as ( + SELECT ST_TRANSFORM(ST_TileEnvelope(%(z)s, %(x)s, %(y)s), %(srid)s) as te +), +tilenvbounds as ( + SELECT + ST_XMin(te) as xmin, + ST_YMin(te) as ymin, + ST_XMax(te) as xmax, + ST_YMax(te) as ymax, + (ST_XMax(te) - ST_XMin(te)) / 4 as segsize + FROM tilenv +), +env as ( + SELECT ST_Segmentize( + ST_MakeEnvelope( + xmin, + ymin, + xmax, + ymax, + %(srid)s + ), + segsize + ) as seg + FROM tilenvbounds +), +bounds as ( + SELECT + seg as geom, + seg::box2d as b2d + FROM env +), +mvtgeom as ( + SELECT + ST_AsMVTGeom( + ST_Transform(t.geometry, %(srid)s), + bounds.b2d + ) AS geom, + t.properties as properties + FROM + core_vectorfeature t, + bounds + WHERE + t.map_layer_id = %(map_layer_id)s + AND ST_Intersects( + ST_Transform(t.geometry, %(srid)s), + ST_Transform(bounds.geom, %(srid)s) + ) + AND ( + ST_GeometryType(ST_AsText(t.geometry)) != 'ST_Point' + OR %(z)s >= 16 + ) +) +SELECT ST_AsMVT(mvtgeom.*) FROM mvtgeom +; +""" + class RasterMapLayerViewSet(ModelViewSet, LargeImageFileDetailMixin): queryset = RasterMapLayer.objects.select_related('file_item__dataset').all() @@ -48,10 +106,22 @@ def retrieve(self, request, *args, **kwargs): url_name='tiles', ) def get_vector_tile(self, request, x: str, y: str, z: str, pk: str): - # Return vector tile or empty tile - try: - tile = VectorTile.objects.get(map_layer_id=pk, x=x, y=y, z=z) - except VectorTile.DoesNotExist: - return Response(VectorTile.EMPTY_TILE_DATA, status=200) + with connection.cursor() as cursor: + cursor.execute( + VECTOR_TILE_SQL, + { + 'z': z, + 'x': x, + 'y': y, + 'srid': 3857, + 'map_layer_id': pk, + }, + ) + row = cursor.fetchone() - return Response(tile.geojson_data, status=200) + tile = row[0] + return HttpResponse( + tile, + content_type='application/octet-stream', + status=200 if tile else 204, + ) diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index b9413ba4..d015473c 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -85,14 +85,6 @@ class Meta: fields = '__all__' -class ExtendedVectorMapLayerSerializer(serializers.ModelSerializer, AbstractMapLayerSerializer): - tile_extents = serializers.JSONField() - - class Meta: - model = VectorMapLayer - exclude = ['geojson_file'] - - class VectorMapLayerSerializer(serializers.ModelSerializer, AbstractMapLayerSerializer): class Meta: model = VectorMapLayer @@ -101,7 +93,6 @@ class Meta: class VectorMapLayerDetailSerializer(serializers.ModelSerializer, AbstractMapLayerSerializer): derived_region_id = serializers.SerializerMethodField('get_derived_region_id') - tile_extents = serializers.SerializerMethodField('get_tile_extents') def get_derived_region_id(self, obj): dr = obj.derivedregion_set.first() @@ -109,9 +100,6 @@ def get_derived_region_id(self, obj): return None return dr.id - def get_tile_extents(self, obj: VectorMapLayer): - return obj.get_tile_extents() - class Meta: model = VectorMapLayer exclude = ['geojson_file'] diff --git a/uvdat/core/tasks/dataset.py b/uvdat/core/tasks/dataset.py index 2bcfb125..65d932c0 100644 --- a/uvdat/core/tasks/dataset.py +++ b/uvdat/core/tasks/dataset.py @@ -9,8 +9,9 @@ SourceRegion, VectorMapLayer, ) +from uvdat.core.tasks.map_layers import save_vector_features -from .map_layers import create_raster_map_layer, create_vector_map_layer, save_vector_tiles +from .map_layers import create_raster_map_layer, create_vector_map_layer from .networks import create_network from .regions import create_source_regions @@ -58,7 +59,7 @@ def convert_dataset( # Create vector tiles after geojson_data may have # been altered by create_network or create_source_regions - save_vector_tiles(vector_map_layer) + save_vector_features(vector_map_layer=vector_map_layer) dataset.processing = False dataset.save() diff --git a/uvdat/core/tasks/map_layers.py b/uvdat/core/tasks/map_layers.py index 12cd8401..e70c7493 100644 --- a/uvdat/core/tasks/map_layers.py +++ b/uvdat/core/tasks/map_layers.py @@ -3,15 +3,16 @@ import tempfile import zipfile +from django.contrib.gis.geos import GEOSGeometry from django.core.files.base import ContentFile -from geojson2vt import geojson2vt, vt2geojson import geopandas import numpy import rasterio import shapefile from webcolors import name_to_hex -from uvdat.core.models import RasterMapLayer, VectorMapLayer, VectorTile +from uvdat.core.models import RasterMapLayer, VectorMapLayer +from uvdat.core.models.map_layers import VectorFeature def add_styling(geojson_data, style_options): @@ -144,8 +145,6 @@ def create_vector_map_layer(file_item, style_options): new_map_layer.write_geojson_data(geojson_data.to_json()) new_map_layer.save() - # save_vector_tiles(vector_map_layer=new_map_layer) - return new_map_layer @@ -170,24 +169,19 @@ def convert_zip_to_geojson(file_item): return geodata -def save_vector_tiles(vector_map_layer): - tile_index = geojson2vt.geojson2vt( - vector_map_layer.read_geojson_data(), - {'indexMaxZoom': 12, 'maxZoom': 12, 'indexMaxPoints': 0}, - ) - - created = 0 - for coord in tile_index.tile_coords: - tile = tile_index.get_tile(coord['z'], coord['x'], coord['y']) - features = tile.get('features') - if features and len(features) > 0: - VectorTile.objects.create( +def save_vector_features(vector_map_layer: VectorMapLayer): + features = vector_map_layer.read_geojson_data()['features'] + vector_features = [] + for feature in features: + vector_features.append( + VectorFeature( map_layer=vector_map_layer, - geojson_data=vt2geojson.vt2geojson(tile), - x=coord['x'], - y=coord['y'], - z=coord['z'], + geometry=GEOSGeometry(json.dumps(feature['geometry'])), + properties=feature['properties'], ) - created += 1 + ) + + created = VectorFeature.objects.bulk_create(vector_features) + print('\t', f'{len(created)} vector features created.') - print('\t', f'{created} vector tiles created.') + return created diff --git a/uvdat/core/tasks/regions.py b/uvdat/core/tasks/regions.py index 49a80571..2af73b3b 100644 --- a/uvdat/core/tasks/regions.py +++ b/uvdat/core/tasks/regions.py @@ -8,7 +8,7 @@ import geopandas from uvdat.core.models import Context, DerivedRegion, SourceRegion, VectorMapLayer -from uvdat.core.tasks.map_layers import save_vector_tiles +from uvdat.core.tasks.map_layers import save_vector_features class DerivedRegionCreationError(Exception): @@ -61,7 +61,7 @@ def create_derived_region(name: str, context: Context, region_ids: List[int], op ) new_map_layer.write_geojson_data(geojson) new_map_layer.save() - save_vector_tiles(new_map_layer) + save_vector_features(new_map_layer) derived_region = DerivedRegion.objects.create( name=name, diff --git a/uvdat/core/tests/test_populate.py b/uvdat/core/tests/test_populate.py index 601b0e07..3b3b8add 100644 --- a/uvdat/core/tests/test_populate.py +++ b/uvdat/core/tests/test_populate.py @@ -12,8 +12,8 @@ RasterMapLayer, SimulationResult, SourceRegion, + VectorFeature, VectorMapLayer, - VectorTile, ) @@ -43,4 +43,4 @@ def test_populate(): assert SimulationResult.objects.all().count() == 0 assert SourceRegion.objects.all().count() == 24 assert VectorMapLayer.objects.all().count() == 5 - assert VectorTile.objects.all().count() == 135 + assert VectorFeature.objects.count() == 351 diff --git a/web/src/layers.ts b/web/src/layers.ts index c2463c04..2914a9d0 100644 --- a/web/src/layers.ts +++ b/web/src/layers.ts @@ -1,9 +1,8 @@ -import GeoJSON from "ol/format/GeoJSON.js"; +import MVT from "ol/format/MVT"; import TileLayer from "ol/layer/Tile"; import XYZSource from "ol/source/XYZ.js"; import VectorTileLayer from "ol/layer/VectorTile"; import VectorTileSource from "ol/source/VectorTile"; -import { TileCoord } from "ol/tilecoord"; import { Circle, Stroke, Style } from "ol/style"; import { Feature } from "ol"; @@ -102,25 +101,8 @@ export function createVectorOpenLayer(mapLayer: VectorMapLayer) { colors: feature.getProperties().colors || defaultColors, }), source: new VectorTileSource({ - maxZoom: Math.max( - ...Object.keys(mapLayer.tile_extents).map((z) => Number(z)) - ), - format: new GeoJSON(), - tileUrlFunction: (tileCoord: TileCoord) => { - const [z, x, y] = tileCoord; - const entry = mapLayer.tile_extents[z]; - if (!entry) { - return undefined; - } - - // Ensure current x/y is within zoom level extent - const { min_x, min_y, max_x, max_y } = entry; - if (!(min_x <= x && x <= max_x && min_y <= y && y <= max_y)) { - return undefined; - } - - return `${baseURL}vectors/${mapLayer.id}/tiles/${z}/${x}/${y}/`; - }, + format: new MVT(), + url: `${baseURL}vectors/${mapLayer.id}/tiles/{z}/{x}/{y}/`, }), }); } diff --git a/web/src/types.ts b/web/src/types.ts index 5fed23d7..ffce2160 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -93,7 +93,6 @@ export interface AbstractMapLayer { }; default_style?: object; index: number; - type: "vector" | "raster"; dataset_id?: number; derived_region_id?: number; @@ -107,10 +106,11 @@ export function isNonNullObject(obj: unknown): obj is object { export interface RasterMapLayer extends AbstractMapLayer { cloud_optimized_geotiff: string; + type: "raster"; } export function isRasterMapLayer(obj: unknown): obj is RasterMapLayer { - return isNonNullObject(obj) && "cloud_optimized_geotiff" in obj; + return isNonNullObject(obj) && "type" in obj && obj.type === "raster"; } export interface RasterData { @@ -124,18 +124,11 @@ export interface RasterData { } export interface VectorMapLayer extends AbstractMapLayer { - tile_extents: { - [z: number]: { - min_x: number; - min_y: number; - max_x: number; - max_y: number; - }; - }; + type: "vector"; } export function isVectorMapLayer(obj: unknown): obj is VectorMapLayer { - return isNonNullObject(obj) && "tile_extents" in obj; + return isNonNullObject(obj) && "type" in obj && obj.type === "vector"; } export interface VectorTile {