Add school and preschool district import

Add management commands for importing school and preschool districts
for Helsinki.

The commands replace the old school and preschool district imports done
 by django-munigeo. The old imports needed to be manually
 updated because of the yearly date and layer changes in the source
 data.

The new import still uses the same source and the field are mapped to
the same fields as before, but now the import can be done automatically
without manual updates to the code or the data.

services/management/commands/school_district_import/school_district_importer.py

@@ -0,0 +1,170 @@
+import logging
+import re
+from datetime import datetime
+
+from django.contrib.gis.gdal import CoordTransform, DataSource, SpatialReference
+from django.contrib.gis.geos import MultiPolygon
+from munigeo import ocd
+from munigeo.models import (
+    AdministrativeDivision,
+    AdministrativeDivisionGeometry,
+    AdministrativeDivisionType,
+    Municipality,
+)
+
+from services.management.commands.lipas_import import MiniWFS
+
+logger = logging.getLogger(__name__)
+SRID = 3067
+
+
+class SchoolDistrictImporter:
+    WFS_BASE = "https://kartta.hel.fi/ws/geoserver/avoindata/wfs"
+
+    def __init__(self, district_type):
+        self.district_type = district_type
+
+    def import_districts(self, data):
+        """
+        Data is a dictionary containing the following keys:
+        - source_type: The name of the WFS layer to fetch
+        - division_type:  AdministrativeDivisionType type of the division
+        - ocd_id: The key to use in the OCD ID generation
+
+        """
+        wfs = MiniWFS(self.WFS_BASE)
+        municipality = Municipality.objects.get(id="helsinki")
+
+        source_type = data["source_type"]
+        division_type = data["division_type"]
+        ocd_id = data["ocd_id"]
+
+        try:
+            url = wfs.get_feature(type_name=source_type)
+            layer = DataSource(url)[0]
+        except Exception as e:
+            logger.error(f"Error retrieving data for {source_type}: {e}")
+            return
+
+        logger.info(f"Retrieved {len(layer)} {source_type} features.")
+        logger.info("Processing data...")
+
+        division_type_obj, _ = AdministrativeDivisionType.objects.get_or_create(
+            type=division_type
+        )
+
+        for feature in layer:
+            self.import_division(
+                feature,
+                division_type_obj,
+                municipality,
+                ocd_id,
+                source_type,
+            )
+
+    def import_division(
+        self, feature, division_type_obj, municipality, ocd_id, source_type
+    ):
+        origin_id = feature.get("id")
+        if not origin_id:
+            logger.info("Skipping feature without id.")
+            return
+
+        division, _ = AdministrativeDivision.objects.get_or_create(
+            origin_id=origin_id, type=division_type_obj
+        )
+
+        division.municipality = municipality
+        division.parent = municipality.division
+
+        division.ocd_id = ocd.make_id(
+            **{ocd_id: str(origin_id), "parent": municipality.division.ocd_id}
+        )
+
+        service_point_id = str(feature.get("toimipiste_id"))
+
+        if self.district_type == "school":
+            division.service_point_id = service_point_id
+            division.units = [service_point_id]
+
+            if "suomi" in source_type:
+                name = feature.get("nimi_fi")
+                division.name_fi = feature.get("nimi_fi")
+                division.start = self.create_start_date(name)
+                division.end = self.create_end_date(name)
+            if "ruotsi" in source_type:
+                name = feature.get("nimi_se")
+                division.name_sv = feature.get("nimi_se")
+                division.start = self.create_start_date(name)
+                division.end = self.create_end_date(name)
+
+        elif self.district_type == "preschool":
+            units = service_point_id.split(",")
+            division.units = units
+
+            division.name_fi = feature.get("nimi_fi")
+            division.name_sv = feature.get("nimi_se")
+
+            division.extra = {"schoolyear": feature.get("lukuvuosi")}
+
+        division.save()
+
+        self.save_geometry(feature, division)
+
+    def create_start_date(self, name):
+        year = re.split(r"[ -]", name)[-2]
+        return f"{year}-08-01"
+
+    def create_end_date(self, name):
+        year = re.split(r"[ -]", name)[-1]
+        return f"{year }-07-31"
+
+    def save_geometry(self, feature, division):
+        geom = feature.geom
+        if not geom.srid:
+            geom.srid = SRID
+        if geom.srid != SRID:
+            geom.transform(SRID)
+            ct = CoordTransform(SpatialReference(geom.srid), SpatialReference(SRID))
+            geom.transform(ct)
+
+        geom = geom.geos
+        if geom.geom_type == "Polygon":
+            geom = MultiPolygon(geom.buffer(0), srid=geom.srid)
+
+        try:
+            geom_obj = division.geometry
+        except AdministrativeDivisionGeometry.DoesNotExist:
+            geom_obj = AdministrativeDivisionGeometry(division=division)
+
+        geom_obj.boundary = geom
+        geom_obj.save()
+
+    def remove_old_school_year(self, division_type):
+        """
+        During 1.8.-15.12. only the current school year is shown.
+        During 16.12.-31.7. both the current and the next school year are shown.
+
+        The source might be named as "tuleva" but it might still actually be the current school year.
+
+        If today is between 1.8.-15.12 delete the previous year.
+        """
+        division_type_obj = AdministrativeDivisionType.objects.get(type=division_type)
+
+        today = datetime.today()
+
+        last_year = today.year - 1
+        last_year_start_date = f"{last_year}-08-01"
+
+        if datetime(today.year, 8, 1) <= today <= datetime(today.year, 12, 15):
+            if self.district_type == "school":
+                AdministrativeDivision.objects.filter(
+                    type=division_type_obj,
+                    start=last_year_start_date,
+                ).delete()
+
+            if self.district_type == "preschool":
+                AdministrativeDivision.objects.filter(
+                    type=division_type_obj,
+                    extra__schoolyear=f"{last_year}-{last_year + 1}",
+                ).delete()

services/management/commands/update_helsinki_preschool_districts.py

@@ -0,0 +1,54 @@
+from django.core.management.base import BaseCommand
+from munigeo.models import AdministrativeDivision
+
+from services.management.commands.school_district_import.school_district_importer import (
+    SchoolDistrictImporter,
+)
+
+PRESCHOOL_DISTRICT_DATA = [
+    {
+        "source_type": "avoindata:Esiopetusalue_suomi",
+        "division_type": "preschool_education_fi",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi",
+    },
+    {
+        "source_type": "avoindata:Esiopetusalue_suomi_tuleva",
+        "division_type": "preschool_education_fi",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi",
+    },
+    {
+        "source_type": "avoindata:Esiopetusalue_ruotsi",
+        "division_type": "preschool_education_sv",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_sv",
+    },
+    {
+        "source_type": "avoindata:Esiopetusalue_ruotsi_tuleva",
+        "division_type": "preschool_education_sv",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_sv",
+    },
+]
+
+
+class Command(BaseCommand):
+    help = (
+        "Update Helsinki preschool districts. "
+        "Usage: ./manage.py update_helsinki_preschool_districts"
+    )
+
+    def handle(self, *args, **options):
+        division_types = list(
+            {data["division_type"] for data in PRESCHOOL_DISTRICT_DATA}
+        )
+
+        # Remove old divisions before importing new ones to avoid possible duplicates as the source layers may change
+        AdministrativeDivision.objects.filter(
+            type__type__in=division_types, municipality__id="helsinki"
+        ).delete()
+
+        importer = SchoolDistrictImporter(district_type="preschool")
+
+        for data in PRESCHOOL_DISTRICT_DATA:
+            importer.import_districts(data)
+
+        for division_type in division_types:
+            importer.remove_old_school_year(division_type)

services/management/commands/update_helsinki_school_districts.py

@@ -0,0 +1,72 @@
+from django.core.management.base import BaseCommand
+from munigeo.models import AdministrativeDivision
+
+from services.management.commands.school_district_import.school_district_importer import (
+    SchoolDistrictImporter,
+)
+
+SCHOOL_DISTRICT_DATA = [
+    {
+        "source_type": "avoindata:Opev_ooa_alaaste_suomi",
+        "division_type": "lower_comprehensive_school_district_fi",
+        "ocd_id": "oppilaaksiottoalue_alakoulu",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_alaaste_suomi_tuleva",
+        "division_type": "lower_comprehensive_school_district_fi",
+        "ocd_id": "oppilaaksiottoalue_alakoulu",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_alaaste_ruotsi",
+        "division_type": "lower_comprehensive_school_district_sv",
+        "ocd_id": "oppilaaksiottoalue_alakoulu_sv",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_alaaste_ruotsi_tuleva",
+        "division_type": "lower_comprehensive_school_district_sv",
+        "ocd_id": "oppilaaksiottoalue_alakoulu_sv",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_ylaaste_suomi",
+        "division_type": "upper_comprehensive_school_district_fi",
+        "ocd_id": "oppilaaksiottoalue_ylakoulu",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_ylaaste_suomi_tuleva",
+        "division_type": "upper_comprehensive_school_district_fi",
+        "ocd_id": "oppilaaksiottoalue_ylakoulu",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_ylaaste_ruotsi",
+        "division_type": "upper_comprehensive_school_district_sv",
+        "ocd_id": "oppilaaksiottoalue_ylakoulu_sv",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_ylaaste_ruotsi_tuleva",
+        "division_type": "upper_comprehensive_school_district_sv",
+        "ocd_id": "oppilaaksiottoalue_ylakoulu_sv",
+    },
+]
+
+
+class Command(BaseCommand):
+    help = (
+        "Update Helsinki school districts. "
+        "Usage: ./manage.py update_helsinki_school_districts"
+    )
+
+    def handle(self, *args, **options):
+        division_types = list({data["division_type"] for data in SCHOOL_DISTRICT_DATA})
+
+        # Remove old divisions before importing new ones to avoid possible duplicates as the source layers may change
+        AdministrativeDivision.objects.filter(
+            type__type__in=division_types, municipality__id="helsinki"
+        ).delete()
+
+        importer = SchoolDistrictImporter(district_type="school")
+
+        for data in SCHOOL_DISTRICT_DATA:
+            importer.import_districts(data)
+
+        for division_type in division_types:
+            importer.remove_old_school_year(division_type)

services/tests/data/Esiopetusalue_suomi.gfs

@@ -0,0 +1,75 @@
+<GMLFeatureClassList>
+  <GMLFeatureClass>
+    <Name>Esiopetusalue_suomi</Name>
+    <ElementPath>Esiopetusalue_suomi</ElementPath>
+    <GeometryName>geom</GeometryName>
+    <GeometryElementPath>geom</GeometryElementPath>
+    <!--POLYGON-->
+    <GeometryType>3</GeometryType>
+    <SRSName>EPSG:3879</SRSName>
+    <DatasetSpecificInfo>
+      <FeatureCount>1</FeatureCount>
+      <ExtentXMin>25503070.45293</ExtentXMin>
+      <ExtentXMax>25508934.14646</ExtentXMax>
+      <ExtentYMin>6670858.66227</ExtentYMin>
+      <ExtentYMax>6673564.24700</ExtentYMax>
+    </DatasetSpecificInfo>
+    <PropertyDefn>
+      <Name>id</Name>
+      <ElementPath>id</ElementPath>
+      <Type>Integer</Type>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>aluejako</Name>
+      <ElementPath>aluejako</ElementPath>
+      <Type>String</Type>
+      <Width>14</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>kunta</Name>
+      <ElementPath>kunta</ElementPath>
+      <Type>Integer</Type>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>nimi_fi</Name>
+      <ElementPath>nimi_fi</ElementPath>
+      <Type>String</Type>
+      <Width>9</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>nimi_se</Name>
+      <ElementPath>nimi_se</ElementPath>
+      <Type>String</Type>
+      <Width>11</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>toimipiste_id</Name>
+      <ElementPath>toimipiste_id</ElementPath>
+      <Type>Integer</Type>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>yhtluontipvm</Name>
+      <ElementPath>yhtluontipvm</ElementPath>
+      <Type>String</Type>
+      <Width>10</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>yhtdatanomistaja</Name>
+      <ElementPath>yhtdatanomistaja</ElementPath>
+      <Type>String</Type>
+      <Width>4</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>paivitetty_tietopalveluun</Name>
+      <ElementPath>paivitetty_tietopalveluun</ElementPath>
+      <Type>String</Type>
+      <Width>10</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>lukuvuosi</Name>
+      <ElementPath>lukuvuosi</ElementPath>
+      <Type>String</Type>
+      <Width>9</Width>
+    </PropertyDefn>
+  </GMLFeatureClass>
+</GMLFeatureClassList>