Add school and preschool district import

services/management/commands/school_district_import/school_district_importer.py

@@ -0,0 +1,170 @@
+import logging
+import re
+from datetime import datetime
+
+from django.contrib.gis.gdal import CoordTransform, DataSource, SpatialReference
+from django.contrib.gis.geos import MultiPolygon
+from munigeo import ocd
+from munigeo.models import (
+    AdministrativeDivision,
+    AdministrativeDivisionGeometry,
+    AdministrativeDivisionType,
+    Municipality,
+)
+
+from services.management.commands.lipas_import import MiniWFS
+
+logger = logging.getLogger(__name__)
+SRID = 3067
+
+
+class SchoolDistrictImporter:
+    WFS_BASE = "https://kartta.hel.fi/ws/geoserver/avoindata/wfs"
+
+    def __init__(self, district_type):
+        self.district_type = district_type
+
+    def import_districts(self, data):
+        """
+        Data is a dictionary containing the following keys:
+        - source_type: The name of the WFS layer to fetch
+        - division_type:  AdministrativeDivisionType type of the division
+        - ocd_id: The key to use in the OCD ID generation
+
+        """
+        wfs = MiniWFS(self.WFS_BASE)
+        municipality = Municipality.objects.get(id="helsinki")
+
+        source_type = data["source_type"]
+        division_type = data["division_type"]
+        ocd_id = data["ocd_id"]
+
+        try:
+            url = wfs.get_feature(type_name=source_type)
+            layer = DataSource(url)[0]
+        except Exception as e:
+            logger.error(f"Error retrieving data for {source_type}: {e}")
+            return
+
+        logger.info(f"Retrieved {len(layer)} {source_type} features.")
+        logger.info("Processing data...")
+
+        division_type_obj, _ = AdministrativeDivisionType.objects.get_or_create(
+            type=division_type
+        )
+
+        for feature in layer:
+            self.import_division(
+                feature,
+                division_type_obj,
+                municipality,
+                ocd_id,
+                source_type,
+            )
+
+    def import_division(
+        self, feature, division_type_obj, municipality, ocd_id, source_type
+    ):
+        origin_id = feature.get("id")
+        if not origin_id:
+            logger.info("Skipping feature without id.")
+            return
+
+        division, _ = AdministrativeDivision.objects.get_or_create(
+            origin_id=origin_id, type=division_type_obj
+        )
+
+        division.municipality = municipality
+        division.parent = municipality.division
+
+        division.ocd_id = ocd.make_id(
+            **{ocd_id: str(origin_id), "parent": municipality.division.ocd_id}
+        )
+
+        service_point_id = str(feature.get("toimipiste_id"))
+
+        if self.district_type == "school":
+            division.service_point_id = service_point_id
+            division.units = [service_point_id]
+
+            if "suomi" in source_type:
+                name = feature.get("nimi_fi")
+                division.name_fi = feature.get("nimi_fi")
+                division.start = self.create_start_date(name)
+                division.end = self.create_end_date(name)
+            if "ruotsi" in source_type:
+                name = feature.get("nimi_se")
+                division.name_sv = feature.get("nimi_se")
+                division.start = self.create_start_date(name)
+                division.end = self.create_end_date(name)
+
+        elif self.district_type == "preschool":
+            units = service_point_id.split(",")
+            division.units = units
+
+            division.name_fi = feature.get("nimi_fi")
+            division.name_sv = feature.get("nimi_se")
+
+            division.extra = {"schoolyear": feature.get("lukuvuosi")}
+
+        division.save()
+
+        self.save_geometry(feature, division)
+
+    def create_start_date(self, name):
+        year = re.split(r"[ -]", name)[-2]
+        return f"{year}-08-01"
+
+    def create_end_date(self, name):
+        year = re.split(r"[ -]", name)[-1]
+        return f"{year }-07-31"
+
+    def save_geometry(self, feature, division):
+        geom = feature.geom
+        if not geom.srid:
+            geom.srid = SRID
+        if geom.srid != SRID:
+            geom.transform(SRID)
+            ct = CoordTransform(SpatialReference(geom.srid), SpatialReference(SRID))
+            geom.transform(ct)
+
+        geom = geom.geos
+        if geom.geom_type == "Polygon":
+            geom = MultiPolygon(geom.buffer(0), srid=geom.srid)
+
+        try:
+            geom_obj = division.geometry
+        except AdministrativeDivisionGeometry.DoesNotExist:
+            geom_obj = AdministrativeDivisionGeometry(division=division)
+
+        geom_obj.boundary = geom
+        geom_obj.save()
+
+    def remove_old_school_year(self, division_type):
+        """
+        During 1.8.-15.12. only the current school year is shown.
+        During 16.12.-31.7. both the current and the next school year are shown.
+
+        The source might be named as "tuleva" but it might still actually be the current school year.
+
+        If today is between 1.8.-15.12 delete the previous year.
+        """
+        division_type_obj = AdministrativeDivisionType.objects.get(type=division_type)
+
+        today = datetime.today()
+
+        last_year = today.year - 1
+        last_year_start_date = f"{last_year}-08-01"
+
+        if datetime(today.year, 8, 1) <= today <= datetime(today.year, 12, 15):
+            if self.district_type == "school":
+                AdministrativeDivision.objects.filter(
+                    type=division_type_obj,
+                    start=last_year_start_date,
+                ).delete()
+
+            if self.district_type == "preschool":
+                AdministrativeDivision.objects.filter(
+                    type=division_type_obj,
+                    extra__schoolyear=f"{last_year}-{last_year + 1}",
+                ).delete()

services/management/commands/update_helsinki_preschool_districts.py

@@ -0,0 +1,54 @@
+from django.core.management.base import BaseCommand
+from munigeo.models import AdministrativeDivision
+
+from services.management.commands.school_district_import.school_district_importer import (
+    SchoolDistrictImporter,
+)
+
+PRESCHOOL_DISTRICT_DATA = [
+    {
+        "source_type": "avoindata:Esiopetusalue_suomi",
+        "division_type": "preschool_education_fi",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi",
+    },
+    {
+        "source_type": "avoindata:Esiopetusalue_suomi_tuleva",
+        "division_type": "preschool_education_fi",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi",
+    },
+    {
+        "source_type": "avoindata:Esiopetusalue_ruotsi",
+        "division_type": "preschool_education_sv",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_sv",
+    },
+    {
+        "source_type": "avoindata:Esiopetusalue_ruotsi_tuleva",
+        "division_type": "preschool_education_sv",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_sv",
+    },
+]
+
+
+class Command(BaseCommand):
+    help = (
+        "Update Helsinki preschool districts. "
+        "Usage: ./manage.py update_helsinki_preschool_districts"
+    )
+
+    def handle(self, *args, **options):
+        division_types = list(
+            {data["division_type"] for data in PRESCHOOL_DISTRICT_DATA}
+        )
+
+        # Remove old divisions before importing new ones to avoid possible duplicates as the source layers may change
+        AdministrativeDivision.objects.filter(
+            type__type__in=division_types, municipality__id="helsinki"
+        ).delete()
+
+        importer = SchoolDistrictImporter(district_type="preschool")
+
+        for data in PRESCHOOL_DISTRICT_DATA:
+            importer.import_districts(data)
+
+        for division_type in division_types:
+            importer.remove_old_school_year(division_type)

services/management/commands/update_helsinki_school_districts.py

@@ -0,0 +1,72 @@
+from django.core.management.base import BaseCommand
+from munigeo.models import AdministrativeDivision
+
+from services.management.commands.school_district_import.school_district_importer import (
+    SchoolDistrictImporter,
+)
+
+SCHOOL_DISTRICT_DATA = [
+    {
+        "source_type": "avoindata:Opev_ooa_alaaste_suomi",
+        "division_type": "lower_comprehensive_school_district_fi",
+        "ocd_id": "oppilaaksiottoalue_alakoulu",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_alaaste_suomi_tuleva",
+        "division_type": "lower_comprehensive_school_district_fi",
+        "ocd_id": "oppilaaksiottoalue_alakoulu",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_alaaste_ruotsi",
+        "division_type": "lower_comprehensive_school_district_sv",
+        "ocd_id": "oppilaaksiottoalue_alakoulu_sv",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_alaaste_ruotsi_tuleva",
+        "division_type": "lower_comprehensive_school_district_sv",
+        "ocd_id": "oppilaaksiottoalue_alakoulu_sv",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_ylaaste_suomi",
+        "division_type": "upper_comprehensive_school_district_fi",
+        "ocd_id": "oppilaaksiottoalue_ylakoulu",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_ylaaste_suomi_tuleva",
+        "division_type": "upper_comprehensive_school_district_fi",
+        "ocd_id": "oppilaaksiottoalue_ylakoulu",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_ylaaste_ruotsi",
+        "division_type": "upper_comprehensive_school_district_sv",
+        "ocd_id": "oppilaaksiottoalue_ylakoulu_sv",
+    },
+    {
+        "source_type": "avoindata:Opev_ooa_ylaaste_ruotsi_tuleva",
+        "division_type": "upper_comprehensive_school_district_sv",
+        "ocd_id": "oppilaaksiottoalue_ylakoulu_sv",
+    },
+]
+
+
+class Command(BaseCommand):
+    help = (
+        "Update Helsinki school districts. "
+        "Usage: ./manage.py update_helsinki_school_districts"
+    )
+
+    def handle(self, *args, **options):
+        division_types = list({data["division_type"] for data in SCHOOL_DISTRICT_DATA})
+
+        # Remove old divisions before importing new ones to avoid possible duplicates as the source layers may change
+        AdministrativeDivision.objects.filter(
+            type__type__in=division_types, municipality__id="helsinki"
+        ).delete()
+
+        importer = SchoolDistrictImporter(district_type="school")
+
+        for data in SCHOOL_DISTRICT_DATA:
+            importer.import_districts(data)
+
+        for division_type in division_types:
+            importer.remove_old_school_year(division_type)

services/tests/data/Esiopetusalue_suomi.gfs

@@ -0,0 +1,75 @@
+<GMLFeatureClassList>
+  <GMLFeatureClass>
+    <Name>Esiopetusalue_suomi</Name>
+    <ElementPath>Esiopetusalue_suomi</ElementPath>
+    <GeometryName>geom</GeometryName>
+    <GeometryElementPath>geom</GeometryElementPath>
+    <!--POLYGON-->
+    <GeometryType>3</GeometryType>
+    <SRSName>EPSG:3879</SRSName>
+    <DatasetSpecificInfo>
+      <FeatureCount>1</FeatureCount>
+      <ExtentXMin>25503070.45293</ExtentXMin>
+      <ExtentXMax>25508934.14646</ExtentXMax>
+      <ExtentYMin>6670858.66227</ExtentYMin>
+      <ExtentYMax>6673564.24700</ExtentYMax>
+    </DatasetSpecificInfo>
+    <PropertyDefn>
+      <Name>id</Name>
+      <ElementPath>id</ElementPath>
+      <Type>Integer</Type>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>aluejako</Name>
+      <ElementPath>aluejako</ElementPath>
+      <Type>String</Type>
+      <Width>14</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>kunta</Name>
+      <ElementPath>kunta</ElementPath>
+      <Type>Integer</Type>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>nimi_fi</Name>
+      <ElementPath>nimi_fi</ElementPath>
+      <Type>String</Type>
+      <Width>9</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>nimi_se</Name>
+      <ElementPath>nimi_se</ElementPath>
+      <Type>String</Type>
+      <Width>11</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>toimipiste_id</Name>
+      <ElementPath>toimipiste_id</ElementPath>
+      <Type>Integer</Type>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>yhtluontipvm</Name>
+      <ElementPath>yhtluontipvm</ElementPath>
+      <Type>String</Type>
+      <Width>10</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>yhtdatanomistaja</Name>
+      <ElementPath>yhtdatanomistaja</ElementPath>
+      <Type>String</Type>
+      <Width>4</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>paivitetty_tietopalveluun</Name>
+      <ElementPath>paivitetty_tietopalveluun</ElementPath>
+      <Type>String</Type>
+      <Width>10</Width>
+    </PropertyDefn>
+    <PropertyDefn>
+      <Name>lukuvuosi</Name>
+      <ElementPath>lukuvuosi</ElementPath>
+      <Type>String</Type>
+      <Width>9</Width>
+    </PropertyDefn>
+  </GMLFeatureClass>
+</GMLFeatureClassList>