diff --git a/services/management/commands/school_district_import/__init__.py b/services/management/commands/school_district_import/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/services/management/commands/school_district_import/school_district_importer.py b/services/management/commands/school_district_import/school_district_importer.py new file mode 100644 index 000000000..f0070e341 --- /dev/null +++ b/services/management/commands/school_district_import/school_district_importer.py @@ -0,0 +1,170 @@ +import logging +import re +from datetime import datetime + +from django.contrib.gis.gdal import CoordTransform, DataSource, SpatialReference +from django.contrib.gis.geos import MultiPolygon +from munigeo import ocd +from munigeo.models import ( + AdministrativeDivision, + AdministrativeDivisionGeometry, + AdministrativeDivisionType, + Municipality, +) + +from services.management.commands.lipas_import import MiniWFS + +logger = logging.getLogger(__name__) +SRID = 3067 + + +class SchoolDistrictImporter: + WFS_BASE = "https://kartta.hel.fi/ws/geoserver/avoindata/wfs" + + def __init__(self, district_type): + self.district_type = district_type + + def import_districts(self, data): + """ + Data is a dictionary containing the following keys: + - source_type: The name of the WFS layer to fetch + - division_type: AdministrativeDivisionType type of the division + - ocd_id: The key to use in the OCD ID generation + + """ + wfs = MiniWFS(self.WFS_BASE) + municipality = Municipality.objects.get(id="helsinki") + + source_type = data["source_type"] + division_type = data["division_type"] + ocd_id = data["ocd_id"] + + try: + url = wfs.get_feature(type_name=source_type) + layer = DataSource(url)[0] + except Exception as e: + logger.error(f"Error retrieving data for {source_type}: {e}") + return + + logger.info(f"Retrieved {len(layer)} {source_type} features.") + logger.info("Processing data...") + + division_type_obj, _ = AdministrativeDivisionType.objects.get_or_create( + type=division_type + ) + + for feature in layer: + self.import_division( + feature, + division_type_obj, + municipality, + ocd_id, + source_type, + ) + + def import_division( + self, feature, division_type_obj, municipality, ocd_id, source_type + ): + origin_id = feature.get("id") + if not origin_id: + logger.info("Skipping feature without id.") + return + + division, _ = AdministrativeDivision.objects.get_or_create( + origin_id=origin_id, type=division_type_obj + ) + + division.municipality = municipality + division.parent = municipality.division + + division.ocd_id = ocd.make_id( + **{ocd_id: str(origin_id), "parent": municipality.division.ocd_id} + ) + + service_point_id = str(feature.get("toimipiste_id")) + + if self.district_type == "school": + division.service_point_id = service_point_id + division.units = [service_point_id] + + if "suomi" in source_type: + name = feature.get("nimi_fi") + division.name_fi = feature.get("nimi_fi") + division.start = self.create_start_date(name) + division.end = self.create_end_date(name) + if "ruotsi" in source_type: + name = feature.get("nimi_se") + division.name_sv = feature.get("nimi_se") + division.start = self.create_start_date(name) + division.end = self.create_end_date(name) + + elif self.district_type == "preschool": + units = service_point_id.split(",") + division.units = units + + division.name_fi = feature.get("nimi_fi") + division.name_sv = feature.get("nimi_se") + + division.extra = {"schoolyear": feature.get("lukuvuosi")} + + division.save() + + self.save_geometry(feature, division) + + def create_start_date(self, name): + year = re.split(r"[ -]", name)[-2] + return f"{year}-08-01" + + def create_end_date(self, name): + year = re.split(r"[ -]", name)[-1] + return f"{year }-07-31" + + def save_geometry(self, feature, division): + geom = feature.geom + if not geom.srid: + geom.srid = SRID + if geom.srid != SRID: + geom.transform(SRID) + ct = CoordTransform(SpatialReference(geom.srid), SpatialReference(SRID)) + geom.transform(ct) + + geom = geom.geos + if geom.geom_type == "Polygon": + geom = MultiPolygon(geom.buffer(0), srid=geom.srid) + + try: + geom_obj = division.geometry + except AdministrativeDivisionGeometry.DoesNotExist: + geom_obj = AdministrativeDivisionGeometry(division=division) + + geom_obj.boundary = geom + geom_obj.save() + + def remove_old_school_year(self, division_type): + """ + During 1.8.-15.12. only the current school year is shown. + During 16.12.-31.7. both the current and the next school year are shown. + + The source might be named as "tuleva" but it might still actually be the current school year. + + If today is between 1.8.-15.12 delete the previous year. + """ + division_type_obj = AdministrativeDivisionType.objects.get(type=division_type) + + today = datetime.today() + + last_year = today.year - 1 + last_year_start_date = f"{last_year}-08-01" + + if datetime(today.year, 8, 1) <= today <= datetime(today.year, 12, 15): + if self.district_type == "school": + AdministrativeDivision.objects.filter( + type=division_type_obj, + start=last_year_start_date, + ).delete() + + if self.district_type == "preschool": + AdministrativeDivision.objects.filter( + type=division_type_obj, + extra__schoolyear=f"{last_year}-{last_year + 1}", + ).delete() diff --git a/services/management/commands/update_helsinki_preschool_districts.py b/services/management/commands/update_helsinki_preschool_districts.py new file mode 100644 index 000000000..3f0d7db89 --- /dev/null +++ b/services/management/commands/update_helsinki_preschool_districts.py @@ -0,0 +1,54 @@ +from django.core.management.base import BaseCommand +from munigeo.models import AdministrativeDivision + +from services.management.commands.school_district_import.school_district_importer import ( + SchoolDistrictImporter, +) + +PRESCHOOL_DISTRICT_DATA = [ + { + "source_type": "avoindata:Esiopetusalue_suomi", + "division_type": "preschool_education_fi", + "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi", + }, + { + "source_type": "avoindata:Esiopetusalue_suomi_tuleva", + "division_type": "preschool_education_fi", + "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi", + }, + { + "source_type": "avoindata:Esiopetusalue_ruotsi", + "division_type": "preschool_education_sv", + "ocd_id": "esiopetuksen_oppilaaksiottoalue_sv", + }, + { + "source_type": "avoindata:Esiopetusalue_ruotsi_tuleva", + "division_type": "preschool_education_sv", + "ocd_id": "esiopetuksen_oppilaaksiottoalue_sv", + }, +] + + +class Command(BaseCommand): + help = ( + "Update Helsinki preschool districts. " + "Usage: ./manage.py update_helsinki_preschool_districts" + ) + + def handle(self, *args, **options): + division_types = list( + {data["division_type"] for data in PRESCHOOL_DISTRICT_DATA} + ) + + # Remove old divisions before importing new ones to avoid possible duplicates as the source layers may change + AdministrativeDivision.objects.filter( + type__type__in=division_types, municipality__id="helsinki" + ).delete() + + importer = SchoolDistrictImporter(district_type="preschool") + + for data in PRESCHOOL_DISTRICT_DATA: + importer.import_districts(data) + + for division_type in division_types: + importer.remove_old_school_year(division_type) diff --git a/services/management/commands/update_helsinki_school_districts.py b/services/management/commands/update_helsinki_school_districts.py new file mode 100644 index 000000000..4256b8e57 --- /dev/null +++ b/services/management/commands/update_helsinki_school_districts.py @@ -0,0 +1,72 @@ +from django.core.management.base import BaseCommand +from munigeo.models import AdministrativeDivision + +from services.management.commands.school_district_import.school_district_importer import ( + SchoolDistrictImporter, +) + +SCHOOL_DISTRICT_DATA = [ + { + "source_type": "avoindata:Opev_ooa_alaaste_suomi", + "division_type": "lower_comprehensive_school_district_fi", + "ocd_id": "oppilaaksiottoalue_alakoulu", + }, + { + "source_type": "avoindata:Opev_ooa_alaaste_suomi_tuleva", + "division_type": "lower_comprehensive_school_district_fi", + "ocd_id": "oppilaaksiottoalue_alakoulu", + }, + { + "source_type": "avoindata:Opev_ooa_alaaste_ruotsi", + "division_type": "lower_comprehensive_school_district_sv", + "ocd_id": "oppilaaksiottoalue_alakoulu_sv", + }, + { + "source_type": "avoindata:Opev_ooa_alaaste_ruotsi_tuleva", + "division_type": "lower_comprehensive_school_district_sv", + "ocd_id": "oppilaaksiottoalue_alakoulu_sv", + }, + { + "source_type": "avoindata:Opev_ooa_ylaaste_suomi", + "division_type": "upper_comprehensive_school_district_fi", + "ocd_id": "oppilaaksiottoalue_ylakoulu", + }, + { + "source_type": "avoindata:Opev_ooa_ylaaste_suomi_tuleva", + "division_type": "upper_comprehensive_school_district_fi", + "ocd_id": "oppilaaksiottoalue_ylakoulu", + }, + { + "source_type": "avoindata:Opev_ooa_ylaaste_ruotsi", + "division_type": "upper_comprehensive_school_district_sv", + "ocd_id": "oppilaaksiottoalue_ylakoulu_sv", + }, + { + "source_type": "avoindata:Opev_ooa_ylaaste_ruotsi_tuleva", + "division_type": "upper_comprehensive_school_district_sv", + "ocd_id": "oppilaaksiottoalue_ylakoulu_sv", + }, +] + + +class Command(BaseCommand): + help = ( + "Update Helsinki school districts. " + "Usage: ./manage.py update_helsinki_school_districts" + ) + + def handle(self, *args, **options): + division_types = list({data["division_type"] for data in SCHOOL_DISTRICT_DATA}) + + # Remove old divisions before importing new ones to avoid possible duplicates as the source layers may change + AdministrativeDivision.objects.filter( + type__type__in=division_types, municipality__id="helsinki" + ).delete() + + importer = SchoolDistrictImporter(district_type="school") + + for data in SCHOOL_DISTRICT_DATA: + importer.import_districts(data) + + for division_type in division_types: + importer.remove_old_school_year(division_type) diff --git a/services/tests/data/Esiopetusalue_suomi.gfs b/services/tests/data/Esiopetusalue_suomi.gfs new file mode 100644 index 000000000..936a5ea61 --- /dev/null +++ b/services/tests/data/Esiopetusalue_suomi.gfs @@ -0,0 +1,75 @@ + + + Esiopetusalue_suomi + Esiopetusalue_suomi + geom + geom + + 3 + EPSG:3879 + + 1 + 25503070.45293 + 25508934.14646 + 6670858.66227 + 6673564.24700 + + + id + id + Integer + + + aluejako + aluejako + String + 14 + + + kunta + kunta + Integer + + + nimi_fi + nimi_fi + String + 9 + + + nimi_se + nimi_se + String + 11 + + + toimipiste_id + toimipiste_id + Integer + + + yhtluontipvm + yhtluontipvm + String + 10 + + + yhtdatanomistaja + yhtdatanomistaja + String + 4 + + + paivitetty_tietopalveluun + paivitetty_tietopalveluun + String + 10 + + + lukuvuosi + lukuvuosi + String + 9 + + + diff --git a/services/tests/data/Esiopetusalue_suomi.gml b/services/tests/data/Esiopetusalue_suomi.gml new file mode 100644 index 000000000..9960eeda5 --- /dev/null +++ b/services/tests/data/Esiopetusalue_suomi.gml @@ -0,0 +1,69 @@ + + + unknown + + + + 333 + VAKA_ESIOPETUS + 091 + Testialue + Testområde + 14 + 2022-11-15 + Kami + 2022-11-16 + + + + + 2.55081158198837E7,6672479.619 + 2.5506981451018E7,6672473.686 2.55068492561502E7,6672472.995 + 2.55058955981039E7,6672754.33 2.55061811608183E7,6673224.981 + 2.55061911393084E7,6673294.682 2.55052862502132E7,6673291.056 + 2.55047783552211E7,6673292.719 2.5504743518256E7,6673311.547 + 2.55045639104356E7,6673412.423 2.55042935902059E7,6673564.247 + 2.55042509657485E7,6673514.299 2.55039736550258E7,6673189.339 + 2.55041551538443E7,6672930.699 2.55041404033591E7,6672830.287 + 2.55038442796552E7,6672757.256 2.55033938181057E7,6672400.326 + 2.55033665761329E7,6672412.891 2.55033393346602E7,6672425.456 + 2.55033120641874E7,6672438.135 2.55032848607146E7,6672450.781 + 2.5503257529242E7,6672463.378 2.55032304397691E7,6672475.946 + 2.55032031832963E7,6672488.69 2.55031759268236E7,6672501.435 + 2.55031410628584E7,6672517.641 2.55031380023615E7,6672513.06 + 2.55030847614147E7,6672467.695 2.55030706954288E7,6672457.205 + 2.5503070452929E7,6672448.021 2.55031214088781E7,6672363.913 + 2.55031672863322E7,6672281.089 2.55031517668477E7,6672222.091 + 2.55031497038498E7,6672200.91 2.55031387098608E7,6672169.589 + 2.55031334433661E7,6672164.145 2.55031237588757E7,6672159.844 + 2.55030973109022E7,6672154.581 2.55030763754231E7,6672150.126 + 2.55030794334201E7,6672111.606 2.5503083452916E7,6672091.707 + 2.55030887424108E7,6672080.275 2.55030974174021E7,6672065.245 + 2.55031092103903E7,6672064.45 2.55031255568739E7,6672060.063 + 2.55031520043475E7,6672046.404 2.55031630633364E7,6672041.667 + 2.55031694108301E7,6672039.761 2.55031858838136E7,6672039.308 + 2.55031886643108E7,6672039.429 2.55032009962985E7,6672037.735 + 2.55032171967823E7,6672035.8 2.55032387172608E7,6672039.913 + 2.55032436042559E7,6672040.702 2.55032463947531E7,6672040.75 + 2.55032483292512E7,6672040.508 2.55032500392495E7,6672039.936 + 2.55032513702481E7,6672039.44 2.55032534432461E7,6672038.098 + 2.55032610857384E7,6672034.497 2.55032677182318E7,6672030.487 + 2.55033351691643E7,6671946.55 2.55033106041889E7,6671934.171 + 2.5503350445149E7,6671796.309 2.55033423376916E7,6671682.01195227 + 2.5503424665418E7,6671636.64737965 2.55037373354363E7,6671464.35879341 + 2.550416572618E7,6671228.30499301 2.55044021235923E7,6671098.04463682 + 2.5505098315645E7,6671097.81208525 2.55054537869944E7,6671092.62764915 + 2.55058316864265E7,6670977.6865874 2.55059473683703E7,6670942.21480012 + 2.55061037888314E7,6670902.98177412 2.5506354888807E7,6670858.66226936 + 2.55068853663486E7,6670907.43120793 2.55082306814437E7,6671039.09070425 + 2.55089341464571E7,6671264.45535869 2.55088476158994E7,6671850.9524692 + 2.55087472972522E7,6672530.904 2.55081158198837E7,6672479.619 + + + + + + 2023-2024 + + + \ No newline at end of file diff --git a/services/tests/data/Esiopetusalue_suomi_tuleva.gfs b/services/tests/data/Esiopetusalue_suomi_tuleva.gfs new file mode 100644 index 000000000..936a5ea61 --- /dev/null +++ b/services/tests/data/Esiopetusalue_suomi_tuleva.gfs @@ -0,0 +1,75 @@ + + + Esiopetusalue_suomi + Esiopetusalue_suomi + geom + geom + + 3 + EPSG:3879 + + 1 + 25503070.45293 + 25508934.14646 + 6670858.66227 + 6673564.24700 + + + id + id + Integer + + + aluejako + aluejako + String + 14 + + + kunta + kunta + Integer + + + nimi_fi + nimi_fi + String + 9 + + + nimi_se + nimi_se + String + 11 + + + toimipiste_id + toimipiste_id + Integer + + + yhtluontipvm + yhtluontipvm + String + 10 + + + yhtdatanomistaja + yhtdatanomistaja + String + 4 + + + paivitetty_tietopalveluun + paivitetty_tietopalveluun + String + 10 + + + lukuvuosi + lukuvuosi + String + 9 + + + diff --git a/services/tests/data/Esiopetusalue_suomi_tuleva.gml b/services/tests/data/Esiopetusalue_suomi_tuleva.gml new file mode 100644 index 000000000..339df53bb --- /dev/null +++ b/services/tests/data/Esiopetusalue_suomi_tuleva.gml @@ -0,0 +1,69 @@ + + + unknown + + + + 444 + VAKA_ESIOPETUS + 091 + Testialue + Testområde + 14 + 2022-11-15 + Kami + 2022-11-16 + + + + + 2.55081158198837E7,6672479.619 + 2.5506981451018E7,6672473.686 2.55068492561502E7,6672472.995 + 2.55058955981039E7,6672754.33 2.55061811608183E7,6673224.981 + 2.55061911393084E7,6673294.682 2.55052862502132E7,6673291.056 + 2.55047783552211E7,6673292.719 2.5504743518256E7,6673311.547 + 2.55045639104356E7,6673412.423 2.55042935902059E7,6673564.247 + 2.55042509657485E7,6673514.299 2.55039736550258E7,6673189.339 + 2.55041551538443E7,6672930.699 2.55041404033591E7,6672830.287 + 2.55038442796552E7,6672757.256 2.55033938181057E7,6672400.326 + 2.55033665761329E7,6672412.891 2.55033393346602E7,6672425.456 + 2.55033120641874E7,6672438.135 2.55032848607146E7,6672450.781 + 2.5503257529242E7,6672463.378 2.55032304397691E7,6672475.946 + 2.55032031832963E7,6672488.69 2.55031759268236E7,6672501.435 + 2.55031410628584E7,6672517.641 2.55031380023615E7,6672513.06 + 2.55030847614147E7,6672467.695 2.55030706954288E7,6672457.205 + 2.5503070452929E7,6672448.021 2.55031214088781E7,6672363.913 + 2.55031672863322E7,6672281.089 2.55031517668477E7,6672222.091 + 2.55031497038498E7,6672200.91 2.55031387098608E7,6672169.589 + 2.55031334433661E7,6672164.145 2.55031237588757E7,6672159.844 + 2.55030973109022E7,6672154.581 2.55030763754231E7,6672150.126 + 2.55030794334201E7,6672111.606 2.5503083452916E7,6672091.707 + 2.55030887424108E7,6672080.275 2.55030974174021E7,6672065.245 + 2.55031092103903E7,6672064.45 2.55031255568739E7,6672060.063 + 2.55031520043475E7,6672046.404 2.55031630633364E7,6672041.667 + 2.55031694108301E7,6672039.761 2.55031858838136E7,6672039.308 + 2.55031886643108E7,6672039.429 2.55032009962985E7,6672037.735 + 2.55032171967823E7,6672035.8 2.55032387172608E7,6672039.913 + 2.55032436042559E7,6672040.702 2.55032463947531E7,6672040.75 + 2.55032483292512E7,6672040.508 2.55032500392495E7,6672039.936 + 2.55032513702481E7,6672039.44 2.55032534432461E7,6672038.098 + 2.55032610857384E7,6672034.497 2.55032677182318E7,6672030.487 + 2.55033351691643E7,6671946.55 2.55033106041889E7,6671934.171 + 2.5503350445149E7,6671796.309 2.55033423376916E7,6671682.01195227 + 2.5503424665418E7,6671636.64737965 2.55037373354363E7,6671464.35879341 + 2.550416572618E7,6671228.30499301 2.55044021235923E7,6671098.04463682 + 2.5505098315645E7,6671097.81208525 2.55054537869944E7,6671092.62764915 + 2.55058316864265E7,6670977.6865874 2.55059473683703E7,6670942.21480012 + 2.55061037888314E7,6670902.98177412 2.5506354888807E7,6670858.66226936 + 2.55068853663486E7,6670907.43120793 2.55082306814437E7,6671039.09070425 + 2.55089341464571E7,6671264.45535869 2.55088476158994E7,6671850.9524692 + 2.55087472972522E7,6672530.904 2.55081158198837E7,6672479.619 + + + + + + 2024-2025 + + + \ No newline at end of file diff --git a/services/tests/data/Opev_ooa_alaaste_suomi.gfs b/services/tests/data/Opev_ooa_alaaste_suomi.gfs new file mode 100644 index 000000000..86470d47a --- /dev/null +++ b/services/tests/data/Opev_ooa_alaaste_suomi.gfs @@ -0,0 +1,62 @@ + + + Opev_ooa_alaaste_suomi + Opev_ooa_alaaste_suomi + geom + geom + + 3 + EPSG:3879 + + 1 + 25506216.82371 + 25511334.40710 + 6676844.46030 + 6680491.48868 + + + id + id + Integer + + + aluejako + aluejako + String + 22 + + + kunta + kunta + Integer + + + tunnus + tunnus + Integer + + + nimi_fi + nimi_fi + String + 26 + + + toimipiste_id + toimipiste_id + Integer + + + yhtdatanomistaja + yhtdatanomistaja + String + 13 + + + paivitetty_tietopalveluun + paivitetty_tietopalveluun + String + 10 + + + diff --git a/services/tests/data/Opev_ooa_alaaste_suomi.gml b/services/tests/data/Opev_ooa_alaaste_suomi.gml new file mode 100644 index 000000000..d58f71da0 --- /dev/null +++ b/services/tests/data/Opev_ooa_alaaste_suomi.gml @@ -0,0 +1,117 @@ + + + unknown + + + + 222 + Opev_ooa_alaaste_suomi + 091 + 0 + Testi peruskoulu 2023-2024 + 13 + Helsinki/Kami + 2024-09-27 + + + + + 2.55075382107937E7,6677435.21188668 + 2.55074906001382E7,6677692.89090943 2.5507600429264E7,6677722.66474609 + 2.5507689158213E7,6677798.45269396 2.55077870692392E7,6677922.2811869 + 2.55079161522918E7,6677923.85989317 2.55080237204139E7,6678000.25387238 + 2.55081642614251E7,6678026.53559838 2.5508209139441E7,6678030.29759756 + 2.55082921941724E7,6678037.25905953 2.55083514173708E7,6678046.54734857 + 2.55096549761638E7,6678243.10385819 2.55109823316143E7,6678374.14011326 + 2.55110832040296E7,6678388.90746421 2.55111653276567E7,6678400.93008846 + 2.55112575809221E7,6679109.30986401 2.55112879656234E7,6679331.66491867 + 2.55113040973355E7,6679474.38291014 2.5511325685422E7,6679703.87886057 + 2.55113344071041E7,6679796.59922638 2.55110234239462E7,6679632.19188944 + 2.55110097597145E7,6679624.96750003 2.55108917002075E7,6679562.55445211 + 2.55108420650894E7,6679533.12916255 2.55106969238568E7,6679450.14640833 + 2.5510539502283E7,6679371.67156822 2.55101209671187E7,6679588.1723399 + 2.55100968363873E7,6679562.80153978 2.55099637008601E7,6679617.00388462 + 2.5509829794421E7,6679671.52008434 2.5509256407571E7,6679904.9582478 + 2.5509218983584E7,6679922.99824 2.55091955173497E7,6679934.31002638 + 2.55090514998465E7,6680003.73279873 2.55090187845773E7,6680019.50305514 + 2.55089982333206E7,6680029.40965263 2.55089903147995E7,6680033.22673018 + 2.55089707638617E7,6680040.58015381 2.55088852027018E7,6680072.91166138 + 2.55088807129746E7,6680054.79856422 2.55088757499525E7,6680054.81673121 + 2.55088678929714E7,6680055.83986635 2.55088363442607E7,6680065.76448623 + 2.55088286142794E7,6680066.78866866 2.55088180920079E7,6680066.54712266 + 2.55088044016451E7,6680064.53640575 2.550879616225E7,6680064.54621213 + 2.55087799428578E7,6680067.22808292 2.55087599159135E7,6680067.25256168 + 2.55087373505516E7,6680064.7433211 2.55087227727813E7,6680061.71998259 + 2.55087079400414E7,6680060.21663347 2.55086969099391E7,6680057.94862095 + 2.55086863856584E7,6680054.53881316 2.55086729422052E7,6680047.96405096 + 2.55086640631179E7,6680042.65128661 2.55086002711374E7,6679993.2461441 + 2.55084090263816E7,6680424.31081241 2.55082967063579E7,6680409.34572752 + 2.55082825490108E7,6679985.16424301 2.55082816425766E7,6679958.1796378 + 2.5507983370921E7,6680156.93801329 2.55079095048199E7,6680204.56558665 + 2.55076939411026E7,6680350.65381494 2.55076840507712E7,6680409.66983947 + 2.55076199812095E7,6680448.19013705 2.55075617249917E7,6680483.22528446 + 2.55075480080821E7,6680491.48868336 2.55075427511189E7,6680477.58026115 + 2.55075412376821E7,6680454.16414543 2.5507524788317E7,6680413.28448501 + 2.55075999753089E7,6680236.49608314 2.55076064928013E7,6680206.35968752 + 2.5507613577783E7,6680141.70349969 2.55076158903417E7,6680033.73403939 + 2.55076163440218E7,6679996.12811889 2.55076153376453E7,6679958.62775268 + 2.55076080344856E7,6679915.17702454 2.55076035185737E7,6679895.2466794 + 2.55075871277984E7,6679839.2218197 2.55075772435658E7,6679805.00139055 + 2.55075670344091E7,6679769.84296267 2.55075472512272E7,6679704.7981837 + 2.55075437433727E7,6679693.15903284 2.55075416873689E7,6679684.40350379 + 2.55075385315184E7,6679669.6936644 2.55075366011507E7,6679660.80298582 + 2.55075302511357E7,6679634.6149166 2.55075281047167E7,6679625.76269735 + 2.55075259839879E7,6679617.15214447 2.550752410445E7,6679608.28680449 + 2.55075230385712E7,6679599.39498652 2.55075216457769E7,6679590.40416272 + 2.55075204871966E7,6679581.51025623 2.55075200228943E7,6679576.01467658 + 2.55075188210638E7,6679570.40548497 2.55075175830459E7,6679566.3867189 + 2.55075154916938E7,6679561.08389332 2.55075130696624E7,6679555.62886706 + 2.55075081122941E7,6679545.71123954 2.55075064165209E7,6679541.64212441 + 2.55075054208952E7,6679538.28455942 2.55075035229377E7,6679530.98292391 + 2.55075008194508E7,6679517.24512965 2.5507498186594E7,6679506.19442011 + 2.55074966274713E7,6679501.58644677 2.55074951634908E7,6679497.76305016 + 2.55074928819307E7,6679492.86764868 2.55074915813995E7,6679489.75214756 + 2.55074903442076E7,6679486.42037273 2.5507489766909E7,6679484.58628756 + 2.55074885195309E7,6679480.85012954 2.55074868624199E7,6679476.00137674 + 2.55074847176254E7,6679469.74792616 2.55074700368093E7,6679444.14606671 + 2.55074558303167E7,6679413.7441912 2.55074437043468E7,6679394.35938972 + 2.55074402883919E7,6679377.39778851 2.55074376265939E7,6679364.98182919 + 2.5507415059277E7,6679328.99186932 2.55074017922741E7,6679320.04852174 + 2.55073574884625E7,6679244.26894603 2.55073272324104E7,6679194.78678305 + 2.55073216061879E7,6679186.56434056 2.55073101596057E7,6679174.99383391 + 2.55072892588084E7,6679150.76303745 2.55072624873629E7,6679120.86446351 + 2.55071716276072E7,6679028.72474839 2.55071478460489E7,6679002.25364088 + 2.55071265608518E7,6678972.77956584 2.55070925396437E7,6678954.82067359 + 2.5507068537545E7,6678952.84955437 2.55070450216134E7,6678939.37798057 + 2.55070365175137E7,6678935.88824476 2.55070343478104E7,6678935.28286031 + 2.55070258098337E7,6678931.89516408 2.55070007969865E7,6678919.31138004 + 2.55069710989367E7,6678886.8865534 2.5506953113668E7,6678855.47103269 + 2.55069467353602E7,6678841.0158649 2.55069325447991E7,6678817.46039287 + 2.55069161953372E7,6678793.63451961 2.55069100956947E7,6678759.19194264 + 2.55069080369514E7,6678755.56015898 2.55069038260084E7,6678751.4925679 + 2.5506901468812E7,6678748.72873326 2.55068971213244E7,6678739.88206036 + 2.55068047563324E7,6678549.97125995 2.55067999286057E7,6678408.51186311 + 2.550679420531E7,6678258.08724436 2.55066477370226E7,6677950.06702883 + 2.55065797847631E7,6677784.44940167 2.55065517412112E7,6677741.47560498 + 2.5506505843664E7,6677670.64168958 2.55064777005009E7,6677627.32491653 + 2.55064296705021E7,6677530.81115525 2.55063324008957E7,6677335.3530145 + 2.55063213005232E7,6677251.76720544 2.55062168237147E7,6677125.62665981 + 2.55063572075945E7,6677047.28120116 2.55064521327026E7,6677003.90332047 + 2.55065681522791E7,6676948.80168823 2.55066501863231E7,6676917.14755907 + 2.55067181573882E7,6676889.01055538 2.55067619751072E7,6676877.34547875 + 2.55068062864538E7,6676866.37332968 2.5506857615263E7,6676857.35642622 + 2.55069150390938E7,6676847.97742499 2.55069642595202E7,6676844.46029953 + 2.5507007620372E7,6676846.80504984 2.5507062700373E7,6676853.83930076 + 2.55071435625021E7,6676871.42492807 2.55072045020777E7,6676896.0448063 + 2.55072502067593E7,6676910.11330815 2.55073134901647E7,6676951.14643854 + 2.55073802893149E7,6676989.83481862 2.5507497522901E7,6677065.23852979 + 2.55074776154139E7,6677145.05231484 2.55075382107937E7,6677317.199225 + 2.55075382107937E7,6677435.21188668 + + + + + + + + \ No newline at end of file diff --git a/services/tests/data/Opev_ooa_alaaste_suomi_tuleva.gfs b/services/tests/data/Opev_ooa_alaaste_suomi_tuleva.gfs new file mode 100644 index 000000000..924a2ac5e --- /dev/null +++ b/services/tests/data/Opev_ooa_alaaste_suomi_tuleva.gfs @@ -0,0 +1,62 @@ + + + Opev_ooa_alaaste_suomi_tuleva + Opev_ooa_alaaste_suomi_tuleva + geom + geom + + 3 + EPSG:3879 + + 1 + 25506216.82371 + 25511334.40710 + 6676844.46030 + 6680491.48868 + + + id + id + Integer + + + aluejako + aluejako + String + 33 + + + kunta + kunta + Integer + + + tunnus + tunnus + Integer + + + nimi_fi + nimi_fi + String + 26 + + + toimipiste_id + toimipiste_id + Integer + + + yhtdatanomistaja + yhtdatanomistaja + String + 13 + + + paivitetty_tietopalveluun + paivitetty_tietopalveluun + String + 10 + + + diff --git a/services/tests/data/Opev_ooa_alaaste_suomi_tuleva.gml b/services/tests/data/Opev_ooa_alaaste_suomi_tuleva.gml new file mode 100644 index 000000000..ea1291709 --- /dev/null +++ b/services/tests/data/Opev_ooa_alaaste_suomi_tuleva.gml @@ -0,0 +1,117 @@ + + + unknown + + + + 111 + opev_ooa_alaaste_suomi_tuleva_tpr + 091 + 0 + Testi peruskoulu 2024-2025 + 13 + Helsinki/Kami + 2024-09-27 + + + + + 2.55075382107937E7,6677435.21188668 + 2.55074906001382E7,6677692.89090943 2.5507600429264E7,6677722.66474609 + 2.5507689158213E7,6677798.45269396 2.55077870692392E7,6677922.2811869 + 2.55079161522918E7,6677923.85989317 2.55080237204139E7,6678000.25387238 + 2.55081642614251E7,6678026.53559838 2.5508209139441E7,6678030.29759756 + 2.55082921941724E7,6678037.25905953 2.55083514173708E7,6678046.54734857 + 2.55096549761638E7,6678243.10385819 2.55109823316143E7,6678374.14011326 + 2.55110832040296E7,6678388.90746421 2.55111653276567E7,6678400.93008846 + 2.55112575809221E7,6679109.30986401 2.55112879656234E7,6679331.66491867 + 2.55113040973355E7,6679474.38291014 2.5511325685422E7,6679703.87886057 + 2.55113344071041E7,6679796.59922638 2.55110234239462E7,6679632.19188944 + 2.55110097597145E7,6679624.96750003 2.55108917002075E7,6679562.55445211 + 2.55108420650894E7,6679533.12916255 2.55106969238568E7,6679450.14640833 + 2.5510539502283E7,6679371.67156822 2.55101209671187E7,6679588.1723399 + 2.55100968363873E7,6679562.80153978 2.55099637008601E7,6679617.00388462 + 2.5509829794421E7,6679671.52008434 2.5509256407571E7,6679904.9582478 + 2.5509218983584E7,6679922.99824 2.55091955173497E7,6679934.31002638 + 2.55090514998465E7,6680003.73279873 2.55090187845773E7,6680019.50305514 + 2.55089982333206E7,6680029.40965263 2.55089903147995E7,6680033.22673018 + 2.55089707638617E7,6680040.58015381 2.55088852027018E7,6680072.91166138 + 2.55088807129746E7,6680054.79856422 2.55088757499525E7,6680054.81673121 + 2.55088678929714E7,6680055.83986635 2.55088363442607E7,6680065.76448623 + 2.55088286142794E7,6680066.78866866 2.55088180920079E7,6680066.54712266 + 2.55088044016451E7,6680064.53640575 2.550879616225E7,6680064.54621213 + 2.55087799428578E7,6680067.22808292 2.55087599159135E7,6680067.25256168 + 2.55087373505516E7,6680064.7433211 2.55087227727813E7,6680061.71998259 + 2.55087079400414E7,6680060.21663347 2.55086969099391E7,6680057.94862095 + 2.55086863856584E7,6680054.53881316 2.55086729422052E7,6680047.96405096 + 2.55086640631179E7,6680042.65128661 2.55086002711374E7,6679993.2461441 + 2.55084090263816E7,6680424.31081241 2.55082967063579E7,6680409.34572752 + 2.55082825490108E7,6679985.16424301 2.55082816425766E7,6679958.1796378 + 2.5507983370921E7,6680156.93801329 2.55079095048199E7,6680204.56558665 + 2.55076939411026E7,6680350.65381494 2.55076840507712E7,6680409.66983947 + 2.55076199812095E7,6680448.19013705 2.55075617249917E7,6680483.22528446 + 2.55075480080821E7,6680491.48868336 2.55075427511189E7,6680477.58026115 + 2.55075412376821E7,6680454.16414543 2.5507524788317E7,6680413.28448501 + 2.55075999753089E7,6680236.49608314 2.55076064928013E7,6680206.35968752 + 2.5507613577783E7,6680141.70349969 2.55076158903417E7,6680033.73403939 + 2.55076163440218E7,6679996.12811889 2.55076153376453E7,6679958.62775268 + 2.55076080344856E7,6679915.17702454 2.55076035185737E7,6679895.2466794 + 2.55075871277984E7,6679839.2218197 2.55075772435658E7,6679805.00139055 + 2.55075670344091E7,6679769.84296267 2.55075472512272E7,6679704.7981837 + 2.55075437433727E7,6679693.15903284 2.55075416873689E7,6679684.40350379 + 2.55075385315184E7,6679669.6936644 2.55075366011507E7,6679660.80298582 + 2.55075302511357E7,6679634.6149166 2.55075281047167E7,6679625.76269735 + 2.55075259839879E7,6679617.15214447 2.550752410445E7,6679608.28680449 + 2.55075230385712E7,6679599.39498652 2.55075216457769E7,6679590.40416272 + 2.55075204871966E7,6679581.51025623 2.55075200228943E7,6679576.01467658 + 2.55075188210638E7,6679570.40548497 2.55075175830459E7,6679566.3867189 + 2.55075154916938E7,6679561.08389332 2.55075130696624E7,6679555.62886706 + 2.55075081122941E7,6679545.71123954 2.55075064165209E7,6679541.64212441 + 2.55075054208952E7,6679538.28455942 2.55075035229377E7,6679530.98292391 + 2.55075008194508E7,6679517.24512965 2.5507498186594E7,6679506.19442011 + 2.55074966274713E7,6679501.58644677 2.55074951634908E7,6679497.76305016 + 2.55074928819307E7,6679492.86764868 2.55074915813995E7,6679489.75214756 + 2.55074903442076E7,6679486.42037273 2.5507489766909E7,6679484.58628756 + 2.55074885195309E7,6679480.85012954 2.55074868624199E7,6679476.00137674 + 2.55074847176254E7,6679469.74792616 2.55074700368093E7,6679444.14606671 + 2.55074558303167E7,6679413.7441912 2.55074437043468E7,6679394.35938972 + 2.55074402883919E7,6679377.39778851 2.55074376265939E7,6679364.98182919 + 2.5507415059277E7,6679328.99186932 2.55074017922741E7,6679320.04852174 + 2.55073574884625E7,6679244.26894603 2.55073272324104E7,6679194.78678305 + 2.55073216061879E7,6679186.56434056 2.55073101596057E7,6679174.99383391 + 2.55072892588084E7,6679150.76303745 2.55072624873629E7,6679120.86446351 + 2.55071716276072E7,6679028.72474839 2.55071478460489E7,6679002.25364088 + 2.55071265608518E7,6678972.77956584 2.55070925396437E7,6678954.82067359 + 2.5507068537545E7,6678952.84955437 2.55070450216134E7,6678939.37798057 + 2.55070365175137E7,6678935.88824476 2.55070343478104E7,6678935.28286031 + 2.55070258098337E7,6678931.89516408 2.55070007969865E7,6678919.31138004 + 2.55069710989367E7,6678886.8865534 2.5506953113668E7,6678855.47103269 + 2.55069467353602E7,6678841.0158649 2.55069325447991E7,6678817.46039287 + 2.55069161953372E7,6678793.63451961 2.55069100956947E7,6678759.19194264 + 2.55069080369514E7,6678755.56015898 2.55069038260084E7,6678751.4925679 + 2.5506901468812E7,6678748.72873326 2.55068971213244E7,6678739.88206036 + 2.55068047563324E7,6678549.97125995 2.55067999286057E7,6678408.51186311 + 2.550679420531E7,6678258.08724436 2.55066477370226E7,6677950.06702883 + 2.55065797847631E7,6677784.44940167 2.55065517412112E7,6677741.47560498 + 2.5506505843664E7,6677670.64168958 2.55064777005009E7,6677627.32491653 + 2.55064296705021E7,6677530.81115525 2.55063324008957E7,6677335.3530145 + 2.55063213005232E7,6677251.76720544 2.55062168237147E7,6677125.62665981 + 2.55063572075945E7,6677047.28120116 2.55064521327026E7,6677003.90332047 + 2.55065681522791E7,6676948.80168823 2.55066501863231E7,6676917.14755907 + 2.55067181573882E7,6676889.01055538 2.55067619751072E7,6676877.34547875 + 2.55068062864538E7,6676866.37332968 2.5506857615263E7,6676857.35642622 + 2.55069150390938E7,6676847.97742499 2.55069642595202E7,6676844.46029953 + 2.5507007620372E7,6676846.80504984 2.5507062700373E7,6676853.83930076 + 2.55071435625021E7,6676871.42492807 2.55072045020777E7,6676896.0448063 + 2.55072502067593E7,6676910.11330815 2.55073134901647E7,6676951.14643854 + 2.55073802893149E7,6676989.83481862 2.5507497522901E7,6677065.23852979 + 2.55074776154139E7,6677145.05231484 2.55075382107937E7,6677317.199225 + 2.55075382107937E7,6677435.21188668 + + + + + + + + \ No newline at end of file diff --git a/services/tests/test_update_school_districts.py b/services/tests/test_update_school_districts.py new file mode 100644 index 000000000..9925387ab --- /dev/null +++ b/services/tests/test_update_school_districts.py @@ -0,0 +1,251 @@ +from datetime import datetime +from unittest.mock import patch + +import pytest +from django.contrib.gis.geos import MultiPolygon +from django.core.management import call_command +from munigeo.models import ( + AdministrativeDivision, + AdministrativeDivisionType, + Municipality, +) + + +@pytest.fixture +def municipality(): + municipality_type = AdministrativeDivisionType.objects.create(type="municipality") + municipality_division = AdministrativeDivision.objects.create( + type=municipality_type, + name="Helsinki", + ocd_id="ocd-division/country:fi/kunta:helsinki", + ) + municipality = Municipality.objects.create( + id="helsinki", name="helsinki", division=municipality_division + ) + return municipality + + +def get_mock_data(source_type): + if source_type == "avoindata:Esiopetusalue_suomi": + return "services/tests/data/Esiopetusalue_suomi.gml" + elif source_type == "avoindata:Esiopetusalue_suomi_tuleva": + return "services/tests/data/Esiopetusalue_suomi_tuleva.gml" + elif source_type == "avoindata:Opev_ooa_alaaste_suomi_tuleva": + return "services/tests/data/Opev_ooa_alaaste_suomi_tuleva.gml" + return "services/tests/data/Opev_ooa_alaaste_suomi.gml" + + +@pytest.mark.django_db +@patch( + "services.management.commands.update_helsinki_school_districts.SCHOOL_DISTRICT_DATA", + [ + { + "source_type": "avoindata:Opev_ooa_alaaste_suomi", + "division_type": "lower_comprehensive_school_district_fi", + "ocd_id": "oppilaaksiottoalue_alakoulu", + }, + { + "source_type": "avoindata:Opev_ooa_alaaste_suomi_tuleva", + "division_type": "lower_comprehensive_school_district_fi", + "ocd_id": "oppilaaksiottoalue_alakoulu", + }, + ], +) +@patch("services.management.commands.lipas_import.MiniWFS.get_feature") +@patch( + "services.management.commands.school_district_import.school_district_importer.datetime" +) +def test_update_school_districts(mock_datetime, get_feature_mock, municipality): + mock_datetime.today.return_value = datetime(2023, 1, 1) + mock_datetime.side_effect = lambda *args, **kwargs: datetime(*args, **kwargs) + + get_feature_mock.side_effect = lambda type_name: get_mock_data(type_name) + + assert not AdministrativeDivision.objects.filter( + type__type="lower_comprehensive_school_district_fi" + ).exists() + + call_command("update_helsinki_school_districts") + + assert ( + AdministrativeDivision.objects.filter( + type__type="lower_comprehensive_school_district_fi" + ).count() + == 2 + ) + + division_1 = AdministrativeDivision.objects.get(origin_id="111") + assert division_1.name == "Testi peruskoulu 2024-2025" + assert division_1.type.type == "lower_comprehensive_school_district_fi" + assert division_1.municipality == municipality + assert division_1.parent == municipality.division + assert ( + division_1.ocd_id + == "ocd-division/country:fi/kunta:helsinki/oppilaaksiottoalue_alakoulu:111" + ) + assert division_1.service_point_id == "13" + assert type(division_1.geometry.boundary) is MultiPolygon + + division_2 = AdministrativeDivision.objects.get(origin_id="222") + assert division_2.name == "Testi peruskoulu 2023-2024" + assert division_2.type.type == "lower_comprehensive_school_district_fi" + assert division_2.municipality == municipality + assert division_2.parent == municipality.division + assert ( + division_2.ocd_id + == "ocd-division/country:fi/kunta:helsinki/oppilaaksiottoalue_alakoulu:222" + ) + assert division_2.service_point_id == "13" + assert type(division_2.geometry.boundary) is MultiPolygon + + +@pytest.mark.django_db +@patch( + "services.management.commands.update_helsinki_school_districts.SCHOOL_DISTRICT_DATA", + [ + { + "source_type": "avoindata:Opev_ooa_alaaste_suomi", + "division_type": "lower_comprehensive_school_district_fi", + "ocd_id": "oppilaaksiottoalue_alakoulu", + }, + { + "source_type": "avoindata:Opev_ooa_alaaste_suomi_tuleva", + "division_type": "lower_comprehensive_school_district_fi", + "ocd_id": "oppilaaksiottoalue_alakoulu", + }, + ], +) +@patch("services.management.commands.lipas_import.MiniWFS.get_feature") +@patch( + "services.management.commands.school_district_import.school_district_importer.datetime" +) +def test_update_school_districts_removes_school_year( + mock_datetime, get_feature_mock, municipality +): + """ + When date is between 1.8. and 15.12. the previous school year should be removed + """ + mock_datetime.today.return_value = datetime(2024, 8, 1) + mock_datetime.side_effect = lambda *args, **kwargs: datetime(*args, **kwargs) + + get_feature_mock.side_effect = lambda type_name: get_mock_data(type_name) + + call_command("update_helsinki_school_districts") + assert ( + AdministrativeDivision.objects.filter( + type__type="lower_comprehensive_school_district_fi" + ).count() + == 1 + ) + assert AdministrativeDivision.objects.filter( + name="Testi peruskoulu 2024-2025" + ).exists() + + +@pytest.mark.django_db +@patch( + "services.management.commands.update_helsinki_preschool_districts.PRESCHOOL_DISTRICT_DATA", + [ + { + "source_type": "avoindata:Esiopetusalue_suomi", + "division_type": "preschool_education_fi", + "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi", + }, + { + "source_type": "avoindata:Esiopetusalue_suomi_tuleva", + "division_type": "preschool_education_fi", + "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi", + }, + ], +) +@patch("services.management.commands.lipas_import.MiniWFS.get_feature") +@patch( + "services.management.commands.school_district_import.school_district_importer.datetime" +) +def test_update_preschool_districtcs(mock_datetime, get_feature_mock, municipality): + mock_datetime.today.return_value = datetime(2023, 1, 1) + mock_datetime.side_effect = lambda *args, **kwargs: datetime(*args, **kwargs) + + get_feature_mock.side_effect = lambda type_name: get_mock_data(type_name) + + assert not AdministrativeDivision.objects.filter( + type__type="preschool_education_fi" + ).exists() + + call_command("update_helsinki_preschool_districts") + + assert ( + AdministrativeDivision.objects.filter( + type__type="preschool_education_fi" + ).count() + == 2 + ) + division_1 = AdministrativeDivision.objects.get(origin_id="333") + assert division_1.name_fi == "Testialue" + assert division_1.name_sv == "Testområde" + assert division_1.type.type == "preschool_education_fi" + assert division_1.municipality == municipality + assert division_1.parent == municipality.division + assert ( + division_1.ocd_id + == "ocd-division/country:fi/kunta:helsinki/esiopetuksen_oppilaaksiottoalue_fi:333" + ) + assert division_1.units == [14] + assert division_1.extra == {"schoolyear": "2023-2024"} + assert type(division_1.geometry.boundary) is MultiPolygon + + division_2 = AdministrativeDivision.objects.get(origin_id="444") + assert division_2.name_fi == "Testialue" + assert division_2.name_sv == "Testområde" + assert division_2.type.type == "preschool_education_fi" + assert division_2.municipality == municipality + assert division_2.parent == municipality.division + assert ( + division_2.ocd_id + == "ocd-division/country:fi/kunta:helsinki/esiopetuksen_oppilaaksiottoalue_fi:444" + ) + assert division_2.units == [14] + assert division_2.extra == {"schoolyear": "2024-2025"} + assert type(division_2.geometry.boundary) is MultiPolygon + + +@pytest.mark.django_db +@patch( + "services.management.commands.update_helsinki_preschool_districts.PRESCHOOL_DISTRICT_DATA", + [ + { + "source_type": "avoindata:Esiopetusalue_suomi", + "division_type": "preschool_education_fi", + "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi", + }, + { + "source_type": "avoindata:Esiopetusalue_suomi_tuleva", + "division_type": "preschool_education_fi", + "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi", + }, + ], +) +@patch("services.management.commands.lipas_import.MiniWFS.get_feature") +@patch( + "services.management.commands.school_district_import.school_district_importer.datetime" +) +def test_update_preschool_districts_removes_school_year( + mock_datetime, get_feature_mock, municipality +): + """ + When date is between 1.8. and 15.12. the previous preschool year should be removed + """ + mock_datetime.today.return_value = datetime(2024, 8, 1) + mock_datetime.side_effect = lambda *args, **kwargs: datetime(*args, **kwargs) + + get_feature_mock.side_effect = lambda type_name: get_mock_data(type_name) + + call_command("update_helsinki_preschool_districts") + + assert ( + AdministrativeDivision.objects.filter( + type__type="preschool_education_fi" + ).count() + == 1 + ) + assert AdministrativeDivision.objects.filter(extra__schoolyear="2024-2025").exists()