From d6c94a8a050a43978e3493f942b7da80e082e162 Mon Sep 17 00:00:00 2001 From: Matti Eiden Date: Tue, 31 Dec 2024 17:19:29 +0200 Subject: [PATCH 1/9] chore: replace black, isort and flake8 with ruff refs: RATYK-43 --- .gitignore | 2 +- .pre-commit-config.yaml | 27 +++++++++++++++++++++++++++ CHANGELOG.md | 2 +- README.md | 15 +++++++++++++++ pyproject.toml | 29 +++++++++++++++++++++++++++++ requirements.txt | 3 +-- setup.cfg | 8 -------- 7 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml delete mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore index 8650d31..5cc98fa 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ __pycache__ django_munigeo.egg-info/ /build /dist -.idea \ No newline at end of file +.idea diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..371f16d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +default_language_version: + python: python3 +default_install_hook_types: [pre-commit, commit-msg] +default_stages: [pre-commit, manual] +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: check-added-large-files + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.4 + hooks: + - id: ruff + name: ruff lint + - id: ruff-format + name: ruff format + args: [ --check ] + - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook + rev: v9.13.0 + hooks: + - id: commitlint + stages: [commit-msg, manual] + additional_dependencies: ["@commitlint/config-conventional"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 2241fac..75f4c1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pinned the `django-parler` version to `>=2` and add a migration required to upgrade it. ### Fixed -- Add a `tzinfo` to `Street` and `Address.modified_at` migrations to fix the warning +- Add a `tzinfo` to `Street` and `Address.modified_at` migrations to fix the warning saying that a timezone-naive date was passed to a `DateTimeField`. - helsinki importer: Reverted the change introduced in v0.3.6 which broke Helsinki division import - helsinki importer: Fixed empty field value handling diff --git a/README.md b/README.md index 81a24f7..1ae519b 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,18 @@ then ``` python manage.py geo_import helsinki --divisions ``` + +## Code format + +This project uses [Ruff](https://docs.astral.sh/ruff/) for code formatting and quality checking. + +Basic `ruff` commands: + +* lint: `ruff check` +* apply safe lint fixes: `ruff check --fix` +* check formatting: `ruff format --check` +* format: `ruff format` + +[`pre-commit`](https://pre-commit.com/) can be used to install and +run all the formatting tools as git hooks automatically before a +commit. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3664016 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[tool.ruff] +target-version = "py39" + +[tool.ruff.lint] +select = [ + # Pyflakes + "F", + # pycodestyle + "E", + "W", + # isort + "I", + # pep8-naming + "N", + # flake8-bugbear without opinionated rules + "B0", + # flake8-pie + "PIE", + # flake8-print + "T20", +] +extend-per-file-ignores = { "*/migrations/*" = ["E501"], "*/tests/*" = ["E501"] } + +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "helusers.tests.settings" + +[tool.coverage.run] +source = ["helusers"] +omit = ["helusers/migrations/*", "*/tests/*"] diff --git a/requirements.txt b/requirements.txt index edc01f0..1e152b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,14 @@ -black django>=3.2.16 django-mptt django-parler>=2 django-parler-rest djangorestframework -isort psycopg2 pytest pytest-cov pyyaml requests requests-cache +ruff six unicodecsv diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index a50e8e6..0000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[aliases] -test=pytest - -[isort] -profile = black -atomic = true -order_by_type = false -extend_skip_glob = *migrations* From 8ed0f637310feb1df926d4253c69be1500a12e12 Mon Sep 17 00:00:00 2001 From: Matti Eiden Date: Tue, 31 Dec 2024 17:24:53 +0200 Subject: [PATCH 2/9] style: apply ruff check --fix --- munigeo/api.py | 4 ++-- munigeo/importer/athens.py | 11 +---------- munigeo/importer/base.py | 4 +--- munigeo/importer/helsinki.py | 8 ++++---- munigeo/importer/manchester.py | 8 +------- munigeo/management/commands/geo_import.py | 2 -- munigeo/migrations/0001_squashed_0004_building.py | 3 ++- munigeo/migrations/0002_add_parler_translations.py | 2 +- .../migrations/0003_migrate_translations_to_parler.py | 2 +- .../0005_update_translation_foreign_keys.py | 2 +- munigeo/ocd.py | 8 ++++---- munigeo/oldapi.py | 2 -- 12 files changed, 18 insertions(+), 38 deletions(-) diff --git a/munigeo/api.py b/munigeo/api.py index 7954aa4..05d5ad0 100644 --- a/munigeo/api.py +++ b/munigeo/api.py @@ -105,7 +105,7 @@ def translated_fields_to_representation(self, obj, ret): for lang_key, trans_dict in ret.pop("translations", {}).items(): for field_name, translation in trans_dict.items(): - if not field_name in translated_fields: + if field_name not in translated_fields: translated_fields[field_name] = {lang_key: translation} else: translated_fields[field_name].update({lang_key: translation}) @@ -238,7 +238,7 @@ class AdministrativeDivisionSerializer( ): def to_representation(self, obj): ret = super(AdministrativeDivisionSerializer, self).to_representation(obj) - if not "request" in self.context: + if "request" not in self.context: return ret qparams = self.context["request"].query_params if qparams.get("geometry", "").lower() in ("true", "1"): diff --git a/munigeo/importer/athens.py b/munigeo/importer/athens.py index 24bf804..5df9b3f 100644 --- a/munigeo/importer/athens.py +++ b/munigeo/importer/athens.py @@ -1,20 +1,11 @@ # -*- coding: utf-8 -*- -import csv -import io -import json import os # import unicodecsv -import requests import requests_cache -from django import db -from django.conf import settings -from django.contrib.gis.gdal import CoordTransform, DataSource, SpatialReference -from django.contrib.gis.geos import GEOSGeometry, MultiPolygon, Point +from django.contrib.gis.geos import Point -from munigeo import ocd from munigeo.importer.base import Importer, register_importer -from munigeo.importer.sync import ModelSyncher from munigeo.models import * CITADEL_LIST = [ diff --git a/munigeo/importer/base.py b/munigeo/importer/base.py index dc62892..70d6796 100644 --- a/munigeo/importer/base.py +++ b/munigeo/importer/base.py @@ -4,11 +4,9 @@ import requests from django.conf import settings -from django.contrib.gis.gdal import CoordTransform, DataSource, SpatialReference -from django.contrib.gis.geos import GEOSGeometry, MultiPolygon, Point +from django.contrib.gis.geos import Point from django.utils.text import slugify -from munigeo.importer.sync import ModelSyncher from munigeo.models import * diff --git a/munigeo/importer/helsinki.py b/munigeo/importer/helsinki.py index b038ab0..d7e35f2 100644 --- a/munigeo/importer/helsinki.py +++ b/munigeo/importer/helsinki.py @@ -216,7 +216,7 @@ def make_div_id(obj): return obj.origin_id self.logger.info(div["name"]) - if not "origin_id" in div["fields"]: + if "origin_id" not in div["fields"]: raise Exception( "Field 'origin_id' not defined in config section '%s'" % div["name"] ) @@ -527,12 +527,12 @@ def import_pois(self): poi = POI(origin_id=srv_id) poi.name = srv_info["name_fi"] poi.category = cat - if not "address_city_fi" in srv_info: + if "address_city_fi" not in srv_info: self.logger.info("No city!") self.logger.info(srv_info) continue city_name = srv_info["address_city_fi"] - if not city_name in muni_dict: + if city_name not in muni_dict: city_name = city_name.encode("utf8") post_code = srv_info.get("address_zip", "") if post_code.startswith("00"): @@ -564,7 +564,7 @@ def import_pois(self): poi.municipality = muni_dict[city_name] poi.street_address = srv_info.get("street_address_fi", None) poi.zip_code = srv_info.get("address_zip", None) - if not "northing_etrs_gk25" in srv_info: + if "northing_etrs_gk25" not in srv_info: self.logger.info("No location!") self.logger.info(srv_info) continue diff --git a/munigeo/importer/manchester.py b/munigeo/importer/manchester.py index d2a899e..6d85398 100644 --- a/munigeo/importer/manchester.py +++ b/munigeo/importer/manchester.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import csv import io import json import os @@ -7,14 +6,9 @@ # import unicodecsv import requests import requests_cache -from django import db -from django.conf import settings -from django.contrib.gis.gdal import CoordTransform, DataSource, SpatialReference -from django.contrib.gis.geos import GEOSGeometry, MultiPolygon, Point +from django.contrib.gis.geos import Point -from munigeo import ocd from munigeo.importer.base import Importer, register_importer -from munigeo.importer.sync import ModelSyncher from munigeo.models import * POI_LIST = [ diff --git a/munigeo/management/commands/geo_import.py b/munigeo/management/commands/geo_import.py index 8c80e37..77bdc70 100644 --- a/munigeo/management/commands/geo_import.py +++ b/munigeo/management/commands/geo_import.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -import os -from optparse import make_option from django.conf import settings from django.core.management.base import BaseCommand, CommandError diff --git a/munigeo/migrations/0001_squashed_0004_building.py b/munigeo/migrations/0001_squashed_0004_building.py index 8b182ec..6501167 100644 --- a/munigeo/migrations/0001_squashed_0004_building.py +++ b/munigeo/migrations/0001_squashed_0004_building.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.11 on 2018-07-11 06:15 import datetime + import django.contrib.gis.db.models.fields -from django.db import migrations, models import django.db.models.deletion import mptt.fields +from django.db import migrations, models from munigeo.utils import get_default_srid diff --git a/munigeo/migrations/0002_add_parler_translations.py b/munigeo/migrations/0002_add_parler_translations.py index 819944d..2b371d6 100644 --- a/munigeo/migrations/0002_add_parler_translations.py +++ b/munigeo/migrations/0002_add_parler_translations.py @@ -1,7 +1,7 @@ # Generated by Django 2.0.5 on 2018-07-16 11:29 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/munigeo/migrations/0003_migrate_translations_to_parler.py b/munigeo/migrations/0003_migrate_translations_to_parler.py index baf53f6..ec8e2dd 100644 --- a/munigeo/migrations/0003_migrate_translations_to_parler.py +++ b/munigeo/migrations/0003_migrate_translations_to_parler.py @@ -1,8 +1,8 @@ # Generated by Django 2.0.5 on 2018-07-16 13:04 -from django.db import migrations from django.conf import settings from django.core.exceptions import ObjectDoesNotExist +from django.db import migrations def _get_munigeo_models(apps): diff --git a/munigeo/migrations/0005_update_translation_foreign_keys.py b/munigeo/migrations/0005_update_translation_foreign_keys.py index 857d624..b725b76 100644 --- a/munigeo/migrations/0005_update_translation_foreign_keys.py +++ b/munigeo/migrations/0005_update_translation_foreign_keys.py @@ -1,8 +1,8 @@ # Generated by Django 3.0.8 on 2020-08-03 06:26 -from django.db import migrations, models import django.db.models.deletion import parler.fields +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/munigeo/ocd.py b/munigeo/ocd.py index 3e8b50f..3376310 100644 --- a/munigeo/ocd.py +++ b/munigeo/ocd.py @@ -10,11 +10,11 @@ def make_id(parent=None, **kwargs): if len(kwargs) > 1: raise ValueError("only one kwarg is allowed for make_id") type, type_id = list(kwargs.items())[0] - if not re.match("^[\w_-]+$", type, re.UNICODE): - raise ValueError("type must match [\w_]+ [%s]" % type) + if not re.match(r"^[\w_-]+$", type, re.UNICODE): + raise ValueError(r"type must match [\w_]+ [%s]" % type) type_id = type_id.lower() - type_id = re.sub("\.? ", "_", type_id) - type_id = re.sub("[^\w0-9~_.-]", "~", type_id, re.UNICODE) + type_id = re.sub(r"\.? ", "_", type_id) + type_id = re.sub(r"[^\w0-9~_.-]", "~", type_id, re.UNICODE) if parent: return "{parent}/{type}:{type_id}".format( parent=parent, type=type, type_id=type_id diff --git a/munigeo/oldapi.py b/munigeo/oldapi.py index 8f79482..a64d46c 100644 --- a/munigeo/oldapi.py +++ b/munigeo/oldapi.py @@ -3,8 +3,6 @@ from django.contrib.gis.gdal import CoordTransform, SpatialReference, SRSException from django.contrib.gis.geos import Point, Polygon -from django.contrib.gis.measure import D -from django.db.models import Q from modeltranslation.translator import NotRegistered, translator from tastypie import fields from tastypie.cache import SimpleCache From afbcfd6127aea5011ca7db9b73653ea7100feec8 Mon Sep 17 00:00:00 2001 From: Matti Eiden Date: Tue, 31 Dec 2024 17:25:06 +0200 Subject: [PATCH 3/9] style: apply ruff format --- munigeo/api.py | 2 -- munigeo/migrations/0001_squashed_0004_building.py | 1 - munigeo/migrations/0002_add_parler_translations.py | 1 - munigeo/migrations/0003_migrate_translations_to_parler.py | 3 --- munigeo/migrations/0004_delete_old_translations.py | 1 - munigeo/migrations/0005_update_translation_foreign_keys.py | 1 - munigeo/models.py | 1 - 7 files changed, 10 deletions(-) diff --git a/munigeo/api.py b/munigeo/api.py index 05d5ad0..e998c25 100644 --- a/munigeo/api.py +++ b/munigeo/api.py @@ -103,7 +103,6 @@ def translated_fields_to_representation(self, obj, ret): translated_fields = {} for lang_key, trans_dict in ret.pop("translations", {}).items(): - for field_name, translation in trans_dict.items(): if field_name not in translated_fields: translated_fields[field_name] = {lang_key: translation} @@ -166,7 +165,6 @@ def geom_to_json(geom, target_srs): class GeoModelSerializer(serializers.ModelSerializer): - def __init__(self, *args, **kwargs): super(GeoModelSerializer, self).__init__(*args, **kwargs) model = self.Meta.model diff --git a/munigeo/migrations/0001_squashed_0004_building.py b/munigeo/migrations/0001_squashed_0004_building.py index 6501167..b4d5db8 100644 --- a/munigeo/migrations/0001_squashed_0004_building.py +++ b/munigeo/migrations/0001_squashed_0004_building.py @@ -13,7 +13,6 @@ class Migration(migrations.Migration): - replaces = [ ("munigeo", "0001_initial"), ("munigeo", "0002_auto_20150608_1607"), diff --git a/munigeo/migrations/0002_add_parler_translations.py b/munigeo/migrations/0002_add_parler_translations.py index 2b371d6..e397502 100644 --- a/munigeo/migrations/0002_add_parler_translations.py +++ b/munigeo/migrations/0002_add_parler_translations.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("munigeo", "0001_squashed_0004_building"), ] diff --git a/munigeo/migrations/0003_migrate_translations_to_parler.py b/munigeo/migrations/0003_migrate_translations_to_parler.py index ec8e2dd..61bea24 100644 --- a/munigeo/migrations/0003_migrate_translations_to_parler.py +++ b/munigeo/migrations/0003_migrate_translations_to_parler.py @@ -23,7 +23,6 @@ def _get_munigeo_models(apps): def forwards_func(apps, schema_editor): - for lang_code, _ in settings.LANGUAGES: name_field_key = "name_" + lang_code @@ -39,7 +38,6 @@ def forwards_func(apps, schema_editor): def backwards_func(apps, schema_editor): - for lang_code, _ in settings.LANGUAGES: name_field_key = "name_" + lang_code @@ -56,7 +54,6 @@ def backwards_func(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("munigeo", "0002_add_parler_translations"), ] diff --git a/munigeo/migrations/0004_delete_old_translations.py b/munigeo/migrations/0004_delete_old_translations.py index 7c1c1a2..c71c4fc 100644 --- a/munigeo/migrations/0004_delete_old_translations.py +++ b/munigeo/migrations/0004_delete_old_translations.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("munigeo", "0003_migrate_translations_to_parler"), ] diff --git a/munigeo/migrations/0005_update_translation_foreign_keys.py b/munigeo/migrations/0005_update_translation_foreign_keys.py index b725b76..0eb1ad1 100644 --- a/munigeo/migrations/0005_update_translation_foreign_keys.py +++ b/munigeo/migrations/0005_update_translation_foreign_keys.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - dependencies = [ ("munigeo", "0004_delete_old_translations"), ] diff --git a/munigeo/models.py b/munigeo/models.py index 2ea76c9..de204e7 100644 --- a/munigeo/models.py +++ b/munigeo/models.py @@ -32,7 +32,6 @@ def __str__(self): class AdministrativeDivisionQuerySet(TranslatableQuerySet): - def by_ancestor(self, ancestor): manager = self.model.objects max_level = manager.determine_max_level() From dfbbd76b006ac8c0b512c49338a7dce8ae510450 Mon Sep 17 00:00:00 2001 From: Matti Eiden Date: Tue, 31 Dec 2024 17:43:26 +0200 Subject: [PATCH 4/9] refactor: remove redundant re.UNICODE Passing it as 4th argument to re.sub actually meant count=32 refs: RATYK-43 --- munigeo/ocd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/munigeo/ocd.py b/munigeo/ocd.py index 3376310..e6debbe 100644 --- a/munigeo/ocd.py +++ b/munigeo/ocd.py @@ -10,11 +10,11 @@ def make_id(parent=None, **kwargs): if len(kwargs) > 1: raise ValueError("only one kwarg is allowed for make_id") type, type_id = list(kwargs.items())[0] - if not re.match(r"^[\w_-]+$", type, re.UNICODE): + if not re.match(r"^[\w_-]+$", type): raise ValueError(r"type must match [\w_]+ [%s]" % type) type_id = type_id.lower() type_id = re.sub(r"\.? ", "_", type_id) - type_id = re.sub(r"[^\w0-9~_.-]", "~", type_id, re.UNICODE) + type_id = re.sub(r"[^\w0-9~_.-]", "~", type_id) if parent: return "{parent}/{type}:{type_id}".format( parent=parent, type=type, type_id=type_id From 636e6efb78246cfdc21b020442b3be63fd533d23 Mon Sep 17 00:00:00 2001 From: Matti Eiden Date: Tue, 31 Dec 2024 17:23:52 +0200 Subject: [PATCH 5/9] refactor: remove star imports * oldapi.py * helsinki.py * manchester.py * base.py * athens.py refs: RATYK-43 --- munigeo/importer/athens.py | 4 ++-- munigeo/importer/base.py | 2 +- munigeo/importer/helsinki.py | 13 ++++++++++++- munigeo/importer/manchester.py | 2 +- munigeo/oldapi.py | 12 +++++++++++- 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/munigeo/importer/athens.py b/munigeo/importer/athens.py index 5df9b3f..685266b 100644 --- a/munigeo/importer/athens.py +++ b/munigeo/importer/athens.py @@ -6,7 +6,7 @@ from django.contrib.gis.geos import Point from munigeo.importer.base import Importer, register_importer -from munigeo.models import * +from munigeo.models import PROJECTION_SRID, Municipality CITADEL_LIST = [ { @@ -76,7 +76,7 @@ def __init__(self, *args, **kwargs): self.muni_data_path = os.path.join(self.data_path, "gr", "athens") def import_municipalities(self): - muni, c = Municipality.objects.get_or_create(id=30001, name="Athens") + Municipality.objects.get_or_create(id=30001, name="Athens") self.logger.info("Athens municipality added.") def import_pois_from_citadel(self): diff --git a/munigeo/importer/base.py b/munigeo/importer/base.py index 70d6796..b8b4013 100644 --- a/munigeo/importer/base.py +++ b/munigeo/importer/base.py @@ -7,7 +7,7 @@ from django.contrib.gis.geos import Point from django.utils.text import slugify -from munigeo.models import * +from munigeo.models import POI, PROJECTION_SRID, POICategory def convert_from_wgs84(coords): diff --git a/munigeo/importer/helsinki.py b/munigeo/importer/helsinki.py index d7e35f2..23e03e1 100644 --- a/munigeo/importer/helsinki.py +++ b/munigeo/importer/helsinki.py @@ -18,7 +18,18 @@ from munigeo import ocd from munigeo.importer.base import Importer, register_importer from munigeo.importer.sync import ModelSyncher -from munigeo.models import * +from munigeo.models import ( + POI, + PROJECTION_SRID, + Address, + AdministrativeDivision, + AdministrativeDivisionGeometry, + AdministrativeDivisionType, + Municipality, + Plan, + POICategory, + Street, +) MUNI_URL = "https://tilastokeskus.fi/meta/luokitukset/kunta/001-2013/tekstitiedosto.txt" diff --git a/munigeo/importer/manchester.py b/munigeo/importer/manchester.py index 6d85398..a7476a8 100644 --- a/munigeo/importer/manchester.py +++ b/munigeo/importer/manchester.py @@ -9,7 +9,7 @@ from django.contrib.gis.geos import Point from munigeo.importer.base import Importer, register_importer -from munigeo.models import * +from munigeo.models import POI, PROJECTION_SRID, Municipality, POICategory POI_LIST = [ { diff --git a/munigeo/oldapi.py b/munigeo/oldapi.py index a64d46c..189f663 100644 --- a/munigeo/oldapi.py +++ b/munigeo/oldapi.py @@ -1,6 +1,7 @@ import json import re +from django.conf import settings from django.contrib.gis.gdal import CoordTransform, SpatialReference, SRSException from django.contrib.gis.geos import Point, Polygon from modeltranslation.translator import NotRegistered, translator @@ -11,7 +12,16 @@ from tastypie.http import HttpBadRequest from tastypie.resources import ModelResource -from munigeo.models import * +from munigeo.models import ( + POI, + PROJECTION_SRID, + Address, + AdministrativeDivision, + AdministrativeDivisionType, + Municipality, + Plan, + POICategory, +) # Use the GPS coordinate system by default DEFAULT_SRID = 4326 From 610f1f7158914e9027a1b6bf912dcd818d20723d Mon Sep 17 00:00:00 2001 From: Matti Eiden Date: Tue, 31 Dec 2024 17:46:46 +0200 Subject: [PATCH 6/9] refactor: apply manual ruff fixes refs: RATYK-43 --- munigeo/importer/finland.py | 4 ++-- munigeo/importer/helsinki.py | 21 +++++++------------ munigeo/importer/manchester.py | 7 ++++--- .../0003_migrate_translations_to_parler.py | 16 +++++++------- munigeo/oldapi.py | 7 +++---- 5 files changed, 24 insertions(+), 31 deletions(-) diff --git a/munigeo/importer/finland.py b/munigeo/importer/finland.py index de0230e..5cc07b3 100644 --- a/munigeo/importer/finland.py +++ b/munigeo/importer/finland.py @@ -16,11 +16,11 @@ from munigeo.importer.base import Importer, register_importer from munigeo.importer.sync import ModelSyncher from munigeo.models import ( + PROJECTION_SRID, AdministrativeDivision, AdministrativeDivisionGeometry, AdministrativeDivisionType, Municipality, - PROJECTION_SRID, ) from .helsinki import FIN_GRID, TM35_SRID @@ -176,7 +176,7 @@ def import_municipalities(self): self._process_muni(syncher, feat) if executor: for f in futures: - res = f.result() + _ = f.result() executor.shutdown() AdministrativeDivision.objects.rebuild() diff --git a/munigeo/importer/helsinki.py b/munigeo/importer/helsinki.py index 23e03e1..409fdd3 100644 --- a/munigeo/importer/helsinki.py +++ b/munigeo/importer/helsinki.py @@ -340,7 +340,7 @@ def import_plans(self): self._import_plans("Lv_rajaus.TAB", True) self._import_plans("Kaava_vir_rajaus.TAB", False) self.logger.info("Saving") - for key, obj in self.plan_map.items(): + for obj in self.plan_map.values(): if obj.found: obj.save() else: @@ -348,8 +348,6 @@ def import_plans(self): @db.transaction.atomic def import_addresses(self): - none_to_str = lambda s: s or "" - wfs_url = ( "https://kartta.hel.fi/ws/geoserver/avoindata/wfs?" "SERVICE=WFS&VERSION=1.0.0&REQUEST=GetFeature&" @@ -395,7 +393,6 @@ def make_addr_id(num, num_end, letter): street.addrs[make_addr_id(a.number, a.number_end, a.letter)] = a bulk_addr_list = [] - bulk_street_list = [] count = 0 self.logger.info("starting data synchronization") @@ -403,8 +400,8 @@ def make_addr_id(num, num_end, letter): count += 1 if count % 1000 == 0: self.logger.debug("{} processed".format(count)) - street_name = none_to_str(feat.get("katunimi")).strip() - street_name_sv = none_to_str(feat.get("gatan")).strip() + street_name = (feat.get("katunimi") or "").strip() + street_name_sv = (feat.get("gatan") or "").strip() num = feat.get("osoitenumero") @@ -424,10 +421,8 @@ def make_addr_id(num, num_end, letter): ) continue - num2 = none_to_str(feat.get("osoitenumero2")) - if num2 == 0: - num2 = "" - letter = none_to_str(feat.get("osoitekirjain")).strip() + num2 = feat.get("osoitenumero2") or "" + letter = (feat.get("osoitekirjain") or "").strip() coord_n = int(feat.get("n")) coord_e = int(feat.get("e")) @@ -486,8 +481,6 @@ def make_addr_id(num, num_end, letter): # addr.save() addr._found = True - # self.logger.info("%s: %s %d%s N%d E%d (%f,%f)" % (muni_name, street, num, letter, coord_n, coord_e, pnt.y, pnt.x)) - if len(bulk_addr_list) >= 10000: self.logger.info("Saving %d new addresses" % len(bulk_addr_list)) @@ -516,7 +509,7 @@ def make_addr_id(num, num_end, letter): self.logger.info("synchronization complete") def import_pois(self): - URL_BASE = "https://www.hel.fi/palvelukarttaws/rest/v2/unit/?service=%d" + url_base = "https://www.hel.fi/palvelukarttaws/rest/v2/unit/?service=%d" muni_dict = {} for muni in Municipality.objects.all(): @@ -529,7 +522,7 @@ def import_pois(self): ) self.logger.info("Importing %s" % cat_type) - ret = requests.get(URL_BASE % srv_id) + ret = requests.get(url_base % srv_id) for srv_info in ret.json(): srv_id = str(srv_info["id"]) try: diff --git a/munigeo/importer/manchester.py b/munigeo/importer/manchester.py index a7476a8..d5ca731 100644 --- a/munigeo/importer/manchester.py +++ b/munigeo/importer/manchester.py @@ -3,7 +3,6 @@ import json import os -# import unicodecsv import requests import requests_cache from django.contrib.gis.geos import Point @@ -85,6 +84,8 @@ def import_pois_from_csv(self): s = resp.text.encode("utf8").decode("utf8") f = io.StringIO(s) + import unicodecsv + reader = unicodecsv.reader(f, delimiter=",", quotechar='"', encoding="utf8") # skip header next(reader) @@ -111,7 +112,7 @@ def import_pois_from_csv(self): poi.save() def import_pois_from_rest(self): - URL_BASE = "http://www.manchester.gov.uk/site/custom_scripts/getServiceDetailsjs.php?service=%d&postcode=M2+5DB&count=10000&format=json" + url_base = "http://www.manchester.gov.uk/site/custom_scripts/getServiceDetailsjs.php?service=%d&postcode=M2+5DB&count=10000&format=json" muni = Municipality.objects.get(id=44001) for srv_id in list(SERVICE_CATEGORY_MAP.keys()): @@ -121,7 +122,7 @@ def import_pois_from_rest(self): ) self.logger.info("Importing %s" % cat_type) - ret = requests.get(URL_BASE % srv_id) + ret = requests.get(url_base % srv_id) if ret.status_code != 200: raise Exception("HTTP request failed with %d" % ret.status_code) # Fix quoting bug diff --git a/munigeo/migrations/0003_migrate_translations_to_parler.py b/munigeo/migrations/0003_migrate_translations_to_parler.py index 61bea24..31b8f47 100644 --- a/munigeo/migrations/0003_migrate_translations_to_parler.py +++ b/munigeo/migrations/0003_migrate_translations_to_parler.py @@ -6,14 +6,14 @@ def _get_munigeo_models(apps): - AD = apps.get_model("munigeo", "AdministrativeDivision") - ADTranslation = apps.get_model("munigeo", "AdministrativeDivisionTranslation") + AD = apps.get_model("munigeo", "AdministrativeDivision") # noqa: N806 + ADTranslation = apps.get_model("munigeo", "AdministrativeDivisionTranslation") # noqa: N806 - Municipality = apps.get_model("munigeo", "Municipality") - MunicipalityTranslation = apps.get_model("munigeo", "MunicipalityTranslation") + Municipality = apps.get_model("munigeo", "Municipality") # noqa: N806 + MunicipalityTranslation = apps.get_model("munigeo", "MunicipalityTranslation") # noqa: N806 - Street = apps.get_model("munigeo", "Street") - StreetTranslation = apps.get_model("munigeo", "StreetTranslation") + Street = apps.get_model("munigeo", "Street") # noqa: N806 + StreetTranslation = apps.get_model("munigeo", "StreetTranslation") # noqa: N806 return [ (AD, ADTranslation), @@ -26,7 +26,7 @@ def forwards_func(apps, schema_editor): for lang_code, _ in settings.LANGUAGES: name_field_key = "name_" + lang_code - for Model, ModelTranslation in _get_munigeo_models(apps): + for Model, ModelTranslation in _get_munigeo_models(apps): # noqa: N806 for object in Model.objects.all(): translated_name = getattr(object, name_field_key) if translated_name: @@ -41,7 +41,7 @@ def backwards_func(apps, schema_editor): for lang_code, _ in settings.LANGUAGES: name_field_key = "name_" + lang_code - for Model, ModelTranslation in _get_munigeo_models(apps): + for Model, ModelTranslation in _get_munigeo_models(apps): # noqa: N806 for object in Model.objects.all(): try: translation = ModelTranslation.objects.get( diff --git a/munigeo/oldapi.py b/munigeo/oldapi.py index 189f663..4519ece 100644 --- a/munigeo/oldapi.py +++ b/munigeo/oldapi.py @@ -100,13 +100,13 @@ def dehydrate(self, bundle): for lang in LANGUAGES[1:]: key = "%s_%s" % (field_name, lang) val = getattr(bundle.obj, key, None) - if val == None: + if val is None: continue d[lang] = val # If no text provided, leave the field as null - for key, val in d.items(): - if val != None: + for val in d.values(): + if val is not None: break else: d = None @@ -213,7 +213,6 @@ class Meta: "end": ALL, "name": ALL, "parent": ALL_WITH_RELATIONS, - "type": ALL_WITH_RELATIONS, } excludes = ["lft", "rght", "tree_id"] list_allowed_methods = ["get"] From 51a6d4c553f97a237fd168533bfc5b07a5818340 Mon Sep 17 00:00:00 2001 From: Matti Eiden Date: Wed, 8 Jan 2025 09:53:50 +0200 Subject: [PATCH 7/9] ci: add .commitlintrc.mjs refs: RATYK-43 --- .commitlintrc.mjs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .commitlintrc.mjs diff --git a/.commitlintrc.mjs b/.commitlintrc.mjs new file mode 100644 index 0000000..ae3ffa3 --- /dev/null +++ b/.commitlintrc.mjs @@ -0,0 +1,42 @@ +export default { + extends: [ + '@commitlint/config-conventional' + ], + ignores: [ + (message) => message.includes('Signed-off-by: dependabot[bot]') + ], + rules: { + 'header-max-length': [ + 2, + 'always', + 72 + ], + 'body-max-line-length': [ + 2, + 'always', + 72 + ], + 'body-leading-blank': [ + 2, + 'always' + ], + 'type-enum': [ + 2, + 'always', + [ + 'build', + 'chore', + 'ci', + 'deps', + 'docs', + 'feat', + 'fix', + 'perf', + 'refactor', + 'revert', + 'style', + 'test' + ] + ] + } +}; From ae05cb46d7ac9df004344f77ac4cab3ed9c6833e Mon Sep 17 00:00:00 2001 From: Matti Eiden Date: Wed, 8 Jan 2025 09:54:54 +0200 Subject: [PATCH 8/9] ci: add commitlint and ruff to GH test action refs: RATYK-43 --- .github/workflows/test.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 06eb771..29e8d1f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,3 +45,20 @@ jobs: - name: Upload Coverage to Codecov if: ${{ matrix.python_version == env.DEFAULT_PYTHON }} uses: codecov/codecov-action@v4 + + commitlint: + name: Commitlint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check commitlint + uses: wagoid/commitlint-github-action@0d749a1a91d4770e983a7b8f83d4a3f0e7e0874e # v5.4.4 + + ruff: + name: Coding style - ruff + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/ruff-action@v3 + - run: ruff check --fix + - run: ruff format From f97d72de8bf1dab0e8d556c37da07f11e31c1fb7 Mon Sep 17 00:00:00 2001 From: Matti Eiden Date: Thu, 9 Jan 2025 12:26:08 +0200 Subject: [PATCH 9/9] chore: freeze ruff version in requirements.txt And add reminder to keep it in sync with .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 371f16d..7b97d2b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: - id: check-toml - id: check-added-large-files - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.4 + rev: v0.8.4 # Sync with requirements.txt hooks: - id: ruff name: ruff lint diff --git a/requirements.txt b/requirements.txt index 1e152b9..30eb950 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,6 @@ pytest-cov pyyaml requests requests-cache -ruff +ruff==0.8.4 # Sync with .pre-commit-config.yaml six unicodecsv