From e28b17fa8611e247845ad6ca2d49fb9b74525b41 Mon Sep 17 00:00:00 2001 From: mrsmrynk Date: Thu, 21 Dec 2023 16:19:24 +0100 Subject: [PATCH] upgrade python version and dependencies --- .flake8 | 3 +- .github/workflows/tests.yaml | 6 +- requirements.txt | 27 +++-- requirements_dev.txt | 31 +++--- src/parsing/config.py | 200 +++++++++++++++++++---------------- 5 files changed, 140 insertions(+), 127 deletions(-) diff --git a/.flake8 b/.flake8 index 9f80abc..ff9e62c 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,7 @@ [flake8] ignore = - W503 # despite being in the anti-pattern section, this will soon be considered the best practice + # despite being in the anti-pattern section, this will soon be considered the best practice + W503 max-line-length = 120 per-file-ignores = */__init__.py: F401 \ No newline at end of file diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b502715..9636a7a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -7,12 +7,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - name: Set up Python 3.8 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.11' - name: Display Python version run: python -c "import sys; print(sys.version)" diff --git a/requirements.txt b/requirements.txt index b152f3f..6dd01f4 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,13 @@ -enlighten==1.10.2 -geopandas==0.11.1 -natsort==8.1.0 -numpy==1.23.1 -onnxruntime==1.12.0 -OWSLib==0.25.0 -pandas==1.4.3 -Pillow==9.2.0 -pydantic==1.9.1 -PyYAML==6.0 -rasterio==1.3.0 -rtree==1.0.0 -Shapely==1.8.2 -topojson==1.4 \ No newline at end of file +geopandas==0.14.1 +natsort==8.4.0 +numpy==1.26.2 +onnxruntime==1.16.3 +OWSLib==0.29.3 +pandas==2.1.4 +Pillow==10.1.0 +pydantic==2.5.2 +PyYAML==6.0.1 +rasterio==1.3.9 +rtree==1.1.0 +Shapely==2.0.2 +topojson==1.7 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 0cfb377..9936e2c 100755 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,18 +1,17 @@ -coverage==7.3.2 -enlighten==1.10.2 -flake8==5.0.4 -geopandas==0.11.1 -natsort==8.1.0 -numpy==1.23.1 -onnxruntime==1.12.0 -OWSLib==0.25.0 -pandas==1.4.3 -Pillow==9.2.0 -pydantic==1.9.1 +coverage==7.3.4 +flake8==6.1.0 +geopandas==0.14.1 +natsort==8.4.0 +numpy==1.26.2 +onnxruntime==1.16.3 +OWSLib==0.29.3 +pandas==2.1.4 +Pillow==10.1.0 +pydantic==2.5.2 pytest==7.4.3 pytest-cov==4.1.0 -PyYAML==6.0 -rasterio==1.3.0 -rtree==1.0.0 -Shapely==1.8.2 -topojson==1.4 \ No newline at end of file +PyYAML==6.0.1 +rasterio==1.3.9 +rtree==1.1.0 +Shapely==2.0.2 +topojson==1.7 \ No newline at end of file diff --git a/src/parsing/config.py b/src/parsing/config.py index ade2100..653f401 100644 --- a/src/parsing/config.py +++ b/src/parsing/config.py @@ -1,11 +1,15 @@ from pathlib import Path -from typing import List, Union +from typing import Self import geopandas as gpd import numpy as np -import pydantic from natsort import natsorted -from pydantic import root_validator, validator + +from pydantic import ( + BaseModel, + field_validator, + model_validator, + ValidationInfo) from src.data import WebMapService @@ -29,37 +33,39 @@ TileSizeError) -class WMS(pydantic.BaseModel): +class WMS(BaseModel): url: str layer: str - @validator('url') + # noinspection PyNestedDecorators + @field_validator('url') + @classmethod def validate_url(cls, - value): + value: str) -> str: """ | Validates url. - :param str value: url + :param value: url :returns: validated url - :rtype: str """ _ = WebMapService(url=value) return value - @validator('layer') + # noinspection PyNestedDecorators + @field_validator('layer') + @classmethod def validate_layer(cls, - value, - values): + value: str, + values: ValidationInfo) -> str: """ | Validates layer. - :param str value: layer - :param dict[str, Any] values: values + :param value: layer + :param values: values :returns: validated layer - :rtype: str :raises WMSLayerError: if layer is not a valid layer """ - web_map_service = WebMapService(url=values['url']) + web_map_service = WebMapService(url=values.data['url']) layers_valid = web_map_service.get_layers() if value not in layers_valid: @@ -69,33 +75,34 @@ def validate_layer(cls, return value -class Data(pydantic.BaseModel): +class Data(BaseModel): rgb: WMS nir: WMS epsg_code: int - path_boundary: str = None - bounding_box: List[int] = None + path_boundary: str | None = None + bounding_box: list[int] | None = None apply_padding: bool = False ignore_processed_tiles: bool = False - @validator('epsg_code') + # noinspection PyNestedDecorators + @field_validator('epsg_code') + @classmethod def validate_epsg_code(cls, - value, - values): + value: int, + values: ValidationInfo) -> int: """ | Validates epsg_code. - :param int value: epsg_code - :param dict[str, Any] values: values + :param value: epsg_code + :param values: values :returns: validated epsg_code - :rtype: int :raises WMSEPSGCodeError: if epsg_code is not a valid epsg code """ - web_map_service_rgb = WebMapService(url=values['rgb'].url) - web_map_service_nir = WebMapService(url=values['nir'].url) + web_map_service_rgb = WebMapService(url=values.data['rgb'].url) + web_map_service_nir = WebMapService(url=values.data['nir'].url) - epsg_codes_valid_rgb = web_map_service_rgb.get_epsg_codes(values['rgb'].layer) - epsg_codes_valid_nir = web_map_service_nir.get_epsg_codes(values['nir'].layer) + epsg_codes_valid_rgb = web_map_service_rgb.get_epsg_codes(values.data['rgb'].layer) + epsg_codes_valid_nir = web_map_service_nir.get_epsg_codes(values.data['nir'].layer) epsg_codes_valid = list(set(epsg_codes_valid_rgb) & set(epsg_codes_valid_nir)) @@ -105,15 +112,16 @@ def validate_epsg_code(cls, return value - @validator('path_boundary') + # noinspection PyNestedDecorators + @field_validator('path_boundary') + @classmethod def validate_path_boundary(cls, - value): + value: str | None) -> Path | None: """ | Validates path_boundary. - :param str or None value: path_boundary + :param value: path_boundary :returns: validated path_boundary - :rtype: Path or None :raises GeoDataEmptyError: if the geo data is empty :raises GeoDataFormatError: if the file extension of the geo data is not .gpkg or .shp :raises GeoDataGeometryError: if the geo data contains invalid polygons @@ -156,25 +164,26 @@ def validate_path_boundary(cls, return value - @validator('bounding_box', always=True) + # noinspection PyNestedDecorators + @field_validator('bounding_box') + @classmethod def validate_bounding_box(cls, - value, - values): + value: list[int] | None, + values: ValidationInfo) -> tuple[int, ...]: """ | Validates bounding_box. - :param list[int] or None value: bounding_box - :param dict[str, Any] values: values + :param value: bounding_box + :param values: values :returns: validated bounding_box - :rtype: (int, int, int, int) :raises BoundingBoxLengthError: if the length of bounding_box is not equal to 4 :raises BoundingBoxNotDefinedError: if neither path_boundary nor bounding_box are defined in the config :raises BoundingBoxValueError: if x_min >= x_max or y_min >= y_max """ - if value is None and values['path_boundary'] is None: + if value is None and values.data['path_boundary'] is None: raise BoundingBoxNotDefinedError() - if values['path_boundary'] is None: + if values.data['path_boundary'] is None: if len(value) != 4: raise BoundingBoxLengthError(bounding_box=value) @@ -182,7 +191,7 @@ def validate_bounding_box(cls, raise BoundingBoxValueError(bounding_box=value) else: - boundary = gpd.read_file(Path(values['path_boundary'])) + boundary = gpd.read_file(Path(values.data['path_boundary'])) bounding_box_boundary = boundary.total_bounds value = ( @@ -193,30 +202,32 @@ def validate_bounding_box(cls, return tuple(value) - @validator('apply_padding') + # noinspection PyNestedDecorators + @field_validator('apply_padding') + @classmethod def validate_apply_padding(cls, - value): + value: bool | None) -> bool: """ | Validates apply_padding. - :param bool or None value: apply_padding + :param value: apply_padding :returns: validated apply_padding - :rtype: bool """ if value is None: value = False return value - @validator('ignore_processed_tiles') + # noinspection PyNestedDecorators + @field_validator('ignore_processed_tiles') + @classmethod def validate_ignore_processed_tiles(cls, - value): + value: bool | None) -> bool: """ | Validates ignore_processed_tiles. - :param bool or None value: ignore_processed_tiles + :param value: ignore_processed_tiles :returns: validated ignore_processed_tiles - :rtype: bool """ if value is None: value = False @@ -224,19 +235,20 @@ def validate_ignore_processed_tiles(cls, return value -class Postprocessing(pydantic.BaseModel): - sieve_size: int = None +class Postprocessing(BaseModel): + sieve_size: int | None = None simplify: bool = False - @validator('sieve_size') + # noinspection PyNestedDecorators + @field_validator('sieve_size') + @classmethod def validate_sieve_size(cls, - value): + value: int | None) -> int | None: """ | Validates sieve_size. - :param int or None value: sieve_size + :param value: sieve_size :returns: validated sieve_size - :rtype: int or None :raises SieveSizeError: if sieve_size is not a number in the range of 0 to 10 """ if value is None: @@ -250,15 +262,16 @@ def validate_sieve_size(cls, return value - @validator('simplify') + # noinspection PyNestedDecorators + @field_validator('simplify') + @classmethod def validate_simplify(cls, - value): + value: bool | None) -> bool: """ | Validates simplify. - :param bool or None value: simplify + :param value: simplify :returns: validated simplify - :rtype: bool """ if value is None: value = False @@ -266,19 +279,20 @@ def validate_simplify(cls, return value -class Aggregation(pydantic.BaseModel): - tile_size: Union[int, List[Union[int, None]], None] = [] - path_aggregation_areas: Union[str, List[Union[str, None]], None] = [] +class Aggregation(BaseModel): + tile_size: int | list[int | None] | None = [] + path_aggregation_areas: str | list[str | None] | None = [] - @validator('tile_size') + # noinspection PyNestedDecorators + @field_validator('tile_size') + @classmethod def validate_tile_size(cls, - value): + value: int | list[int | None] | None) -> list[int]: """ | Validates tile_size. - :param int or list[int or None] or None value: tile_size + :param value: tile_size :returns: validated tile_size - :rtype: list[int] :raises TileSizeError: if tile_size is not a number greater than 0 """ if value is None: @@ -301,15 +315,16 @@ def validate_tile_size(cls, return list(set(value)) - @validator('path_aggregation_areas') + # noinspection PyNestedDecorators + @field_validator('path_aggregation_areas') + @classmethod def validate_path_aggregation_areas(cls, - value): + value: str | list[str | None] | None) -> list[Path]: """ | Validates path_aggregation_areas. - :param str or list[str or None] or None value: path_aggregation_areas + :param value: path_aggregation_areas :returns: validated path_aggregation_areas - :rtype: list[Path] :raises GeoDataEmptyError: if the geo data is empty :raises GeoDataFormatError: if the file extension of the geo data is not .gpkg or .shp :raises GeoDataGeometryError: if the geo data contains invalid polygons @@ -362,19 +377,20 @@ def validate_path_aggregation_areas(cls, return natsorted(list(set(value))) -class Export(pydantic.BaseModel): +class Export(BaseModel): path_output_dir: str prefix: str - @validator('path_output_dir') + # noinspection PyNestedDecorators + @field_validator('path_output_dir') + @classmethod def validate_path_output_dir(cls, - value): + value: str) -> Path: """ | Validates path_output_dir. - :param str value: path_output_dir + :param value: path_output_dir :returns: validated path_output_dir - :rtype: Path :raises OutputDirNotFoundError: if the output directory does not exist """ value = Path(value) @@ -384,15 +400,16 @@ def validate_path_output_dir(cls, return value - @validator('prefix') + # noinspection PyNestedDecorators + @field_validator('prefix') + @classmethod def validate_prefix(cls, - value): + value: str) -> str: """ | Validates prefix. - :param str value: prefix + :param value: prefix :returns: validated prefix - :rtype: str :raises PrefixError: if prefix contains only whitespaces or underscores """ value = value.replace(' ', '').rstrip('_') @@ -403,30 +420,27 @@ def validate_prefix(cls, return value -class Config(pydantic.BaseModel): +class Config(BaseModel): data: Data postprocessing: Postprocessing = Postprocessing() aggregation: Aggregation = Aggregation() export: Export - @root_validator - def validate_tile_size(cls, - values): + @model_validator(mode='after') + def validate_tile_size(self) -> Self: """ | Validates tile_size. - :param dict[str, Any] values: values :returns: validated values - :rtype: dict[str, Any] """ - if not values['Aggregation'].tile_size: - return values + if not self.aggregation.tile_size: + return self - tile_size_max = min(values['data'].bounding_box[2] - values['data'].bounding_box[0], - values['data'].bounding_box[3] - values['data'].bounding_box[1]) + tile_size_max = min(self.data.bounding_box[2] - self.data.bounding_box[0], + self.data.bounding_box[3] - self.data.bounding_box[1]) - values['Aggregation'].tile_size = [tile_size - for tile_size in values['Aggregation'].tile_size - if tile_size <= tile_size_max] + self.aggregation.tile_size = [tile_size + for tile_size in self.aggregation.tile_size + if tile_size <= tile_size_max] - return values + return self