Staupegel: 122,88 mNN
Stauinhalt: 38,516 Mio. m3
Füllungsgrad: 94,16 %
Staupegel: 122,88 mNN
Stauinhalt: 38,516 Mio. m3
+ Füllungsgrad: 94,16 %
Staupegel: 123,12 mNN
Stauinhalt: 38,890 Mio. m3
Füllungsgrad: 95,29 %
Staupegel: 123,12 mNN
Stauinhalt: 38,890 Mio. m3
+ Füllungsgrad: 95,29 %
Staupegel: 123,21 mNN
Stauinhalt: 39,155 Mio. m3
Füllungsgrad: 95,72 %
Staupegel: 123,21 mNN
Stauinhalt: 39,155 Mio. m3
+ Füllungsgrad: 95,72 %
(Eine normale Abgabemenge liegt bei zirka 130.000 m³/d.)
-193 400 m³ am 03. August 1990
189.450 m³ am 07. August 2020
189.062 m³ am 06. August 2020
193 400 m³ am 03. August 1990
189.450 m³ am 07. August 2020
+ 189.062 m³ am 06. August 2020
(Eine normale Abgabemenge liegt bei zirka 130.000 m³/d.)
-193 400 m³ am 03. August 1990
189.450 m³ am 07. August 2020
189.062 m³ am 06. August 2020
193 400 m³ am 03. August 1990
189.450 m³ am 07. August 2020
+ 189.062 m³ am 06. August 2020
""" diff --git a/ddj_cloud/scrapers/talsperren/federations/wupper.py b/ddj_cloud/scrapers/talsperren/federations/wupper.py index ba5feb2..1309e6d 100644 --- a/ddj_cloud/scrapers/talsperren/federations/wupper.py +++ b/ddj_cloud/scrapers/talsperren/federations/wupper.py @@ -1,9 +1,15 @@ import datetime as dt -from typing import Iterable +from collections.abc import Iterable import requests -from ..common import TZ_UTC, ReservoirMeta, ReservoirRecord, Federation, apply_guarded +from ddj_cloud.scrapers.talsperren.common import ( + TZ_UTC, + Federation, + ReservoirMeta, + ReservoirRecord, + apply_guarded, +) class WupperReservoirMeta(ReservoirMeta): @@ -13,7 +19,7 @@ class WupperReservoirMeta(ReservoirMeta): class WupperFederation(Federation): name = "Wupperverband" - reservoirs: dict[str, WupperReservoirMeta] = { + reservoirs: dict[str, WupperReservoirMeta] = { # type: ignore "Bevertalsperre": { "ajax_id": "SBE$-T", "capacity_mio_m3": 23.76, @@ -143,6 +149,9 @@ def _get_reservoir_records(self, name: str) -> list[ReservoirRecord]: if row[value_idx] is not None ] - def get_data(self, **kwargs) -> Iterable[ReservoirRecord]: + def get_data( + self, + **kwargs, # noqa: ARG002 + ) -> Iterable[ReservoirRecord]: for records in apply_guarded(self._get_reservoir_records, self.reservoirs.keys()): yield from records diff --git a/ddj_cloud/scrapers/talsperren/locator_maps.py b/ddj_cloud/scrapers/talsperren/locator_maps.py index a70ea8c..cfa8fcf 100644 --- a/ddj_cloud/scrapers/talsperren/locator_maps.py +++ b/ddj_cloud/scrapers/talsperren/locator_maps.py @@ -3,14 +3,14 @@ import re from typing import Literal -from datawrapper import Datawrapper import pandas as pd import sentry_sdk +from datawrapper import Datawrapper from ddj_cloud.scrapers.talsperren.common import FEDERATION_RENAMES, RESERVOIR_RENAMES +from ddj_cloud.utils.date_and_time import BERLIN from ddj_cloud.utils.formatting import format_datetime, format_number - REVERSE_RESERVOIR_RENAMES = {v: k for k, v in RESERVOIR_RENAMES.items()} DATAWRAPPER_TOKEN = os.environ.get("TALSPERREN_DATAWRAPPER_TOKEN") @@ -51,18 +51,17 @@ def _get_color(fill_percent: float, color_map: dict) -> str: def _make_tooltip(current: dict, variant: Literal["desktop", "mobile"]) -> str: - bar_text = format_number(current["fill_percent"], places=1) + " %" bar_color = _get_color(current["fill_percent"], color_map_fill) bar_text_color = _get_color(current["fill_percent"], color_map_text) - bar_text_margin = "28px" if current["fill_percent"] < 25 else "3px" + bar_text_margin = "28px" if current["fill_percent"] < 25 else "3px" # noqa: PLR2004 name = RESERVOIR_RENAMES.get(current["name"], current["name"]) fill_percent = max(min(current["fill_percent"], 100), 0) content_mio_m3 = format_number(current["content_mio_m3"], places=2) capacity_mio_m3 = format_number(current["capacity_mio_m3"], places=2) federation_name = FEDERATION_RENAMES.get(current["federation_name"], current["federation_name"]) - ts_measured = format_datetime(current["ts_measured"]) + ts_measured = format_datetime(current["ts_measured"].astimezone(BERLIN)) width = "156px" # if variant == "desktop" else "100px" # Doesn't seem to shrink anyways font_size_header = "13px" if variant == "desktop" else "10px" @@ -93,11 +92,7 @@ def _make_tooltip(current: dict, variant: Literal["desktop", "mobile"]) -> str: -""".replace( - " ", "" - ).replace( - "\n", "" - ) +""".replace(" ", "").replace("\n", "") return tooltip_html @@ -125,6 +120,7 @@ def run(df_base: pd.DataFrame) -> None: ("cZfsi", "BL07F"), ("WSgd6", "aHate"), ("QZfQN", "kcuUG"), + ("ZyHl6", "nDeDt"), ] for chart_id_base, chart_id_live in charts: print() @@ -138,7 +134,7 @@ def run(df_base: pd.DataFrame) -> None: sentry_sdk.capture_exception(e) -def _process_chart(current: dict, dw: Datawrapper, chart_id_base: str, chart_id_live: str) -> None: +def _process_chart(current: dict, dw: Datawrapper, chart_id_base: str, chart_id_live: str) -> None: # noqa: PLR0915 chart_base = dw.get_chart(chart_id_base) chart_live = dw.get_chart(chart_id_live) @@ -185,7 +181,7 @@ def _process_chart(current: dict, dw: Datawrapper, chart_id_base: str, chart_id_ print("Updating tooltip markers") for marker in chart_data_base["markers"]: - if not marker["type"] == "point": + if marker["type"] != "point": continue match = re.match(r'Tooltipmarker: "(.*?)".*', marker["title"]) @@ -204,7 +200,9 @@ def _process_chart(current: dict, dw: Datawrapper, chart_id_base: str, chart_id_ variant = ( "desktop" if marker["visibility"]["desktop"] - else "mobile" if marker["visibility"]["mobile"] else None + else "mobile" + if marker["visibility"]["mobile"] + else None ) if variant is None: diff --git a/ddj_cloud/scrapers/talsperren/locator_maps_create_tooltip_markers.py b/ddj_cloud/scrapers/talsperren/locator_maps_create_tooltip_markers.py index 5b9aafb..1da7528 100644 --- a/ddj_cloud/scrapers/talsperren/locator_maps_create_tooltip_markers.py +++ b/ddj_cloud/scrapers/talsperren/locator_maps_create_tooltip_markers.py @@ -1,9 +1,9 @@ import os from uuid import uuid4 as uuid -from datawrapper import Datawrapper import pandas as pd import sentry_sdk +from datawrapper import Datawrapper from .locator_maps import RENAMES @@ -11,7 +11,6 @@ def _make_marker(current: dict) -> dict: - return { "title": f'Tooltipmarker: "{current["name"]}"', "id": str(uuid()), diff --git a/ddj_cloud/scrapers/talsperren/talsperren.py b/ddj_cloud/scrapers/talsperren/talsperren.py index c9807e5..2f57ed9 100644 --- a/ddj_cloud/scrapers/talsperren/talsperren.py +++ b/ddj_cloud/scrapers/talsperren/talsperren.py @@ -1,19 +1,20 @@ -from traceback import print_exc +import datetime as dt from os import getenv +from traceback import print_exc import pandas as pd -import datetime as dt import sentry_sdk from ddj_cloud.scrapers.talsperren.exporters.map import filtered_map_exporters from ddj_cloud.utils.storage import ( DownloadFailedException, + download_file, upload_dataframe, upload_file, - download_file, ) -from .common import Federation, Exporter, ReservoirMeta, to_parquet_bio + from . import locator_maps +from .common import Exporter, Federation, ReservoirMeta, to_parquet_bio IGNORE_LIST = [ "Rurtalsperre Gesamt", @@ -43,7 +44,7 @@ def _get_base_dataset(): # Instantiate all federation classes federation_classes = Federation.__subclasses__() - federations = [cls() for cls in federation_classes] # type: ignore + federations = [cls() for cls in federation_classes] # Get data from all federations data = [] @@ -62,13 +63,10 @@ def _get_base_dataset(): df_new["ts_measured"] = pd.to_datetime(df_new["ts_measured"], utc=True) # Add timestamp - df_new["ts_scraped"] = dt.datetime.now(dt.timezone.utc) + df_new["ts_scraped"] = dt.datetime.now(dt.UTC) # Merge with existing data - if not is_first_run: - df = pd.concat([df_db, df_new]) - else: - df = df_new + df = pd.concat([df_db, df_new]) if not is_first_run else df_new # Deduplicate, but keep new data if there are duplicates df = df.drop_duplicates( @@ -106,7 +104,7 @@ def _get_base_dataset(): continue df[column] = df.apply( - lambda row: metas[(row["federation_name"], row["name"])][column], + lambda row: metas[(row["federation_name"], row["name"])][column], # noqa: B023 axis=1, ) diff --git a/ddj_cloud/utils/bigquery.py b/ddj_cloud/utils/bigquery.py index 41db3ce..5e3c79a 100644 --- a/ddj_cloud/utils/bigquery.py +++ b/ddj_cloud/utils/bigquery.py @@ -1,6 +1,7 @@ """Utility functions .""" + import re -from typing import Callable, Generator +from collections.abc import Callable, Generator import pandas as pd from google.cloud import bigquery @@ -57,7 +58,7 @@ def iter_results( client: bigquery.Client, query: str, job_config: QueryJobConfig, - df_cleaner: Callable[[pd.DataFrame], pd.DataFrame] = None, + df_cleaner: Callable[[pd.DataFrame], pd.DataFrame] | None = None, ) -> Generator[pd.Series, None, None]: """ Page through the results of a query and yield each row as a pandas Series @@ -75,6 +76,7 @@ def iter_results( query_job.result() # Get reference to destination table + assert query_job.destination is not None destination = client.get_table(query_job.destination) rows = client.list_rows(destination, page_size=10000) @@ -89,8 +91,9 @@ def iter_results( _pandas_helpers.verify_pandas_imports = imports_verifier_orig for df in dfs: + df_cleaned = df if df_cleaner is not None: - df = df_cleaner(df) + df_cleaned = df_cleaner(df) - for index, row in df.iterrows(): + for _, row in df_cleaned.iterrows(): yield row diff --git a/ddj_cloud/utils/datawrapper_patched.py b/ddj_cloud/utils/datawrapper_patched.py index 0af291b..4e494f4 100644 --- a/ddj_cloud/utils/datawrapper_patched.py +++ b/ddj_cloud/utils/datawrapper_patched.py @@ -1,7 +1,7 @@ import json -from datawrapper import Datawrapper as DatawrapperOriginal import requests as r +from datawrapper import Datawrapper as DatawrapperOriginal class Datawrapper(DatawrapperOriginal): diff --git a/ddj_cloud/utils/date_and_time.py b/ddj_cloud/utils/date_and_time.py index 64fd04d..4fca767 100644 --- a/ddj_cloud/utils/date_and_time.py +++ b/ddj_cloud/utils/date_and_time.py @@ -1,7 +1,6 @@ """Provides helper functions for dates.""" import datetime as dt -from typing import List, Optional from zoneinfo import ZoneInfo BERLIN = ZoneInfo("Europe/Berlin") @@ -44,7 +43,7 @@ def local_yesterday() -> dt.date: return local_today() - dt.timedelta(days=1) -def date_range(start: dt.date, end: dt.date) -> List[dt.date]: +def date_range(start: dt.date, end: dt.date) -> list[dt.date]: """Generate a list of dates within a range. Start and end are both inclusive. @@ -61,12 +60,12 @@ def date_range(start: dt.date, end: dt.date) -> List[dt.date]: def date_param( - date: Optional[dt.date], + date: dt.date | None, *, - default: Optional[dt.date] = None, - earliest: Optional[dt.date] = None, - latest: Optional[dt.date] = None, -) -> Optional[dt.date]: + default: dt.date | None = None, + earliest: dt.date | None = None, + latest: dt.date | None = None, +) -> dt.date | None: """For when you have an optional date parameter in your function but you want to limit the range of dates allowed. Also allows you to set a default. @@ -91,7 +90,7 @@ def date_param( return date -def to_timedelta(seconds: Optional[int]) -> Optional[dt.timedelta]: +def to_timedelta(seconds: int | None) -> dt.timedelta | None: """Generate a timedelta from an int containing a number of seconds. Args: @@ -107,7 +106,7 @@ def to_timedelta(seconds: Optional[int]) -> Optional[dt.timedelta]: return None -def iso_as_local(date_time: Optional[str], tz: ZoneInfo = BERLIN) -> Optional[dt.datetime]: +def iso_as_local(date_time: str | None, tz: ZoneInfo = BERLIN) -> dt.datetime | None: """Add timezone info to timezone naive isoformat date/time string. Args: diff --git a/ddj_cloud/utils/storage.py b/ddj_cloud/utils/storage.py index 315f323..9520962 100644 --- a/ddj_cloud/utils/storage.py +++ b/ddj_cloud/utils/storage.py @@ -1,14 +1,14 @@ import os -from os.path import commonprefix as common_prefix +from collections.abc import Callable from io import BytesIO +from os.path import commonprefix as common_prefix from pathlib import Path -from typing import Any, Callable, Optional, Union +from typing import Any from uuid import uuid4 import pandas as pd - -from boto3 import client import sentry_sdk +from boto3 import client from ddj_cloud.utils.date_and_time import local_today @@ -47,7 +47,7 @@ def describe_events(*, clear: bool = True) -> list[str]: list[str]: List of events """ - def _describe(fs_event): + def _describe(fs_event): # noqa: PLR0911 if fs_event["type"] == "download": if fs_event["success"]: return f'Downloaded file "{fs_event["filename"]}" from storage' @@ -84,7 +84,7 @@ def simple_compare(old: Any, new: Any) -> bool: def make_df_compare_fn( - *, ignore_columns: Union[str, list[str], None] = None + *, ignore_columns: str | list[str] | None = None ) -> Callable[[bytes, bytes], bool]: """Create a function that can be used as the ``compare_fn`` argument to ``upload_dataframe`` to compare two pandas DataFrames by their contents while ignoring specified columns. @@ -122,11 +122,13 @@ def _download_file(filename: str) -> BytesIO: with open(LOCAL_STORAGE_ROOT / filename, "rb") as fp: bio = BytesIO(fp.read()) else: + assert s3 is not None bio = BytesIO() s3.download_fileobj(BUCKET_NAME, filename, bio) - except: - raise DownloadFailedException(f"Failed to download file {filename}") + except Exception as err: + msg = f"Failed to download file {filename}" + raise DownloadFailedException(msg) from err bio.seek(0) return bio @@ -158,8 +160,8 @@ def __upload_file( content: bytes, filename: str, *, - acl: Optional[str] = None, - content_type: Optional[str] = None, + acl: str | None = None, + content_type: str | None = None, ): bio = BytesIO(content) @@ -183,6 +185,7 @@ def __upload_file( if content_type is not None: extra_args["ContentType"] = content_type + assert s3 is not None s3.upload_fileobj( bio, BUCKET_NAME, @@ -195,8 +198,8 @@ def _upload_file( content: bytes, filename: str, *, - acl: Optional[str] = None, - content_type: Optional[str] = None, + acl: str | None = None, + content_type: str | None = None, archive: bool = True, ) -> list[str]: """Internal file upload function that performs optional achiving and storage event tracking""" @@ -223,14 +226,14 @@ def _upload_file( return filenames -def upload_file( +def upload_file( # noqa: PLR0913 content: bytes, filename: str, *, - content_type: Optional[str] = None, - change_notification: Optional[str] = None, + content_type: str | None = None, + change_notification: str | None = None, compare_fn: Callable[[bytes, bytes], bool] = simple_compare, - acl: Optional[str] = "public-read", + acl: str | None = "public-read", create_cloudfront_invalidation: bool = False, ): """Upload a file to storage. @@ -291,13 +294,13 @@ def upload_file( sentry_sdk.capture_message(change_notification) -def upload_dataframe( +def upload_dataframe( # noqa: PLR0913 df: pd.DataFrame, filename: str, *, - change_notification: Optional[str] = None, + change_notification: str | None = None, compare_fn: Callable[[bytes, bytes], bool] = simple_compare, - acl: Optional[str] = "public-read", + acl: str | None = "public-read", create_cloudfront_invalidation: bool = False, datawrapper_datetimes: bool = False, ): @@ -352,7 +355,7 @@ def _queue_cloudfront_invalidation(filename: str) -> Any: CLOUDFRONT_INVALIDATIONS_TO_CREATE.append(filename) -def run_cloudfront_invalidations(*, caller_reference: Optional[str] = None): +def run_cloudfront_invalidations(*, caller_reference: str | None = None): """Run CloudFront invalidations""" caller_reference = caller_reference or str(uuid4()) @@ -363,7 +366,8 @@ def run_cloudfront_invalidations(*, caller_reference: Optional[str] = None): invalidation_path = common_prefix(CLOUDFRONT_INVALIDATIONS_TO_CREATE) + "*" if "/" not in invalidation_path: - raise Exception("CloudFront invalidation path is too broad:", invalidation_path) + msg = f"CloudFront invalidation path is too broad: {invalidation_path}" + raise Exception(msg) if not USE_LOCAL_STORAGE and cloudfront: STORAGE_EVENTS.append({"type": "invalidation", "path": invalidation_path}) diff --git a/pyproject.toml b/pyproject.toml index 28c2070..86b24a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,43 @@ -[tool.black] + +[tool.ruff] +target-version = "py311" +src = ["ddj_cloud", "scraper_template"] +indent-width = 4 line-length = 100 -target-version = ['py39'] +exclude = [] + +[tool.ruff.lint] +extend-select = [ + "ARG", # unused-arguments + "B", # bugbear + "E", # pycodestyle + "EM", # errmsg + "F", # flake8 + "I", # import order + "PYI", # pyi + "SIM", # simplify + # "T20", # print + "TCH", # type-checking + "TID", # tidy-imports + "UP", # pyupgrade + "FURB", # refurb + "W", # pycodestyle + # "D", # pydocstyle + "PL", # Pylint +] +ignore = [ + "E501", # line too long +] + + +[tool.pyright] +pythonVersion = "3.11" +include = ["ddj_cloud", "scraper_template"] +exclude = [] +typeCheckingMode = "standard" +reportUnnecessaryTypeIgnoreComment = "warning" +reportUnusedImport = "none" +# reportUnknownMemberType = "warning" +# reportUnknownVariableType = "warning" +# reportUnknownArgumentType = "warning" +# reportUnknownFunctionType = "warning"