diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8848656fd..68ea04dfd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ repos: # Normalise all Python code. (Black + isort + pyupgrade + autoflake) - repo: https://github.com/Zac-HD/shed - rev: 2023.6.1 + rev: 2024.1.1 hooks: - id: shed # Python Linting diff --git a/cubedash/_filters.py b/cubedash/_filters.py index b41ba18d9..e4209312b 100644 --- a/cubedash/_filters.py +++ b/cubedash/_filters.py @@ -2,7 +2,6 @@ Common global filters for templates. """ - import calendar import logging from datetime import datetime diff --git a/cubedash/_model.py b/cubedash/_model.py index 998e39ddc..9973d3f3b 100644 --- a/cubedash/_model.py +++ b/cubedash/_model.py @@ -38,9 +38,11 @@ if os.getenv("SENTRY_DSN"): sentry_sdk.init( dsn=os.getenv("SENTRY_DSN"), - environment=os.getenv("SENTRY_ENV_TAG") - if os.getenv("SENTRY_ENV_TAG") - else "dev-explorer", + environment=( + os.getenv("SENTRY_ENV_TAG") + if os.getenv("SENTRY_ENV_TAG") + else "dev-explorer" + ), integrations=[ FlaskIntegration(), ], diff --git a/cubedash/_pages.py b/cubedash/_pages.py index 9dd0a1eff..83dfbc252 100644 --- a/cubedash/_pages.py +++ b/cubedash/_pages.py @@ -116,9 +116,11 @@ def product_page( datasets_geojson=None, # _model.get_datasets_geojson(product_name, year, month, day), footprint_geojson=_model.get_footprint_geojson(product_name, year, month, day), product=product, - product_region_info=_model.STORE.get_product_region_info(product_name) - if region_geojson - else None, + product_region_info=( + _model.STORE.get_product_region_info(product_name) + if region_geojson + else None + ), # Summary for the whole product product_summary=product_summary, # Summary for the users' currently selected filters. @@ -425,9 +427,7 @@ def timeline_page(product_name: str): return redirect(url_for("product_page", product_name=product_name)) -def _load_product( - product_name, year, month, day -) -> Tuple[ +def _load_product(product_name, year, month, day) -> Tuple[ DatasetType, ProductSummary, TimePeriodOverview, diff --git a/cubedash/_stac.py b/cubedash/_stac.py index cad5cd493..9e17f194b 100644 --- a/cubedash/_stac.py +++ b/cubedash/_stac.py @@ -196,18 +196,22 @@ def as_stac_item(dataset: DatasetItem) -> pystac.Item: "odc:processing_datetime": utc(dataset.creation_time), } ), - measurements={ - name: _band_to_measurement( - b, - dataset_location=ds.uris[0] if ds is not None and ds.uris else None, - ) - for name, b in ds.measurements.items() - } - if ds is not None - else {}, - accessories=_accessories_from_eo1(ds.metadata_doc) - if ds is not None - else {}, + measurements=( + { + name: _band_to_measurement( + b, + dataset_location=( + ds.uris[0] if ds is not None and ds.uris else None + ), + ) + for name, b in ds.measurements.items() + } + if ds is not None + else {} + ), + accessories=( + _accessories_from_eo1(ds.metadata_doc) if ds is not None else {} + ), # TODO: Fill in lineage. The datacube API only gives us full datasets, which is # expensive. We only need a list of IDs here. lineage={}, diff --git a/cubedash/_stac_legacy.py b/cubedash/_stac_legacy.py index 7cb997185..7fb31d890 100644 --- a/cubedash/_stac_legacy.py +++ b/cubedash/_stac_legacy.py @@ -1,6 +1,7 @@ """ Legacy redirects to maintain old stac api URLs """ + import json import flask diff --git a/cubedash/_utils.py b/cubedash/_utils.py index 5ceba6e8f..249c7c366 100644 --- a/cubedash/_utils.py +++ b/cubedash/_utils.py @@ -726,9 +726,9 @@ def get_property_priority(ordered_properties: List, keyval): for type_, source_dataset_doc in ordered_metadata["lineage"][ "source_datasets" ].items(): - ordered_metadata["lineage"]["source_datasets"][ - type_ - ] = prepare_document_formatting(source_dataset_doc) + ordered_metadata["lineage"]["source_datasets"][type_] = ( + prepare_document_formatting(source_dataset_doc) + ) # Products have an embedded metadata doc (subset of dataset metadata) if "metadata" in ordered_metadata: diff --git a/cubedash/gunicorn_config.py b/cubedash/gunicorn_config.py index f8a6eec98..249a78ad9 100644 --- a/cubedash/gunicorn_config.py +++ b/cubedash/gunicorn_config.py @@ -1,5 +1,6 @@ """Gunicorn config for Prometheus internal metrics """ + import os diff --git a/cubedash/logs.py b/cubedash/logs.py index c5ea1956e..966f1e8c7 100644 --- a/cubedash/logs.py +++ b/cubedash/logs.py @@ -47,9 +47,11 @@ def lenient_json_dump(obj, *args, **kwargs): structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, # Coloured output if to terminal, otherwise json - BytesConsoleRenderer() - if not write_as_json - else structlog.processors.JSONRenderer(serializer=lenient_json_dump), + ( + BytesConsoleRenderer() + if not write_as_json + else structlog.processors.JSONRenderer(serializer=lenient_json_dump) + ), ] hide_logging_levels = { diff --git a/cubedash/run.py b/cubedash/run.py index 0dc7ccc07..60a21e2b6 100644 --- a/cubedash/run.py +++ b/cubedash/run.py @@ -7,6 +7,7 @@ as DATACUBE_ENVIRONMENT) """ + from textwrap import dedent import click diff --git a/cubedash/summary/_stores.py b/cubedash/summary/_stores.py index 4c03d8dce..c44e47c96 100644 --- a/cubedash/summary/_stores.py +++ b/cubedash/summary/_stores.py @@ -1347,9 +1347,11 @@ def search_items( dataset_id=r.id, bbox=_box2d_to_bbox(r.bbox) if r.bbox else None, product_name=self.index.products.get(r.dataset_type_ref).name, - geometry=_get_shape(r.geometry, self._get_srid_name(r.geometry.srid)) - if r.geometry is not None - else None, + geometry=( + _get_shape(r.geometry, self._get_srid_name(r.geometry.srid)) + if r.geometry is not None + else None + ), region_code=r.region_code, creation_time=r.creation_time, center_time=r.center_time, @@ -1814,16 +1816,22 @@ def _summary_from_row(res, product_name, grouping_timezone=default_timezone): region_dataset_counts=region_dataset_counts, timeline_period=res["timeline_period"], # : Range - time_range=Range( - res["time_earliest"].astimezone(grouping_timezone) + time_range=( + Range( + ( + res["time_earliest"].astimezone(grouping_timezone) + if res["time_earliest"] + else res["time_earliest"] + ), + ( + res["time_latest"].astimezone(grouping_timezone) + if res["time_latest"] + else res["time_latest"] + ), + ) if res["time_earliest"] - else res["time_earliest"], - res["time_latest"].astimezone(grouping_timezone) - if res["time_latest"] - else res["time_latest"], - ) - if res["time_earliest"] - else None, + else None + ), # shapely.geometry.base.BaseGeometry footprint_geometry=( None diff --git a/cubedash/summary/show.py b/cubedash/summary/show.py index d0856c334..0ec791230 100644 --- a/cubedash/summary/show.py +++ b/cubedash/summary/show.py @@ -5,6 +5,7 @@ Useful for testing Explorer-generated summaries from scripts and the command-line. """ + import sys import time from textwrap import dedent diff --git a/integration_tests/dumpdatasets.py b/integration_tests/dumpdatasets.py index 67173436c..8cd54741b 100644 --- a/integration_tests/dumpdatasets.py +++ b/integration_tests/dumpdatasets.py @@ -1,6 +1,7 @@ """ Util script to dump datasets from a datacube for use as test data. """ + import gzip import random from datetime import datetime diff --git a/integration_tests/test_arb_crs.py b/integration_tests/test_arb_crs.py index 7604563eb..ef1cec945 100644 --- a/integration_tests/test_arb_crs.py +++ b/integration_tests/test_arb_crs.py @@ -2,6 +2,7 @@ Test utility method for creating valid EPSG code based CRS from possible WKT String """ + from pyproj import CRS from cubedash._utils import infer_crs diff --git a/integration_tests/test_dataset_maturity.py b/integration_tests/test_dataset_maturity.py index 10b37db57..4bd4dd2ce 100644 --- a/integration_tests/test_dataset_maturity.py +++ b/integration_tests/test_dataset_maturity.py @@ -3,6 +3,7 @@ - 4 datasets have maturity level: interim - 16 datasets have maturity level: final """ + from pathlib import Path import pytest diff --git a/integration_tests/test_filter_geom.py b/integration_tests/test_filter_geom.py index fc47afb49..5c38555f7 100644 --- a/integration_tests/test_filter_geom.py +++ b/integration_tests/test_filter_geom.py @@ -1,6 +1,7 @@ """ Unit test for re-cursive geometry filtering """ + import json from pathlib import Path diff --git a/integration_tests/test_page_loads.py b/integration_tests/test_page_loads.py index a09fb7844..c4f4ebe3d 100644 --- a/integration_tests/test_page_loads.py +++ b/integration_tests/test_page_loads.py @@ -1,6 +1,7 @@ """ Tests that load pages and check the contained text. """ + import json from datetime import datetime from io import StringIO diff --git a/integration_tests/test_raw_yaml.py b/integration_tests/test_raw_yaml.py index c044695f8..6d1ad5461 100644 --- a/integration_tests/test_raw_yaml.py +++ b/integration_tests/test_raw_yaml.py @@ -4,6 +4,7 @@ - odc-product.yaml (cli command: datacube product) - odc-metadata.yaml (cli command: datacube dataset) """ + import tempfile import datacube.scripts.cli_app diff --git a/integration_tests/test_resolved_uri.py b/integration_tests/test_resolved_uri.py index 7ad6448a8..090df2312 100644 --- a/integration_tests/test_resolved_uri.py +++ b/integration_tests/test_resolved_uri.py @@ -1,6 +1,7 @@ """ Unit test for various app.config["CUBEDASH_DATA_S3_REGION"] """ + import pytest from flask import Flask, current_app diff --git a/integration_tests/test_s2_l2a_footprint.py b/integration_tests/test_s2_l2a_footprint.py index 05ce9dbca..b403cfbf4 100644 --- a/integration_tests/test_s2_l2a_footprint.py +++ b/integration_tests/test_s2_l2a_footprint.py @@ -1,6 +1,7 @@ """ Tests that load pages and check the contained text. """ + from datetime import datetime from pathlib import Path diff --git a/integration_tests/test_summarise_data.py b/integration_tests/test_summarise_data.py index dfaa863c0..8b78898b5 100644 --- a/integration_tests/test_summarise_data.py +++ b/integration_tests/test_summarise_data.py @@ -3,6 +3,7 @@ And then check their statistics match expected. """ + from datetime import datetime, timedelta from uuid import UUID diff --git a/integration_tests/test_utc_tst.py b/integration_tests/test_utc_tst.py index 97da17136..7de1da6f3 100644 --- a/integration_tests/test_utc_tst.py +++ b/integration_tests/test_utc_tst.py @@ -1,6 +1,7 @@ """ Tests that load pages and check the contained text. """ + from datetime import datetime from pathlib import Path diff --git a/integration_tests/test_wagl_data.py b/integration_tests/test_wagl_data.py index bccf66c61..a064b298a 100644 --- a/integration_tests/test_wagl_data.py +++ b/integration_tests/test_wagl_data.py @@ -1,6 +1,7 @@ """ Tests that load pages and check the contained text. """ + from datetime import datetime import pytest