diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 306f35180..c59b9dc0f 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -46,9 +46,9 @@ line-length = 120 disable = ["missing-docstring", "line-too-long", "logging-fstring-interpolation", "duplicate-code"] extension-pkg-whitelist = "pydantic" ignored-modules = "pyarrow.compute" -good-names = ["i", "x", "df"] +good-names = ["i", "x", "df", "xy"] min-public-methods = 0 -max-args = 7 +max-args = 9 max-locals = 19 max-branches = 14 diff --git a/backend/src/backend/__init__.py b/backend/src/backend/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/src/backend/primary/routers/correlations/router.py b/backend/src/backend/primary/routers/correlations/router.py index 6d7404311..f474495b2 100644 --- a/backend/src/backend/primary/routers/correlations/router.py +++ b/backend/src/backend/primary/routers/correlations/router.py @@ -1,17 +1,6 @@ -import datetime import logging -from typing import List - -from fastapi import APIRouter, Depends, Query - -from src.services.sumo_access.parameter_access import ParameterAccess -from src.services.sumo_access.generic_types import EnsembleCorrelations -from src.services.parameter_correlations import correlate_parameters_with_response -from src.services.sumo_access.summary_access import SummaryAccess -from src.services.sumo_access.inplace_volumetrics_access import InplaceVolumetricsAccess -from src.services.utils.authenticated_user import AuthenticatedUser -from src.backend.auth.auth_helper import AuthHelper +from fastapi import APIRouter LOGGER = logging.getLogger(__name__) diff --git a/backend/src/backend/primary/routers/general.py b/backend/src/backend/primary/routers/general.py index 29e9c7f9f..5ec0687c5 100644 --- a/backend/src/backend/primary/routers/general.py +++ b/backend/src/backend/primary/routers/general.py @@ -41,7 +41,7 @@ def alive_protected() -> str: @router.get("/logged_in_user", response_model=UserInfo) async def logged_in_user( request: Request, - includeGraphApiInfo: bool = Query( + includeGraphApiInfo: bool = Query( # pylint: disable=invalid-name False, description="Set to true to include user avatar and display name from Microsoft Graph API" ), ) -> UserInfo: @@ -75,10 +75,10 @@ async def logged_in_user( user_info.avatar_b64str = avatar_b64str if graph_user_info is not None: user_info.display_name = graph_user_info.get("displayName", None) - except httpx.HTTPError as e: - print("Error while fetching user avatar and info from Microsoft Graph API (HTTP error):\n", e) - except httpx.InvalidURL as e: - print("Error while fetching user avatar and info from Microsoft Graph API (Invalid URL):\n", e) + except httpx.HTTPError as exc: + print("Error while fetching user avatar and info from Microsoft Graph API (HTTP error):\n", exc) + except httpx.InvalidURL as exc: + print("Error while fetching user avatar and info from Microsoft Graph API (Invalid URL):\n", exc) return user_info diff --git a/backend/src/backend/primary/routers/grid/router.py b/backend/src/backend/primary/routers/grid/router.py index 8b658d34a..456cc25a1 100644 --- a/backend/src/backend/primary/routers/grid/router.py +++ b/backend/src/backend/primary/routers/grid/router.py @@ -1,8 +1,8 @@ -from fastapi import APIRouter, Depends +from typing import List + +from fastapi import APIRouter, Depends, Query from starlette.requests import Request -from typing import Any, List -from fastapi import APIRouter, Depends, HTTPException, Query, Response from src.services.utils.authenticated_user import AuthenticatedUser from src.backend.auth.auth_helper import AuthHelper from src.backend.primary.user_session_proxy import proxy_to_user_session @@ -68,7 +68,7 @@ async def grid_surface( "query_string": request.url.include_query_params(**query_params).query.encode("utf-8"), "headers": request.headers.raw, }, - receive=request._receive, + receive=request._receive, # pylint: disable=protected-access ) response = await proxy_to_user_session(updated_request, authenticated_user) @@ -104,7 +104,7 @@ async def grid_parameter( "query_string": request.url.include_query_params(**query_params).query.encode("utf-8"), "headers": request.headers.raw, }, - receive=request._receive, + receive=request._receive, # pylint: disable=protected-access ) response = await proxy_to_user_session(updated_request, authenticated_user) @@ -140,7 +140,7 @@ async def grid_parameter_intersection( "query_string": request.url.include_query_params(**query_params).query.encode("utf-8"), "headers": request.headers.raw, }, - receive=request._receive, + receive=request._receive, # pylint: disable=protected-access ) response = await proxy_to_user_session(updated_request, authenticated_user) @@ -176,7 +176,7 @@ async def statistical_grid_parameter_intersection( "query_string": request.url.include_query_params(**query_params).query.encode("utf-8"), "headers": request.headers.raw, }, - receive=request._receive, + receive=request._receive, # pylint: disable=protected-access ) response = await proxy_to_user_session(updated_request, authenticated_user) @@ -211,7 +211,7 @@ async def statistical_grid_parameter( "query_string": request.url.include_query_params(**query_params).query.encode("utf-8"), "headers": request.headers.raw, }, - receive=request._receive, + receive=request._receive, # pylint: disable=protected-access ) response = await proxy_to_user_session(updated_request, authenticated_user) diff --git a/backend/src/backend/primary/routers/pvt/converters.py b/backend/src/backend/primary/routers/pvt/converters.py index 6206763ed..160cb9e6a 100644 --- a/backend/src/backend/primary/routers/pvt/converters.py +++ b/backend/src/backend/primary/routers/pvt/converters.py @@ -24,7 +24,7 @@ class PHASES(str, Enum): def pvt_dataframe_to_api_data(data_frame: pd.DataFrame) -> List[PvtData]: """Converts the PVT table from Sumo/Ecl2Df to a list of PvtData objects""" - """Dataframe manipulation is copied from webviz-subsurface""" + # Dataframe manipulation is copied from webviz-subsurface data_frame = data_frame.rename(str.upper, axis="columns").rename( columns={ @@ -51,13 +51,13 @@ def pvt_dataframe_to_api_data(data_frame: pd.DataFrame) -> List[PvtData]: list_of_pvtdata: List[PvtData] = [] for keyword, df_grouped_on_keyword in data_frame.groupby("KEYWORD"): - if keyword in OIL_KEYWORDS.keys(): + if keyword in OIL_KEYWORDS: phase = PHASES.OIL.value name = OIL_KEYWORDS[keyword] - elif keyword in GAS_KEYWORDS.keys(): + elif keyword in GAS_KEYWORDS: phase = PHASES.GAS.value name = GAS_KEYWORDS[keyword] - elif keyword in WATER_KEYWORDS.keys(): + elif keyword in WATER_KEYWORDS: phase = PHASES.WATER.value name = WATER_KEYWORDS[keyword] else: diff --git a/backend/src/backend/primary/routers/pvt/router.py b/backend/src/backend/primary/routers/pvt/router.py index 324c7cdbf..d68ea835b 100644 --- a/backend/src/backend/primary/routers/pvt/router.py +++ b/backend/src/backend/primary/routers/pvt/router.py @@ -1,8 +1,7 @@ import logging -from typing import List, Optional, Sequence, Union +from typing import List from fastapi import APIRouter, Depends, HTTPException, Query -from fastapi.responses import ORJSONResponse from src.backend.auth.auth_helper import AuthHelper from src.services.sumo_access.table_access import TableAccess diff --git a/backend/src/backend/primary/routers/surface/converters.py b/backend/src/backend/primary/routers/surface/converters.py index 6e813634a..591c15871 100644 --- a/backend/src/backend/primary/routers/surface/converters.py +++ b/backend/src/backend/primary/routers/surface/converters.py @@ -38,5 +38,5 @@ def to_api_surface_data(xtgeo_surf: xtgeo.RegularSurface) -> schemas.SurfaceData val_min=xtgeo_surf.values.min(), val_max=xtgeo_surf.values.max(), rot_deg=xtgeo_surf.rotation, - mesh_data=orjson.dumps(float32values).decode(), + mesh_data=orjson.dumps(float32values).decode(), # pylint: disable=maybe-no-member ) diff --git a/backend/src/backend/primary/routers/well/router.py b/backend/src/backend/primary/routers/well/router.py index c9601a5b1..a9ede000a 100644 --- a/backend/src/backend/primary/routers/well/router.py +++ b/backend/src/backend/primary/routers/well/router.py @@ -1,15 +1,11 @@ -import datetime import logging -from typing import List, Optional, Sequence, Union +from typing import List, Union -import pyarrow as pa -import pyarrow.compute as pc -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Depends, Query from src.services.smda_access import mocked_drogon_smda_access from src.services.smda_access.well_access import WellAccess from src.services.sumo_access.case_inspector import CaseInspector -from src.services.smda_access.stratigraphy_access import StratigraphyAccess from src.services.utils.authenticated_user import AuthenticatedUser from src.backend.auth.auth_helper import AuthHelper diff --git a/backend/src/backend/primary/user_session_proxy.py b/backend/src/backend/primary/user_session_proxy.py index d0a00b654..e18eef06a 100644 --- a/backend/src/backend/primary/user_session_proxy.py +++ b/backend/src/backend/primary/user_session_proxy.py @@ -20,7 +20,7 @@ def __init__(self, name: str, port: int) -> None: self._name = name self._port = port - # TODO: This should be moved to Redis + # This should be moved to Redis - https://github.com/equinor/webviz/issues/357 # key: user_id, value: name of Radix job instance self._existing_job_names: Dict[str, str] = {} @@ -30,13 +30,13 @@ async def _active_running_job(self, user_id: str) -> bool: existing_job_name = self._existing_job_names.get(user_id) if not existing_job_name: return False - elif LOCALHOST_DEVELOPMENT: + if LOCALHOST_DEVELOPMENT: return True async with httpx.AsyncClient() as client: - r = await client.get(f"http://{self._name}:{self._port}/api/v1/jobs/{existing_job_name}") + res = await client.get(f"http://{self._name}:{self._port}/api/v1/jobs/{existing_job_name}") - job = r.json() + job = res.json() if job.get("status") != "Running" or not job.get("started"): return False @@ -58,7 +58,7 @@ async def _create_new_job(self, user_id: str) -> None: self._existing_job_names[user_id] = self._name else: async with httpx.AsyncClient() as client: - r = await client.post( + res = await client.post( f"http://{self._name}:{self._port}/api/v1/jobs", # Maximum limits in "resources" for a Radix job is as of May 2023 # the specs of a single Standard_E16as_v4 node, i.e.: @@ -75,7 +75,7 @@ async def _create_new_job(self, user_id: str) -> None: } }, ) - self._existing_job_names[user_id] = r.json()["name"] + self._existing_job_names[user_id] = res.json()["name"] while not await self._active_running_job(user_id): # It takes a couple of seconds before Radix job uvicorn process has @@ -101,7 +101,9 @@ async def proxy_to_user_session(request: Request, authenticated_user: Authentica # Ideally this function should probably be a starlette/FastAPI middleware, but it appears that # it is not yet possible to put middleware on single routes through decorator like in express.js. - base_url = await RADIX_JOB_SCHEDULER_INSTANCE.get_base_url(authenticated_user._user_id) + base_url = await RADIX_JOB_SCHEDULER_INSTANCE.get_base_url( + authenticated_user._user_id # pylint: disable=protected-access + ) # See https://github.com/tiangolo/fastapi/discussions/7382: diff --git a/backend/src/backend/user_session/routers/grid/router.py b/backend/src/backend/user_session/routers/grid/router.py index 1313465eb..13e2b775d 100644 --- a/backend/src/backend/user_session/routers/grid/router.py +++ b/backend/src/backend/user_session/routers/grid/router.py @@ -3,17 +3,19 @@ from functools import cache from typing import List, Tuple import logging -import os, psutil +import os +from concurrent.futures import ThreadPoolExecutor +import psutil import numpy as np import orjson import xtgeo from vtkmodules.util.numpy_support import vtk_to_numpy -from fastapi import APIRouter, Depends, Request, Response +from fastapi import APIRouter, Depends, Request from fastapi.responses import ORJSONResponse + from src.backend.auth.auth_helper import AuthenticatedUser, AuthHelper from src.backend.primary.routers.grid.schemas import ( - B64EncodedNumpyArray, GridSurface, GridIntersection, ) @@ -42,7 +44,6 @@ async def grid_surface( request: Request, authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user), ): - """ """ case_uuid = request.query_params.get("case_uuid") ensemble_name = request.query_params.get("ensemble_name") grid_name = request.query_params.get("grid_name") @@ -61,11 +62,11 @@ async def grid_surface( grid_geometrics = xtgeo_grid.get_geometrics(allcells=True, return_dict=True) # Get grid surface (visible cells) - grid_surface = get_grid_surface(grid_geometry=xtgeo_grid) + grid_surface_instance = get_grid_surface(grid_geometry=xtgeo_grid) # Extract points and polygons from surface - points_np = vtk_to_numpy(grid_surface.polydata.GetPoints().GetData()).ravel().astype(np.float32) - polys_np = vtk_to_numpy(grid_surface.polydata.GetPolys().GetData()).astype(np.int64) + points_np = vtk_to_numpy(grid_surface_instance.polydata.GetPoints().GetData()).ravel().astype(np.float32) + polys_np = vtk_to_numpy(grid_surface_instance.polydata.GetPolys().GetData()).astype(np.int64) # Reduce precision of points to 2 decimals points_np = np.around(points_np, decimals=2) @@ -86,7 +87,6 @@ async def grid_parameter( request: Request, authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user), ): - """ """ case_uuid = request.query_params.get("case_uuid") ensemble_name = request.query_params.get("ensemble_name") grid_name = request.query_params.get("grid_name") @@ -129,11 +129,10 @@ async def grid_parameter( "/grid_parameter_intersection", response_model=List[float] ) # stating response_model here instead of return type apparently disables pydantic validation of the response (https://stackoverflow.com/a/65715205) # type: ignore -async def grid_parameter( +async def grid_parameter_intersection( # pylint: disable=too-many-locals request: Request, authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user), ): - """ """ case_uuid = request.query_params.get("case_uuid") ensemble_name = request.query_params.get("ensemble_name") grid_name = request.query_params.get("grid_name") @@ -230,7 +229,7 @@ async def grid_parameter( # Create the intersection data object intersection_data = GridIntersection( - image="data:image/png;base64,{}".format(image_data), + image=f"data:image/png;base64,{image_data}", polyline_x=polyline_x.tolist(), polyline_y=polyline_y.tolist(), x_min=float(x_min), @@ -245,11 +244,10 @@ async def grid_parameter( "/statistical_grid_parameter_intersection", response_model=List[float] ) # stating response_model here instead of return type apparently disables pydantic validation of the response (https://stackoverflow.com/a/65715205) # type: ignore -async def grid_parameter( +async def statistical_grid_parameter_intersection( # pylint: disable=too-many-locals request: Request, authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user), ): - """ """ timer = PerfTimer() print("#" * 80, flush=True) print("ENTERING STATISTICAL GRID PARAMETER INTERSECTION", flush=True) @@ -264,7 +262,7 @@ async def grid_parameter( grid_name = request.query_params.get("grid_name") parameter_name = request.query_params.get("parameter_name") # convert json string of realizations into list - realizations = orjson.loads(request.query_params.get("realizations")) + realizations = orjson.loads(request.query_params.get("realizations")) # pylint: disable=maybe-no-member grid_access = GridAccess(authenticated_user.get_sumo_access_token(), case_uuid, ensemble_name) @@ -293,8 +291,6 @@ async def grid_parameter( print("GETTING GRID PARAMETERS", flush=True) ### Using ThreadPoolExecutor to parallelize the download of the grid parameters - from concurrent.futures import ThreadPoolExecutor - def worker(real): return get_grid_parameter( authenticated_user=authenticated_user, @@ -342,7 +338,7 @@ def worker(real): ] # Calculate the mean scalar value for each cell - values = np.nanmean([scalar_values for scalar_values in all_scalar_values], axis=0) + values = np.nanmean(all_scalar_values, axis=0) # Handle xtgeo undefined values and truncate values[values < np.nanmin(values)] = np.nanmin(values) @@ -382,7 +378,7 @@ def worker(real): # Create the intersection data object intersection_data = GridIntersection( - image="data:image/png;base64,{}".format(image_data), + image=f"data:image/png;base64,{image_data}", polyline_x=polyline_x.tolist(), polyline_y=polyline_y.tolist(), x_min=float(x_min), @@ -410,13 +406,12 @@ async def statistical_grid_parameter( request: Request, authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user), ): - """ """ case_uuid = request.query_params.get("case_uuid") ensemble_name = request.query_params.get("ensemble_name") grid_name = request.query_params.get("grid_name") parameter_name = request.query_params.get("parameter_name") # convert json string of realizations into list - realizations = orjson.loads(request.query_params.get("realizations")) + realizations = orjson.loads(request.query_params.get("realizations")) # pylint: disable=maybe-no-member grid_access = GridAccess(authenticated_user.get_sumo_access_token(), case_uuid, ensemble_name) @@ -456,7 +451,7 @@ async def statistical_grid_parameter( ] # Calculate the mean scalar values for each cell - mean_scalar_values = np.nanmean([scalar_values for scalar_values in all_scalar_values], axis=0) + mean_scalar_values = np.nanmean(all_scalar_values, axis=0) # Handle xtgeo undefined values and truncate mean_scalar_values[mean_scalar_values == -999.0] = np.nan @@ -484,9 +479,7 @@ def get_grid_geometry( @cache def get_grid_surface(grid_geometry: xtgeo.Grid) -> VtkGridSurface: - grid_surface = get_surface(grid_geometry) - - return grid_surface + return get_surface(grid_geometry) @cache @@ -501,9 +494,7 @@ def get_grid_parameter( token = authenticated_user.get_sumo_access_token() grid_access = GridAccess(token, case_uuid, ensemble_name) - grid_parameter = grid_access.get_grid_parameter(grid_name, parameter_name, int(realization)) - - return grid_parameter + return grid_access.get_grid_parameter(grid_name, parameter_name, int(realization)) @cache