From 0880f5ad4c0a88d46a239209fb17c6be37012ec7 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 15:04:52 +0100 Subject: [PATCH 01/15] get rid of generated module version in NF workflow scripts --- .../operandi_utils/hpc/nextflow_workflows/default_workflow.nf | 2 +- .../hpc/nextflow_workflows/default_workflow_with_MS.nf | 2 +- .../operandi_utils/hpc/nextflow_workflows/odem_workflow.nf | 2 +- .../hpc/nextflow_workflows/odem_workflow_with_MS.nf | 2 +- .../operandi_utils/hpc/nextflow_workflows/sbb_workflow.nf | 2 +- .../hpc/nextflow_workflows/sbb_workflow_with_MS.nf | 2 +- .../operandi_utils/hpc/nextflow_workflows/template_workflow.nf | 2 +- .../hpc/nextflow_workflows/template_workflow_with_MS.nf | 2 +- src/utils/operandi_utils/oton/constants.py | 3 +-- 9 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow.nf index 4e9c069..464b2c6 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow_with_MS.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow_with_MS.nf index ba9bf52..6be5d92 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow_with_MS.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow.nf index a40c8b3..a22b221 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow_with_MS.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow_with_MS.nf index 7fc2a55..4b7b94d 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow_with_MS.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow.nf index 559c6b8..0dcc63a 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow_with_MS.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow_with_MS.nf index b6e1250..840fae0 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow_with_MS.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow.nf index 537ac15..859958d 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow_with_MS.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow_with_MS.nf index 0d00642..ddbe731 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow_with_MS.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/oton/constants.py b/src/utils/operandi_utils/oton/constants.py index 180fc94..9376f84 100644 --- a/src/utils/operandi_utils/oton/constants.py +++ b/src/utils/operandi_utils/oton/constants.py @@ -1,7 +1,6 @@ from json import load from os import environ from pkg_resources import resource_filename -from operandi_utils.constants import OPERANDI_VERSION BS: str = '{}' SPACES = ' ' @@ -32,4 +31,4 @@ PARAMS_KEY_CPUS_PER_FORK: str = 'params.cpus_per_fork' PARAMS_KEY_RAM_PER_FORK: str = 'params.ram_per_fork' -WORKFLOW_COMMENT = f"// This workflow was automatically generated by the v{OPERANDI_VERSION} operandi_utils.oton module" +WORKFLOW_COMMENT = f"// This workflow was automatically generated by the operandi_utils.oton module" From 1709f307af586ad9ff665b3ecaffeba22e639bc4 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 15:35:31 +0100 Subject: [PATCH 02/15] refactor: user auth invokation --- .../operandi_server/routers/admin_panel.py | 5 +-- .../operandi_server/routers/discovery.py | 9 ++-- src/server/operandi_server/routers/user.py | 29 ++++-------- .../operandi_server/routers/user_utils.py | 45 +++++++++++++------ .../operandi_server/routers/workflow.py | 21 +++++---- .../operandi_server/routers/workspace.py | 17 ++++--- 6 files changed, 63 insertions(+), 63 deletions(-) diff --git a/src/server/operandi_server/routers/admin_panel.py b/src/server/operandi_server/routers/admin_panel.py index 1539fc4..0a21a4f 100644 --- a/src/server/operandi_server/routers/admin_panel.py +++ b/src/server/operandi_server/routers/admin_panel.py @@ -11,13 +11,12 @@ db_get_workflow, db_get_workspace, db_get_all_workspaces_by_user, db_get_all_workflows_by_user ) from operandi_utils.utils import send_bag_to_ola_hd -from .user import RouterUser +from .user_utils import user_auth_with_handling from .workspace_utils import create_workspace_bag, get_db_workspace_with_handling, validate_bag_with_handling class RouterAdminPanel: def __init__(self): self.logger = getLogger("operandi_server.routers.user") - self.user_authenticator = RouterUser() self.router = APIRouter(tags=[ServerApiTag.ADMIN]) self.router.add_api_route( path="/admin/users", @@ -51,7 +50,7 @@ def __init__(self): ) async def auth_admin_with_handling(self, auth: HTTPBasicCredentials): - py_user_action = await self.user_authenticator.user_login(auth) + py_user_action = await user_auth_with_handling(self.logger, auth) if py_user_action.account_type != AccountType.ADMIN: message = f"Admin privileges required for the endpoint" self.logger.error(f"{message}") diff --git a/src/server/operandi_server/routers/discovery.py b/src/server/operandi_server/routers/discovery.py index a2f8de7..ef674b0 100644 --- a/src/server/operandi_server/routers/discovery.py +++ b/src/server/operandi_server/routers/discovery.py @@ -12,13 +12,12 @@ from operandi_utils.constants import ServerApiTag from operandi_utils.oton.constants import OCRD_ALL_JSON from operandi_server.models import PYDiscovery -from .user import RouterUser +from .user_utils import user_auth_with_handling class RouterDiscovery: def __init__(self): self.logger = getLogger("operandi_server.routers.discovery") - self.user_authenticator = RouterUser() self.router = APIRouter(tags=[ServerApiTag.DISCOVERY]) self.router.add_api_route( @@ -39,7 +38,7 @@ def __init__(self): ) async def discovery(self, auth: HTTPBasicCredentials = Depends(HTTPBasic())) -> PYDiscovery: - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) response = PYDiscovery( ram=virtual_memory().total / (1024.0 ** 3), cpu_cores=cpu_count(), @@ -52,7 +51,7 @@ async def discovery(self, auth: HTTPBasicCredentials = Depends(HTTPBasic())) -> return response async def get_processor_names(self, auth: HTTPBasicCredentials = Depends(HTTPBasic())) -> List[str]: - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) try: processor_names = list(OCRD_ALL_JSON.keys()) return processor_names @@ -65,7 +64,7 @@ async def get_processor_names(self, auth: HTTPBasicCredentials = Depends(HTTPBas raise HTTPException(status_code=500, detail="An unexpected error occurred while loading processor names.") async def get_processor_info(self, processor_name: str, auth: HTTPBasicCredentials = Depends(HTTPBasic())) -> Dict: - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) try: if processor_name not in OCRD_ALL_JSON: raise HTTPException(status_code=404, detail=f"Processor '{processor_name}' not found.") diff --git a/src/server/operandi_server/routers/user.py b/src/server/operandi_server/routers/user.py index 600a1e0..1c4f8b9 100644 --- a/src/server/operandi_server/routers/user.py +++ b/src/server/operandi_server/routers/user.py @@ -1,7 +1,7 @@ from logging import getLogger from typing import List, Optional from datetime import datetime -from fastapi import APIRouter, Depends, HTTPException, status +from fastapi import APIRouter, Depends, status from fastapi.security import HTTPBasic, HTTPBasicCredentials from operandi_utils.constants import AccountType, ServerApiTag @@ -9,10 +9,9 @@ db_get_processing_stats, db_get_all_workflow_jobs_by_user, db_get_user_account_with_email, db_get_workflow, db_get_workspace, db_get_all_workspaces_by_user, db_get_all_workflows_by_user ) -from operandi_server.exceptions import AuthenticationError from operandi_server.models import PYUserAction, WorkflowJobRsrc, WorkspaceRsrc, WorkflowRsrc from operandi_utils.database.models import DBProcessingStatistics -from .user_utils import user_auth, user_register_with_handling +from .user_utils import user_auth_with_handling, user_register_with_handling class RouterUser: @@ -60,19 +59,7 @@ async def user_login(self, auth: HTTPBasicCredentials = Depends(HTTPBasic())) -> """ Used for user authentication. """ - email = auth.username - password = auth.password - headers = {"WWW-Authenticate": "Basic"} - if not (email and password): - message = f"User login failed, missing e-mail or password field." - self.logger.error(f"{message}") - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, headers=headers, detail=message) - try: - db_user_account = await user_auth(email=email, password=password) - except AuthenticationError as error: - self.logger.error(f"{error}") - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, headers=headers, detail=str(error)) - return PYUserAction.from_db_user_account(action="Successfully logged!", db_user_account=db_user_account) + return await user_auth_with_handling(logger=self.logger, auth=auth) async def user_register( self, email: str, password: str, institution_id: str, account_type: AccountType = AccountType.USER, @@ -80,7 +67,7 @@ async def user_register( ) -> PYUserAction: """ Used for registration. - There are 3 account types: + There are 4 account types: 1) ADMIN 2) USER 3) HARVESTER @@ -101,7 +88,7 @@ async def user_register( return PYUserAction.from_db_user_account(action=action, db_user_account=db_user_account) async def user_processing_stats(self, auth: HTTPBasicCredentials = Depends(HTTPBasic())): - await self.user_login(auth) + await user_auth_with_handling(self.logger, auth) db_user_account = await db_get_user_account_with_email(email=auth.username) db_processing_stats = await db_get_processing_stats(db_user_account.user_id) return db_processing_stats @@ -113,7 +100,7 @@ async def user_workflow_jobs( """ The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ - await self.user_login(auth) + await user_auth_with_handling(self.logger, auth) db_user_account = await db_get_user_account_with_email(email=auth.username) db_workflow_jobs = await db_get_all_workflow_jobs_by_user( user_id=db_user_account.user_id, start_date=start_date, end_date=end_date) @@ -131,7 +118,7 @@ async def user_workspaces( """ The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ - await self.user_login(auth) + await user_auth_with_handling(self.logger, auth) db_user_account = await db_get_user_account_with_email(email=auth.username) db_workspaces = await db_get_all_workspaces_by_user( user_id=db_user_account.user_id, start_date=start_date, end_date=end_date) @@ -144,7 +131,7 @@ async def user_workflows( """ The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ - await self.user_login(auth) + await user_auth_with_handling(self.logger, auth) db_user_account = await db_get_user_account_with_email(email=auth.username) db_workflows = await db_get_all_workflows_by_user( user_id=db_user_account.user_id, start_date=start_date, end_date=end_date) diff --git a/src/server/operandi_server/routers/user_utils.py b/src/server/operandi_server/routers/user_utils.py index cdacf3d..054c464 100644 --- a/src/server/operandi_server/routers/user_utils.py +++ b/src/server/operandi_server/routers/user_utils.py @@ -1,4 +1,5 @@ -from fastapi import HTTPException, status +from fastapi import Depends, HTTPException, status +from fastapi.security import HTTPBasic, HTTPBasicCredentials from hashlib import sha512 from random import random from typing import Tuple @@ -7,8 +8,7 @@ from operandi_utils.database import ( db_create_processing_stats, db_create_user_account, db_get_user_account, db_get_user_account_with_email, DBUserAccount) -from operandi_server.exceptions import AuthenticationError - +from operandi_server.models import PYUserAction async def create_user_if_not_available( @@ -22,23 +22,40 @@ async def create_user_if_not_available( logger, email=username, password=password, account_type=account_type, approved_user=approved_user, details=details, institution_id=institution_id) -async def user_auth(email: str, password: str) -> DBUserAccount: +async def user_auth_with_handling( + logger, auth: HTTPBasicCredentials = Depends(HTTPBasic()), headers=None +) -> PYUserAction: + email = auth.username + password = auth.password + if headers is None: + headers = {"WWW-Authenticate": "Basic"} + if not (email and password): + message = f"User login failed, missing e-mail or password field." + logger.error(f"{message}") + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, headers=headers, detail=message) try: - db_user = await db_get_user_account_with_email(email=email) + db_user_account = await db_get_user_account_with_email(email=email) except RuntimeError: - raise AuthenticationError(f"Not found user account for email: {email}") - password_status = validate_password(plain_password=password, encrypted_password=db_user.encrypted_pass) + message = f"Not found user account for email: {email}" + logger.error(f"{message}") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, headers=headers, detail=message) + if not db_user_account.approved_user: + message = f"The account has not been approved by the admin yet." + logger.error(f"{message}") + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, headers=headers, detail=message) + password_status = validate_password(plain_password=password, encrypted_password=db_user_account.encrypted_pass) if not password_status: - raise AuthenticationError(f"Wrong credentials for email: {email}") - if not db_user.approved_user: - raise AuthenticationError(f"The account has not been approved by the admin yet.") - return db_user + message = f"Wrong credentials for email: {email}" + logger.error(f"{message}") + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, headers=headers, detail=message) + return PYUserAction.from_db_user_account(action="Successfully logged!", db_user_account=db_user_account) async def user_register_with_handling( logger, email: str, password: str, account_type: AccountType, institution_id: str, approved_user: bool = False, - details: str = "User Account" -): - headers = {"WWW-Authenticate": "Basic"} + details: str = "User Account", headers=None +) -> DBUserAccount: + if headers is None: + headers = {"WWW-Authenticate": "Basic"} if account_type not in AccountType: message = f"Wrong account type. Must be one of: {AccountType}" logger.error(f"{message}") diff --git a/src/server/operandi_server/routers/workflow.py b/src/server/operandi_server/routers/workflow.py index f06ca74..8afc72f 100644 --- a/src/server/operandi_server/routers/workflow.py +++ b/src/server/operandi_server/routers/workflow.py @@ -34,13 +34,12 @@ validate_oton_with_handling, nf_script_executable_steps_with_handling ) from .workspace_utils import check_if_file_group_exists_with_handling, get_db_workspace_with_handling -from .user import RouterUser +from .user_utils import user_auth_with_handling class RouterWorkflow: def __init__(self): self.logger = getLogger("operandi_server.routers.workflow") - self.user_authenticator = RouterUser() # The workflows available to all users by default self.production_workflows = [] @@ -190,7 +189,7 @@ async def list_workflows(self, auth: HTTPBasicCredentials = Depends(HTTPBasic()) Curl equivalent: `curl SERVER_ADDR/workflow` """ - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) workflows = get_all_resources_url(SERVER_WORKFLOWS_ROUTER) response = [] for workflow in workflows: @@ -206,7 +205,7 @@ async def download_workflow_script( Curl equivalent: `curl -X GET SERVER_ADDR/workflow/{workflow_id} -H "accept: text/vnd.ocrd.workflow" -o foo.nf` """ - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) db_workflow = await get_db_workflow_with_handling(self.logger, workflow_id=workflow_id) return FileResponse( path=db_workflow.workflow_script_path, @@ -222,7 +221,7 @@ async def upload_workflow_script( Curl equivalent: `curl -X POST SERVER_ADDR/workflow -F nextflow_script=example.nf` """ - py_user_action = await self.user_authenticator.user_login(auth) + py_user_action = await user_auth_with_handling(self.logger, auth) workflow_id, workflow_dir = create_resource_dir(SERVER_WORKFLOWS_ROUTER, resource_id=None) nf_script_dest = join(workflow_dir, nextflow_script.filename) try: @@ -247,7 +246,7 @@ async def update_workflow_script( Curl equivalent: `curl -X PUT SERVER_ADDR/workflow/{workflow_id} -F nextflow_script=example.nf` """ - py_user_action = await self.user_authenticator.user_login(auth) + py_user_action = await user_auth_with_handling(self.logger, auth) if workflow_id in self.production_workflows: message = f"Production workflow cannot be replaced. Tried to replace: {workflow_id}" self.logger.error(message) @@ -282,7 +281,7 @@ async def get_workflow_job_status( Curl equivalent: `curl -X GET SERVER_ADDR/workflow/{workflow_id}/{job_id}` """ - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) db_wf_job = await get_db_workflow_job_with_handling(self.logger, job_id=job_id, check_local_existence=True) workspace_id = db_wf_job.workspace_id @@ -316,7 +315,7 @@ async def download_workflow_job_logs( Curl equivalent: `curl -X GET SERVER_ADDR/workflow/{workflow_id}/logs -H "accept: application/vnd.zip" -o foo.zip` """ - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) await self._push_status_request_to_rabbitmq(job_id=job_id) db_wf_job = await get_db_workflow_job_with_handling(self.logger, job_id=job_id, check_local_existence=True) @@ -340,7 +339,7 @@ async def download_workflow_job_logs( async def download_workflow_job_hpc_log( self, workflow_id: str, job_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic())): - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) await self._push_status_request_to_rabbitmq(job_id=job_id) db_wf_job = await get_db_workflow_job_with_handling(self.logger, job_id=job_id, check_local_existence=True) @@ -369,7 +368,7 @@ async def submit_to_rabbitmq_queue( self, workflow_id: str, workflow_args: WorkflowArguments, sbatch_args: SbatchArguments, details: str = "Workflow job", auth: HTTPBasicCredentials = Depends(HTTPBasic()) ): - py_user_action = await self.user_authenticator.user_login(auth) + py_user_action = await user_auth_with_handling(self.logger, auth) user_account_type = py_user_action.account_type try: @@ -474,7 +473,7 @@ async def convert_txt_to_nextflow( self, txt_file: UploadFile, environment: str, with_mets_server: bool = True, auth: HTTPBasicCredentials = Depends(HTTPBasic()) ): - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) oton_id, oton_dir = create_resource_dir(SERVER_OTON_CONVERSIONS, resource_id=None) ocrd_process_txt = join(oton_dir, f"ocrd_process_input.txt") nf_script_dest = join(oton_dir, f"nextflow_output.nf") diff --git a/src/server/operandi_server/routers/workspace.py b/src/server/operandi_server/routers/workspace.py index db07230..afb7001 100644 --- a/src/server/operandi_server/routers/workspace.py +++ b/src/server/operandi_server/routers/workspace.py @@ -26,13 +26,12 @@ parse_file_groups_with_handling, remove_file_groups_with_handling ) -from .user import RouterUser +from .user_utils import user_auth_with_handling class RouterWorkspace: def __init__(self): self.logger = getLogger("operandi_server.routers.workspace") - self.user_authenticator = RouterUser() self.router = APIRouter(tags=[ServerApiTag.WORKSPACE]) self.router.add_api_route( path="/import_external_workspace", @@ -82,7 +81,7 @@ async def list_workspaces(self, auth: HTTPBasicCredentials = Depends(HTTPBasic() Curl equivalent: `curl -X GET SERVER_ADDR/workspace` """ - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) workspaces = get_all_resources_url(SERVER_WORKSPACES_ROUTER) response = [] for workspace in workspaces: @@ -100,7 +99,7 @@ async def download_workspace( Curl equivalent: `curl -X GET SERVER_ADDR/workspace/{workspace_id} -H "accept: application/vnd.ocrd+zip" -o foo.zip` """ - py_user_action = await self.user_authenticator.user_login(auth) + py_user_action = await user_auth_with_handling(self.logger, auth) db_workspace = await get_db_workspace_with_handling( self.logger, workspace_id, check_ready=True, check_deleted=True, check_local_existence=True) @@ -124,7 +123,7 @@ async def upload_workspace_from_url( self, mets_url: str, preserve_file_grps: str, mets_basename: str = DEFAULT_METS_BASENAME, details: str = f"Workspace imported from a mets file url", auth: HTTPBasicCredentials = Depends(HTTPBasic()) ) -> WorkspaceRsrc: - py_user_action = await self.user_authenticator.user_login(auth) + py_user_action = await user_auth_with_handling(self.logger, auth) file_grps_to_preserve = parse_file_groups_with_handling(self.logger, file_groups=preserve_file_grps) workspace_id, workspace_dir = create_resource_dir(SERVER_WORKSPACES_ROUTER) @@ -161,7 +160,7 @@ async def upload_workspace( Curl equivalent: `curl -X POST SERVER_ADDR/workspace -H "content-type: multipart/form-data" -F workspace=example_ws.ocrd.zip` """ - py_user_action = await self.user_authenticator.user_login(auth) + py_user_action = await user_auth_with_handling(self.logger, auth) ws_id, ws_dir = create_resource_dir(SERVER_WORKSPACES_ROUTER, resource_id=None) bag_dest = f"{ws_dir}.zip" try: @@ -194,7 +193,7 @@ async def put_workspace( `curl -X PUT SERVER_ADDR/workspace/{workspace_id} -H "content-type: multipart/form-data" -F workspace=example_ws.ocrd.zip` """ - py_user_action = await self.user_authenticator.user_login(auth) + py_user_action = await user_auth_with_handling(self.logger, auth) try: await db_get_workspace(workspace_id=workspace_id) # Note: This check raises HTTP errors on RuntimeError for @@ -240,7 +239,7 @@ async def delete_workspace( Curl equivalent: `curl -X DELETE SERVER_ADDR/workspace/{workspace_id}` """ - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) await get_db_workspace_with_handling( self.logger, workspace_id, check_ready=True, check_deleted=True, check_local_existence=True) @@ -259,7 +258,7 @@ async def remove_file_group_from_workspace( self, workspace_id: str, remove_file_grps: str, recursive: bool = True, force: bool = True, auth: HTTPBasicCredentials = Depends(HTTPBasic()) ) -> WorkspaceRsrc: - await self.user_authenticator.user_login(auth) + await user_auth_with_handling(self.logger, auth) db_workspace = await get_db_workspace_with_handling( self.logger, workspace_id, check_ready=True, check_deleted=True, check_local_existence=True ) From bb1dccfd0fdb9e0a964ab22ed7da456ac0ed70af Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 15:35:47 +0100 Subject: [PATCH 03/15] refactor: NF workflows without version --- .../operandi_utils/hpc/nextflow_workflows/default_workflow.nf | 2 +- .../hpc/nextflow_workflows/default_workflow_with_MS.nf | 2 +- .../operandi_utils/hpc/nextflow_workflows/odem_workflow.nf | 2 +- .../hpc/nextflow_workflows/odem_workflow_with_MS.nf | 2 +- src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow.nf | 2 +- .../hpc/nextflow_workflows/sbb_workflow_with_MS.nf | 2 +- .../operandi_utils/hpc/nextflow_workflows/template_workflow.nf | 2 +- .../hpc/nextflow_workflows/template_workflow_with_MS.nf | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow.nf index 464b2c6..0b63c78 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow_with_MS.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow_with_MS.nf index 6be5d92..dbdf345 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow_with_MS.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/default_workflow_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow.nf index a22b221..2a5176f 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow_with_MS.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow_with_MS.nf index 4b7b94d..9427780 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow_with_MS.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/odem_workflow_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow.nf index 0dcc63a..4a69f92 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow_with_MS.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow_with_MS.nf index 840fae0..822670a 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow_with_MS.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/sbb_workflow_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow.nf index 859958d..ad74b0f 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow_with_MS.nf b/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow_with_MS.nf index ddbe731..adde017 100755 --- a/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow_with_MS.nf +++ b/src/utils/operandi_utils/hpc/nextflow_workflows/template_workflow_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.2 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" From bdf4e819ba8f153cecf48b12c08f7aefe86a232b Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 15:54:11 +0100 Subject: [PATCH 04/15] refactor: password utils --- .../operandi_server/routers/password_utils.py | 19 ++++++++++++++ .../operandi_server/routers/user_utils.py | 26 ++----------------- 2 files changed, 21 insertions(+), 24 deletions(-) create mode 100644 src/server/operandi_server/routers/password_utils.py diff --git a/src/server/operandi_server/routers/password_utils.py b/src/server/operandi_server/routers/password_utils.py new file mode 100644 index 0000000..e7168ef --- /dev/null +++ b/src/server/operandi_server/routers/password_utils.py @@ -0,0 +1,19 @@ +from hashlib import sha512 +from random import random +from typing import Tuple + +def get_random_salt() -> str: + return sha512(f"{hash(str(random()))}".encode("utf-8")).hexdigest()[:8] + +def get_hex_digest(salt: str, plain_password: str): + return sha512(f"{salt}{plain_password}".encode("utf-8")).hexdigest() + +def encrypt_password(plain_password: str) -> Tuple[str, str]: + salt = get_random_salt() + hashed_password = get_hex_digest(salt, plain_password) + encrypted_password = f"{salt}${hashed_password}" + return salt, encrypted_password + +def validate_password(plain_password: str, encrypted_password: str) -> bool: + salt, hashed_password = encrypted_password.split(sep='$', maxsplit=1) + return hashed_password == get_hex_digest(salt, plain_password) diff --git a/src/server/operandi_server/routers/user_utils.py b/src/server/operandi_server/routers/user_utils.py index 054c464..6a624b7 100644 --- a/src/server/operandi_server/routers/user_utils.py +++ b/src/server/operandi_server/routers/user_utils.py @@ -1,14 +1,12 @@ from fastapi import Depends, HTTPException, status from fastapi.security import HTTPBasic, HTTPBasicCredentials -from hashlib import sha512 -from random import random -from typing import Tuple from operandi_utils.constants import AccountType from operandi_utils.database import ( db_create_processing_stats, db_create_user_account, db_get_user_account, db_get_user_account_with_email, DBUserAccount) from operandi_server.models import PYUserAction +from .password_utils import encrypt_password, validate_password async def create_user_if_not_available( @@ -60,10 +58,10 @@ async def user_register_with_handling( message = f"Wrong account type. Must be one of: {AccountType}" logger.error(f"{message}") raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, headers=headers, detail=message) - salt, encrypted_password = encrypt_password(password) try: await db_get_user_account(email) except RuntimeError: + salt, encrypted_password = encrypt_password(password) # No user existing with the provided e-mail, register db_user_account = await db_create_user_account( institution_id=institution_id, email=email, encrypted_pass=encrypted_password, salt=salt, @@ -73,23 +71,3 @@ async def user_register_with_handling( message = f"Another user is already registered with email: {email}" logger.error(f"{message}") raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, headers=headers, detail=message) - - -def encrypt_password(plain_password: str) -> Tuple[str, str]: - salt = get_random_salt() - hashed_password = get_hex_digest(salt, plain_password) - encrypted_password = f"{salt}${hashed_password}" - return salt, encrypted_password - - -def get_hex_digest(salt: str, plain_password: str): - return sha512(f"{salt}{plain_password}".encode("utf-8")).hexdigest() - - -def get_random_salt() -> str: - return sha512(f"{hash(str(random()))}".encode("utf-8")).hexdigest()[:8] - - -def validate_password(plain_password: str, encrypted_password: str) -> bool: - salt, hashed_password = encrypted_password.split(sep='$', maxsplit=1) - return hashed_password == get_hex_digest(salt, plain_password) From 2b7093af642ff4a8ffd1b420808c9b72d957339b Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 15:58:45 +0100 Subject: [PATCH 05/15] refactor: match same name of the same method --- src/server/operandi_server/routers/admin_panel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/operandi_server/routers/admin_panel.py b/src/server/operandi_server/routers/admin_panel.py index 0a21a4f..fd1ddef 100644 --- a/src/server/operandi_server/routers/admin_panel.py +++ b/src/server/operandi_server/routers/admin_panel.py @@ -25,7 +25,7 @@ def __init__(self): ) self.router.add_api_route( path="/admin/processing_stats/{user_id}", - endpoint=self.get_processing_stats_for_user, methods=["GET"], status_code=status.HTTP_200_OK, + endpoint=self.user_processing_stats, methods=["GET"], status_code=status.HTTP_200_OK, summary="Get processing stats for a specific user by user_id" ) self.router.add_api_route( @@ -85,7 +85,7 @@ async def get_users(self, auth: HTTPBasicCredentials = Depends(HTTPBasic())): users = await db_get_all_user_accounts() return [PYUserInfo.from_db_user_account(user) for user in users] - async def get_processing_stats_for_user(self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic())): + async def user_processing_stats(self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic())): await self.auth_admin_with_handling(auth) try: db_processing_stats = await db_get_processing_stats(user_id) From c65c158067b25e017729672147d2249109d4cc91 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 16:15:56 +0100 Subject: [PATCH 06/15] refactor: remove duplication of user workflows --- src/server/operandi_server/routers/admin_panel.py | 8 ++++---- src/server/operandi_server/routers/user.py | 12 +++++------- src/server/operandi_server/routers/workflow_utils.py | 12 ++++++++++-- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/server/operandi_server/routers/admin_panel.py b/src/server/operandi_server/routers/admin_panel.py index fd1ddef..e6b74c6 100644 --- a/src/server/operandi_server/routers/admin_panel.py +++ b/src/server/operandi_server/routers/admin_panel.py @@ -8,10 +8,11 @@ from operandi_utils.constants import AccountType, ServerApiTag from operandi_utils.database import ( db_get_all_user_accounts, db_get_processing_stats, db_get_all_workflow_jobs_by_user, - db_get_workflow, db_get_workspace, db_get_all_workspaces_by_user, db_get_all_workflows_by_user + db_get_workflow, db_get_workspace, db_get_all_workspaces_by_user ) from operandi_utils.utils import send_bag_to_ola_hd from .user_utils import user_auth_with_handling +from .workflow_utils import get_workflows_of_user from .workspace_utils import create_workspace_bag, get_db_workspace_with_handling, validate_bag_with_handling class RouterAdminPanel: @@ -130,10 +131,9 @@ async def user_workspaces( async def user_workflows( self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic()), start_date: Optional[datetime] = None, end_date: Optional[datetime] = None - ) -> List: + ) -> List[WorkflowRsrc]: """ The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ await self.auth_admin_with_handling(auth) - db_workflows = await db_get_all_workflows_by_user(user_id=user_id, start_date=start_date, end_date=end_date) - return [WorkflowRsrc.from_db_workflow(db_workflow) for db_workflow in db_workflows] + return await get_workflows_of_user(user_id=user_id, start_date=start_date, end_date=end_date) diff --git a/src/server/operandi_server/routers/user.py b/src/server/operandi_server/routers/user.py index 1c4f8b9..e0ac8ad 100644 --- a/src/server/operandi_server/routers/user.py +++ b/src/server/operandi_server/routers/user.py @@ -7,10 +7,11 @@ from operandi_utils.constants import AccountType, ServerApiTag from operandi_utils.database import ( db_get_processing_stats, db_get_all_workflow_jobs_by_user, db_get_user_account_with_email, - db_get_workflow, db_get_workspace, db_get_all_workspaces_by_user, db_get_all_workflows_by_user + db_get_workflow, db_get_workspace, db_get_all_workspaces_by_user ) from operandi_server.models import PYUserAction, WorkflowJobRsrc, WorkspaceRsrc, WorkflowRsrc from operandi_utils.database.models import DBProcessingStatistics +from .workflow_utils import get_workflows_of_user from .user_utils import user_auth_with_handling, user_register_with_handling @@ -127,12 +128,9 @@ async def user_workspaces( async def user_workflows( self, auth: HTTPBasicCredentials = Depends(HTTPBasic()), start_date: Optional[datetime] = None, end_date: Optional[datetime] = None - ) -> List: + ) -> List[WorkflowRsrc]: """ The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ - await user_auth_with_handling(self.logger, auth) - db_user_account = await db_get_user_account_with_email(email=auth.username) - db_workflows = await db_get_all_workflows_by_user( - user_id=db_user_account.user_id, start_date=start_date, end_date=end_date) - return [WorkflowRsrc.from_db_workflow(db_workflow) for db_workflow in db_workflows] + py_user_action = await user_auth_with_handling(self.logger, auth) + return await get_workflows_of_user(user_id=py_user_action.user_id, start_date=start_date, end_date=end_date) diff --git a/src/server/operandi_server/routers/workflow_utils.py b/src/server/operandi_server/routers/workflow_utils.py index 591798b..84e9ca2 100644 --- a/src/server/operandi_server/routers/workflow_utils.py +++ b/src/server/operandi_server/routers/workflow_utils.py @@ -1,11 +1,13 @@ +from datetime import datetime from fastapi import HTTPException, status from pathlib import Path -from typing import List +from typing import List, Optional -from operandi_utils.database import db_get_workflow, db_get_workflow_job +from operandi_utils.database import db_get_workflow, db_get_workflow_job, db_get_all_workflows_by_user from operandi_utils.database.models import DBWorkflow, DBWorkflowJob from operandi_utils.oton import OTONConverter, OCRDValidator from operandi_utils.oton.constants import PARAMS_KEY_METS_SOCKET_PATH +from operandi_server.models import WorkflowRsrc async def get_db_workflow_with_handling( @@ -112,3 +114,9 @@ async def convert_oton_with_handling( message = "Failed to convert ocrd process workflow to nextflow workflow" logger.error(f"{message}, error: {error}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) + +async def get_workflows_of_user( + user_id: str, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None +) -> List[WorkflowRsrc]: + db_workflows = await db_get_all_workflows_by_user(user_id=user_id, start_date=start_date, end_date=end_date) + return [WorkflowRsrc.from_db_workflow(db_workflow) for db_workflow in db_workflows] From 8ce6701225418a81de92d52317de814a021589e5 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 16:24:34 +0100 Subject: [PATCH 07/15] refactor: remove duplication of user workspaces --- src/server/operandi_server/routers/admin_panel.py | 11 ++++++----- src/server/operandi_server/routers/user.py | 14 ++++++-------- .../operandi_server/routers/workspace_utils.py | 12 ++++++++++-- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/server/operandi_server/routers/admin_panel.py b/src/server/operandi_server/routers/admin_panel.py index e6b74c6..b9e9243 100644 --- a/src/server/operandi_server/routers/admin_panel.py +++ b/src/server/operandi_server/routers/admin_panel.py @@ -8,12 +8,14 @@ from operandi_utils.constants import AccountType, ServerApiTag from operandi_utils.database import ( db_get_all_user_accounts, db_get_processing_stats, db_get_all_workflow_jobs_by_user, - db_get_workflow, db_get_workspace, db_get_all_workspaces_by_user + db_get_workflow, db_get_workspace ) from operandi_utils.utils import send_bag_to_ola_hd from .user_utils import user_auth_with_handling from .workflow_utils import get_workflows_of_user -from .workspace_utils import create_workspace_bag, get_db_workspace_with_handling, validate_bag_with_handling +from .workspace_utils import ( + create_workspace_bag, get_workspaces_of_user, get_db_workspace_with_handling, validate_bag_with_handling +) class RouterAdminPanel: def __init__(self): @@ -120,13 +122,12 @@ async def user_workflow_jobs( async def user_workspaces( self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic()), start_date: Optional[datetime] = None, end_date: Optional[datetime] = None - ) -> List: + ) -> List[WorkspaceRsrc]: """ The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ await self.auth_admin_with_handling(auth) - db_workspaces = await db_get_all_workspaces_by_user(user_id=user_id, start_date=start_date, end_date=end_date) - return [WorkspaceRsrc.from_db_workspace(db_workspace) for db_workspace in db_workspaces] + return await get_workspaces_of_user(user_id=user_id, start_date=start_date, end_date=end_date) async def user_workflows( self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic()), diff --git a/src/server/operandi_server/routers/user.py b/src/server/operandi_server/routers/user.py index e0ac8ad..a0ac3be 100644 --- a/src/server/operandi_server/routers/user.py +++ b/src/server/operandi_server/routers/user.py @@ -7,11 +7,12 @@ from operandi_utils.constants import AccountType, ServerApiTag from operandi_utils.database import ( db_get_processing_stats, db_get_all_workflow_jobs_by_user, db_get_user_account_with_email, - db_get_workflow, db_get_workspace, db_get_all_workspaces_by_user + db_get_workflow, db_get_workspace ) from operandi_server.models import PYUserAction, WorkflowJobRsrc, WorkspaceRsrc, WorkflowRsrc from operandi_utils.database.models import DBProcessingStatistics from .workflow_utils import get_workflows_of_user +from .workspace_utils import get_workspaces_of_user from .user_utils import user_auth_with_handling, user_register_with_handling @@ -97,7 +98,7 @@ async def user_processing_stats(self, auth: HTTPBasicCredentials = Depends(HTTPB async def user_workflow_jobs( self, auth: HTTPBasicCredentials = Depends(HTTPBasic()), start_date: Optional[datetime] = None, end_date: Optional[datetime] = None - ) -> List: + ) -> List[WorkflowJobRsrc]: """ The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ @@ -115,15 +116,12 @@ async def user_workflow_jobs( async def user_workspaces( self, auth: HTTPBasicCredentials = Depends(HTTPBasic()), start_date: Optional[datetime] = None, end_date: Optional[datetime] = None - ) -> List: + ) -> List[WorkspaceRsrc]: """ The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ - await user_auth_with_handling(self.logger, auth) - db_user_account = await db_get_user_account_with_email(email=auth.username) - db_workspaces = await db_get_all_workspaces_by_user( - user_id=db_user_account.user_id, start_date=start_date, end_date=end_date) - return [WorkspaceRsrc.from_db_workspace(db_workspace) for db_workspace in db_workspaces] + py_user_action = await user_auth_with_handling(self.logger, auth) + return await get_workspaces_of_user(user_id=py_user_action.user_id, start_date=start_date, end_date=end_date) async def user_workflows( self, auth: HTTPBasicCredentials = Depends(HTTPBasic()), diff --git a/src/server/operandi_server/routers/workspace_utils.py b/src/server/operandi_server/routers/workspace_utils.py index f3ceb85..79cee56 100644 --- a/src/server/operandi_server/routers/workspace_utils.py +++ b/src/server/operandi_server/routers/workspace_utils.py @@ -1,9 +1,10 @@ import bagit +from datetime import datetime from fastapi import HTTPException, status from os.path import join from pathlib import Path from tempfile import NamedTemporaryFile -from typing import List, Union +from typing import List, Optional, Union from zipfile import ZipFile from ocrd import Resolver @@ -14,8 +15,9 @@ from operandi_server.constants import DEFAULT_FILE_GRP, DEFAULT_METS_BASENAME from operandi_server.exceptions import WorkspaceNotValidException from operandi_utils.constants import StateWorkspace -from operandi_utils.database import db_get_workspace +from operandi_utils.database import db_get_workspace, db_get_all_workspaces_by_user from operandi_utils.database.models import DBWorkspace +from operandi_server.models import WorkspaceRsrc def get_ocrd_workspace_physical_pages(mets_path: str) -> List[str]: @@ -216,3 +218,9 @@ def extract_file_groups_from_db_model_with_handling(logger, db_workspace) -> Lis def check_if_file_group_exists_with_handling(logger, db_workspace, file_group: str) -> bool: file_groups = extract_file_groups_from_db_model_with_handling(logger, db_workspace) return file_group in file_groups + +async def get_workspaces_of_user( + user_id: str, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None +) -> List[WorkspaceRsrc]: + db_workspaces = await db_get_all_workspaces_by_user(user_id=user_id, start_date=start_date, end_date=end_date) + return [WorkspaceRsrc.from_db_workspace(db_workspace) for db_workspace in db_workspaces] From a4cdcf64bd17a667614d9efde1c4fb54a62f5d45 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 16:34:50 +0100 Subject: [PATCH 08/15] remove duplication of user workflow jobs --- .../operandi_server/routers/admin_panel.py | 18 ++++-------------- src/server/operandi_server/routers/user.py | 19 ++++--------------- .../operandi_server/routers/workflow_utils.py | 18 ++++++++++++++++-- 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/src/server/operandi_server/routers/admin_panel.py b/src/server/operandi_server/routers/admin_panel.py index b9e9243..079f8e7 100644 --- a/src/server/operandi_server/routers/admin_panel.py +++ b/src/server/operandi_server/routers/admin_panel.py @@ -6,13 +6,10 @@ from operandi_server.models import PYUserInfo, WorkflowJobRsrc, WorkspaceRsrc, WorkflowRsrc from operandi_utils.constants import AccountType, ServerApiTag -from operandi_utils.database import ( - db_get_all_user_accounts, db_get_processing_stats, db_get_all_workflow_jobs_by_user, - db_get_workflow, db_get_workspace -) +from operandi_utils.database import db_get_all_user_accounts, db_get_processing_stats from operandi_utils.utils import send_bag_to_ola_hd from .user_utils import user_auth_with_handling -from .workflow_utils import get_workflows_of_user +from .workflow_utils import get_workflows_of_user, get_workflow_jobs_of_user from .workspace_utils import ( create_workspace_bag, get_workspaces_of_user, get_db_workspace_with_handling, validate_bag_with_handling ) @@ -105,19 +102,12 @@ async def user_processing_stats(self, user_id: str, auth: HTTPBasicCredentials = async def user_workflow_jobs( self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic()), start_date: Optional[datetime] = None, end_date: Optional[datetime] = None - ) -> List: + ) -> List[WorkflowJobRsrc]: """ The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ await self.auth_admin_with_handling(auth) - db_workflow_jobs = await db_get_all_workflow_jobs_by_user( - user_id=user_id, start_date=start_date, end_date=end_date) - response = [] - for db_workflow_job in db_workflow_jobs: - db_workflow = await db_get_workflow(db_workflow_job.workflow_id) - db_workspace = await db_get_workspace(db_workflow_job.workspace_id) - response.append(WorkflowJobRsrc.from_db_workflow_job(db_workflow_job, db_workflow, db_workspace)) - return response + return await get_workflow_jobs_of_user(user_id=user_id, start_date=start_date, end_date=end_date) async def user_workspaces( self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic()), diff --git a/src/server/operandi_server/routers/user.py b/src/server/operandi_server/routers/user.py index a0ac3be..be6fc93 100644 --- a/src/server/operandi_server/routers/user.py +++ b/src/server/operandi_server/routers/user.py @@ -5,13 +5,10 @@ from fastapi.security import HTTPBasic, HTTPBasicCredentials from operandi_utils.constants import AccountType, ServerApiTag -from operandi_utils.database import ( - db_get_processing_stats, db_get_all_workflow_jobs_by_user, db_get_user_account_with_email, - db_get_workflow, db_get_workspace -) +from operandi_utils.database import db_get_processing_stats, db_get_user_account_with_email from operandi_server.models import PYUserAction, WorkflowJobRsrc, WorkspaceRsrc, WorkflowRsrc from operandi_utils.database.models import DBProcessingStatistics -from .workflow_utils import get_workflows_of_user +from .workflow_utils import get_workflows_of_user, get_workflow_jobs_of_user from .workspace_utils import get_workspaces_of_user from .user_utils import user_auth_with_handling, user_register_with_handling @@ -102,16 +99,8 @@ async def user_workflow_jobs( """ The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ - await user_auth_with_handling(self.logger, auth) - db_user_account = await db_get_user_account_with_email(email=auth.username) - db_workflow_jobs = await db_get_all_workflow_jobs_by_user( - user_id=db_user_account.user_id, start_date=start_date, end_date=end_date) - response = [] - for db_workflow_job in db_workflow_jobs: - db_workflow = await db_get_workflow(db_workflow_job.workflow_id) - db_workspace = await db_get_workspace(db_workflow_job.workspace_id) - response.append(WorkflowJobRsrc.from_db_workflow_job(db_workflow_job, db_workflow, db_workspace)) - return response + py_user_action = await user_auth_with_handling(self.logger, auth) + return await get_workflow_jobs_of_user(user_id=py_user_action.user_id, start_date=start_date, end_date=end_date) async def user_workspaces( self, auth: HTTPBasicCredentials = Depends(HTTPBasic()), diff --git a/src/server/operandi_server/routers/workflow_utils.py b/src/server/operandi_server/routers/workflow_utils.py index 84e9ca2..9f814db 100644 --- a/src/server/operandi_server/routers/workflow_utils.py +++ b/src/server/operandi_server/routers/workflow_utils.py @@ -3,11 +3,14 @@ from pathlib import Path from typing import List, Optional -from operandi_utils.database import db_get_workflow, db_get_workflow_job, db_get_all_workflows_by_user +from operandi_utils.database import ( + db_get_all_workflows_by_user, db_get_all_workflow_jobs_by_user, + db_get_workflow, db_get_workflow_job, db_get_workspace +) from operandi_utils.database.models import DBWorkflow, DBWorkflowJob from operandi_utils.oton import OTONConverter, OCRDValidator from operandi_utils.oton.constants import PARAMS_KEY_METS_SOCKET_PATH -from operandi_server.models import WorkflowRsrc +from operandi_server.models import WorkflowRsrc, WorkflowJobRsrc async def get_db_workflow_with_handling( @@ -120,3 +123,14 @@ async def get_workflows_of_user( ) -> List[WorkflowRsrc]: db_workflows = await db_get_all_workflows_by_user(user_id=user_id, start_date=start_date, end_date=end_date) return [WorkflowRsrc.from_db_workflow(db_workflow) for db_workflow in db_workflows] + +async def get_workflow_jobs_of_user( + user_id: str, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None +) -> List[WorkflowJobRsrc]: + db_workflow_jobs = await db_get_all_workflow_jobs_by_user(user_id=user_id, start_date=start_date, end_date=end_date) + response = [] + for db_workflow_job in db_workflow_jobs: + db_workflow = await db_get_workflow(db_workflow_job.workflow_id) + db_workspace = await db_get_workspace(db_workflow_job.workspace_id) + response.append(WorkflowJobRsrc.from_db_workflow_job(db_workflow_job, db_workflow, db_workspace)) + return response From 183cb6864558033e68eeb7fda28e16fd9d774d33 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 16:40:19 +0100 Subject: [PATCH 09/15] refactor: method names to respect prefix --- src/server/operandi_server/routers/admin_panel.py | 10 +++++----- src/server/operandi_server/routers/user.py | 10 +++++----- src/server/operandi_server/routers/workflow_utils.py | 4 ++-- src/server/operandi_server/routers/workspace_utils.py | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/server/operandi_server/routers/admin_panel.py b/src/server/operandi_server/routers/admin_panel.py index 079f8e7..4a50a59 100644 --- a/src/server/operandi_server/routers/admin_panel.py +++ b/src/server/operandi_server/routers/admin_panel.py @@ -9,9 +9,9 @@ from operandi_utils.database import db_get_all_user_accounts, db_get_processing_stats from operandi_utils.utils import send_bag_to_ola_hd from .user_utils import user_auth_with_handling -from .workflow_utils import get_workflows_of_user, get_workflow_jobs_of_user +from .workflow_utils import get_user_workflows, get_user_workflow_jobs from .workspace_utils import ( - create_workspace_bag, get_workspaces_of_user, get_db_workspace_with_handling, validate_bag_with_handling + create_workspace_bag, get_user_workspaces, get_db_workspace_with_handling, validate_bag_with_handling ) class RouterAdminPanel: @@ -107,7 +107,7 @@ async def user_workflow_jobs( The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ await self.auth_admin_with_handling(auth) - return await get_workflow_jobs_of_user(user_id=user_id, start_date=start_date, end_date=end_date) + return await get_user_workflow_jobs(user_id=user_id, start_date=start_date, end_date=end_date) async def user_workspaces( self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic()), @@ -117,7 +117,7 @@ async def user_workspaces( The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ await self.auth_admin_with_handling(auth) - return await get_workspaces_of_user(user_id=user_id, start_date=start_date, end_date=end_date) + return await get_user_workspaces(user_id=user_id, start_date=start_date, end_date=end_date) async def user_workflows( self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic()), @@ -127,4 +127,4 @@ async def user_workflows( The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ await self.auth_admin_with_handling(auth) - return await get_workflows_of_user(user_id=user_id, start_date=start_date, end_date=end_date) + return await get_user_workflows(user_id=user_id, start_date=start_date, end_date=end_date) diff --git a/src/server/operandi_server/routers/user.py b/src/server/operandi_server/routers/user.py index be6fc93..86f9931 100644 --- a/src/server/operandi_server/routers/user.py +++ b/src/server/operandi_server/routers/user.py @@ -8,8 +8,8 @@ from operandi_utils.database import db_get_processing_stats, db_get_user_account_with_email from operandi_server.models import PYUserAction, WorkflowJobRsrc, WorkspaceRsrc, WorkflowRsrc from operandi_utils.database.models import DBProcessingStatistics -from .workflow_utils import get_workflows_of_user, get_workflow_jobs_of_user -from .workspace_utils import get_workspaces_of_user +from .workflow_utils import get_user_workflows, get_user_workflow_jobs +from .workspace_utils import get_user_workspaces from .user_utils import user_auth_with_handling, user_register_with_handling @@ -100,7 +100,7 @@ async def user_workflow_jobs( The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ py_user_action = await user_auth_with_handling(self.logger, auth) - return await get_workflow_jobs_of_user(user_id=py_user_action.user_id, start_date=start_date, end_date=end_date) + return await get_user_workflow_jobs(user_id=py_user_action.user_id, start_date=start_date, end_date=end_date) async def user_workspaces( self, auth: HTTPBasicCredentials = Depends(HTTPBasic()), @@ -110,7 +110,7 @@ async def user_workspaces( The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ py_user_action = await user_auth_with_handling(self.logger, auth) - return await get_workspaces_of_user(user_id=py_user_action.user_id, start_date=start_date, end_date=end_date) + return await get_user_workspaces(user_id=py_user_action.user_id, start_date=start_date, end_date=end_date) async def user_workflows( self, auth: HTTPBasicCredentials = Depends(HTTPBasic()), @@ -120,4 +120,4 @@ async def user_workflows( The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ py_user_action = await user_auth_with_handling(self.logger, auth) - return await get_workflows_of_user(user_id=py_user_action.user_id, start_date=start_date, end_date=end_date) + return await get_user_workflows(user_id=py_user_action.user_id, start_date=start_date, end_date=end_date) diff --git a/src/server/operandi_server/routers/workflow_utils.py b/src/server/operandi_server/routers/workflow_utils.py index 9f814db..700db6d 100644 --- a/src/server/operandi_server/routers/workflow_utils.py +++ b/src/server/operandi_server/routers/workflow_utils.py @@ -118,13 +118,13 @@ async def convert_oton_with_handling( logger.error(f"{message}, error: {error}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) -async def get_workflows_of_user( +async def get_user_workflows( user_id: str, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None ) -> List[WorkflowRsrc]: db_workflows = await db_get_all_workflows_by_user(user_id=user_id, start_date=start_date, end_date=end_date) return [WorkflowRsrc.from_db_workflow(db_workflow) for db_workflow in db_workflows] -async def get_workflow_jobs_of_user( +async def get_user_workflow_jobs( user_id: str, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None ) -> List[WorkflowJobRsrc]: db_workflow_jobs = await db_get_all_workflow_jobs_by_user(user_id=user_id, start_date=start_date, end_date=end_date) diff --git a/src/server/operandi_server/routers/workspace_utils.py b/src/server/operandi_server/routers/workspace_utils.py index 79cee56..effcf0a 100644 --- a/src/server/operandi_server/routers/workspace_utils.py +++ b/src/server/operandi_server/routers/workspace_utils.py @@ -219,7 +219,7 @@ def check_if_file_group_exists_with_handling(logger, db_workspace, file_group: s file_groups = extract_file_groups_from_db_model_with_handling(logger, db_workspace) return file_group in file_groups -async def get_workspaces_of_user( +async def get_user_workspaces( user_id: str, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None ) -> List[WorkspaceRsrc]: db_workspaces = await db_get_all_workspaces_by_user(user_id=user_id, start_date=start_date, end_date=end_date) From 5505f8a52b012ce4edd5b2252f95f6747c05c67c Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 17:00:06 +0100 Subject: [PATCH 10/15] refactor: get all users --- .../operandi_server/routers/admin_panel.py | 20 ++++--------------- src/server/operandi_server/routers/user.py | 9 +++------ .../operandi_server/routers/user_utils.py | 20 ++++++++++++++++--- src/utils/operandi_utils/database/__init__.py | 3 ++- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/server/operandi_server/routers/admin_panel.py b/src/server/operandi_server/routers/admin_panel.py index 4a50a59..4974359 100644 --- a/src/server/operandi_server/routers/admin_panel.py +++ b/src/server/operandi_server/routers/admin_panel.py @@ -6,9 +6,8 @@ from operandi_server.models import PYUserInfo, WorkflowJobRsrc, WorkspaceRsrc, WorkflowRsrc from operandi_utils.constants import AccountType, ServerApiTag -from operandi_utils.database import db_get_all_user_accounts, db_get_processing_stats from operandi_utils.utils import send_bag_to_ola_hd -from .user_utils import user_auth_with_handling +from .user_utils import get_user_accounts, get_user_processing_stats_with_handling, user_auth_with_handling from .workflow_utils import get_user_workflows, get_user_workflow_jobs from .workspace_utils import ( create_workspace_bag, get_user_workspaces, get_db_workspace_with_handling, validate_bag_with_handling @@ -80,24 +79,13 @@ async def push_to_ola_hd(self, workspace_id: str, auth: HTTPBasicCredentials = D } return response_message - async def get_users(self, auth: HTTPBasicCredentials = Depends(HTTPBasic())): + async def get_users(self, auth: HTTPBasicCredentials = Depends(HTTPBasic())) -> List[PYUserInfo]: await self.auth_admin_with_handling(auth) - users = await db_get_all_user_accounts() - return [PYUserInfo.from_db_user_account(user) for user in users] + return await get_user_accounts() async def user_processing_stats(self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic())): await self.auth_admin_with_handling(auth) - try: - db_processing_stats = await db_get_processing_stats(user_id) - if not db_processing_stats: - message = f"Processing stats not found for the user_id: {user_id}" - self.logger.error(message) - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=message) - except Exception as error: - message = f"Failed to fetch processing stats for user_id: {user_id}, error: {error}" - self.logger.error(message) - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) - return db_processing_stats + return await get_user_processing_stats_with_handling(self.logger, user_id=user_id) async def user_workflow_jobs( self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic()), diff --git a/src/server/operandi_server/routers/user.py b/src/server/operandi_server/routers/user.py index 86f9931..87cb93f 100644 --- a/src/server/operandi_server/routers/user.py +++ b/src/server/operandi_server/routers/user.py @@ -5,12 +5,11 @@ from fastapi.security import HTTPBasic, HTTPBasicCredentials from operandi_utils.constants import AccountType, ServerApiTag -from operandi_utils.database import db_get_processing_stats, db_get_user_account_with_email from operandi_server.models import PYUserAction, WorkflowJobRsrc, WorkspaceRsrc, WorkflowRsrc from operandi_utils.database.models import DBProcessingStatistics from .workflow_utils import get_user_workflows, get_user_workflow_jobs from .workspace_utils import get_user_workspaces -from .user_utils import user_auth_with_handling, user_register_with_handling +from .user_utils import get_user_processing_stats_with_handling, user_auth_with_handling, user_register_with_handling class RouterUser: @@ -87,10 +86,8 @@ async def user_register( return PYUserAction.from_db_user_account(action=action, db_user_account=db_user_account) async def user_processing_stats(self, auth: HTTPBasicCredentials = Depends(HTTPBasic())): - await user_auth_with_handling(self.logger, auth) - db_user_account = await db_get_user_account_with_email(email=auth.username) - db_processing_stats = await db_get_processing_stats(db_user_account.user_id) - return db_processing_stats + py_user_action = await user_auth_with_handling(self.logger, auth) + return await get_user_processing_stats_with_handling(self.logger, user_id=py_user_action.user_id) async def user_workflow_jobs( self, auth: HTTPBasicCredentials = Depends(HTTPBasic()), diff --git a/src/server/operandi_server/routers/user_utils.py b/src/server/operandi_server/routers/user_utils.py index 6a624b7..f3b70be 100644 --- a/src/server/operandi_server/routers/user_utils.py +++ b/src/server/operandi_server/routers/user_utils.py @@ -1,11 +1,12 @@ from fastapi import Depends, HTTPException, status from fastapi.security import HTTPBasic, HTTPBasicCredentials +from typing import List from operandi_utils.constants import AccountType from operandi_utils.database import ( - db_create_processing_stats, db_create_user_account, db_get_user_account, db_get_user_account_with_email, - DBUserAccount) -from operandi_server.models import PYUserAction + db_create_processing_stats, db_create_user_account, db_get_all_user_accounts, db_get_user_account, + db_get_user_account_with_email, db_get_processing_stats, DBProcessingStatistics, DBUserAccount) +from operandi_server.models import PYUserAction, PYUserInfo from .password_utils import encrypt_password, validate_password @@ -71,3 +72,16 @@ async def user_register_with_handling( message = f"Another user is already registered with email: {email}" logger.error(f"{message}") raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, headers=headers, detail=message) + +async def get_user_processing_stats_with_handling(logger, user_id: str) -> DBProcessingStatistics: + try: + db_processing_stats = await db_get_processing_stats(user_id=user_id) + except RuntimeError as error: + message = f"Processing stats not found for the user_id: {user_id}" + logger.error(f"{message}, error: {error}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) + return db_processing_stats + +async def get_user_accounts() -> List[PYUserInfo]: + users = await db_get_all_user_accounts() + return [PYUserInfo.from_db_user_account(user) for user in users] diff --git a/src/utils/operandi_utils/database/__init__.py b/src/utils/operandi_utils/database/__init__.py index 27a22aa..d6e9c84 100644 --- a/src/utils/operandi_utils/database/__init__.py +++ b/src/utils/operandi_utils/database/__init__.py @@ -1,5 +1,6 @@ __all__ = [ "DBHPCSlurmJob", + "DBProcessingStatistics", "DBUserAccount", "DBWorkflow", "DBWorkflowJob", @@ -56,7 +57,7 @@ ] from .base import db_initiate_database, sync_db_initiate_database -from .models import DBHPCSlurmJob, DBUserAccount, DBWorkflow, DBWorkflowJob, DBWorkspace +from .models import DBHPCSlurmJob, DBProcessingStatistics, DBUserAccount, DBWorkflow, DBWorkflowJob, DBWorkspace from .db_hpc_slurm_job import ( db_create_hpc_slurm_job, db_get_hpc_slurm_job, From b2d3b0f6745836707405abbea0cac9634e1ebd95 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 17:26:47 +0100 Subject: [PATCH 11/15] refactor: push status request --- .../operandi_server/routers/workflow.py | 29 +++++-------------- .../operandi_server/routers/workflow_utils.py | 21 ++++++++++++++ 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/src/server/operandi_server/routers/workflow.py b/src/server/operandi_server/routers/workflow.py index 8afc72f..7ed76e8 100644 --- a/src/server/operandi_server/routers/workflow.py +++ b/src/server/operandi_server/routers/workflow.py @@ -10,7 +10,6 @@ from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status, UploadFile from fastapi.responses import FileResponse from fastapi.security import HTTPBasic, HTTPBasicCredentials -from starlette.status import HTTP_404_NOT_FOUND from operandi_utils import get_nf_wfs_dir, get_ocrd_process_wfs_dir from operandi_utils.constants import AccountType, ServerApiTag, StateJob, StateWorkspace @@ -18,8 +17,7 @@ db_create_workflow, db_create_workflow_job, db_get_hpc_slurm_job, db_get_workflow, db_update_workspace, db_increase_processing_stats_with_handling) from operandi_utils.oton import OTONConverter -from operandi_utils.rabbitmq import ( - get_connection_publisher, RABBITMQ_QUEUE_JOB_STATUSES, RABBITMQ_QUEUE_HARVESTER, RABBITMQ_QUEUE_USERS) +from operandi_utils.rabbitmq import get_connection_publisher, RABBITMQ_QUEUE_HARVESTER, RABBITMQ_QUEUE_USERS from operandi_server.constants import ( SERVER_OTON_CONVERSIONS, SERVER_WORKFLOWS_ROUTER, SERVER_WORKFLOW_JOBS_ROUTER, SERVER_WORKSPACES_ROUTER) from operandi_server.files_manager import ( @@ -30,8 +28,10 @@ convert_oton_with_handling, get_db_workflow_job_with_handling, get_db_workflow_with_handling, + nf_script_executable_steps_with_handling, nf_script_uses_mets_server_with_handling, - validate_oton_with_handling, nf_script_executable_steps_with_handling + push_status_request_to_rabbitmq, + validate_oton_with_handling ) from .workspace_utils import check_if_file_group_exists_with_handling, get_db_workspace_with_handling from .user_utils import user_auth_with_handling @@ -119,19 +119,6 @@ def __del__(self): if self.rmq_publisher: self.rmq_publisher.disconnect() - async def _push_status_request_to_rabbitmq(self, job_id: str): - # Create the job status message to be sent to the RabbitMQ queue - try: - job_status_message = {"job_id": f"{job_id}"} - self.logger.debug(f"Encoding the job status RabbitMQ message: {job_status_message}") - encoded_wf_message = dumps(job_status_message).encode(encoding="utf-8") - self.logger.debug(f"Pushing to the RabbitMQ queue for job statuses: {RABBITMQ_QUEUE_JOB_STATUSES}") - self.rmq_publisher.publish_to_queue(queue_name=RABBITMQ_QUEUE_JOB_STATUSES, message=encoded_wf_message) - except Exception as error: - message = "Failed to push status request to RabbitMQ" - self.logger.error(f"{message}, error: {error}") - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) - async def produce_production_workflows( self, ocrd_process_wf_dir: Path = get_ocrd_process_wfs_dir(), @@ -292,7 +279,7 @@ async def get_workflow_job_status( self.logger, workflow_id=workflow_id, check_deleted=False, check_local_existence=False) if db_wf_job.job_state != StateJob.FAILED and db_wf_job.job_state != StateJob.SUCCESS: - await self._push_status_request_to_rabbitmq(job_id=job_id) + await push_status_request_to_rabbitmq(self.logger, self.rmq_publisher, job_id=job_id) # TODO: Fix that by getting rid of the FileManager module try: @@ -316,11 +303,11 @@ async def download_workflow_job_logs( `curl -X GET SERVER_ADDR/workflow/{workflow_id}/logs -H "accept: application/vnd.zip" -o foo.zip` """ await user_auth_with_handling(self.logger, auth) - await self._push_status_request_to_rabbitmq(job_id=job_id) db_wf_job = await get_db_workflow_job_with_handling(self.logger, job_id=job_id, check_local_existence=True) job_state = db_wf_job.job_state if job_state != StateJob.SUCCESS and job_state != StateJob.FAILED: + await push_status_request_to_rabbitmq(self.logger, self.rmq_publisher, job_id=job_id) message = f"Cannot download logs of a job unless it succeeds or fails: {job_id}" self.logger.exception(message) raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=message) @@ -340,11 +327,11 @@ async def download_workflow_job_logs( async def download_workflow_job_hpc_log( self, workflow_id: str, job_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic())): await user_auth_with_handling(self.logger, auth) - await self._push_status_request_to_rabbitmq(job_id=job_id) db_wf_job = await get_db_workflow_job_with_handling(self.logger, job_id=job_id, check_local_existence=True) job_state = db_wf_job.job_state if job_state != StateJob.SUCCESS and job_state != StateJob.FAILED: + await push_status_request_to_rabbitmq(self.logger, self.rmq_publisher, job_id=job_id) message = f"Cannot download logs of a job unless it succeeds or fails: {job_id}" self.logger.exception(message) raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=message) @@ -361,7 +348,7 @@ async def download_workflow_job_hpc_log( slurm_job_log_path = Path(wf_job_local, slurm_job_log) if not slurm_job_log_path.exists(): message = f"No slurm job log file was found for job id: {job_id}" - raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=message) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=message) return FileResponse(path=slurm_job_log_path, filename=slurm_job_log, media_type="application/text") async def submit_to_rabbitmq_queue( diff --git a/src/server/operandi_server/routers/workflow_utils.py b/src/server/operandi_server/routers/workflow_utils.py index 700db6d..75cc81b 100644 --- a/src/server/operandi_server/routers/workflow_utils.py +++ b/src/server/operandi_server/routers/workflow_utils.py @@ -1,8 +1,10 @@ from datetime import datetime from fastapi import HTTPException, status +from json import dumps from pathlib import Path from typing import List, Optional +from operandi_utils.constants import StateJob from operandi_utils.database import ( db_get_all_workflows_by_user, db_get_all_workflow_jobs_by_user, db_get_workflow, db_get_workflow_job, db_get_workspace @@ -10,6 +12,7 @@ from operandi_utils.database.models import DBWorkflow, DBWorkflowJob from operandi_utils.oton import OTONConverter, OCRDValidator from operandi_utils.oton.constants import PARAMS_KEY_METS_SOCKET_PATH +from operandi_utils.rabbitmq import RABBITMQ_QUEUE_JOB_STATUSES from operandi_server.models import WorkflowRsrc, WorkflowJobRsrc @@ -124,12 +127,30 @@ async def get_user_workflows( db_workflows = await db_get_all_workflows_by_user(user_id=user_id, start_date=start_date, end_date=end_date) return [WorkflowRsrc.from_db_workflow(db_workflow) for db_workflow in db_workflows] +async def push_status_request_to_rabbitmq(logger, rmq_publisher, job_id: str): + # Create the job status message to be sent to the RabbitMQ queue + try: + job_status_message = {"job_id": f"{job_id}"} + logger.debug(f"Encoding the job status RabbitMQ message: {job_status_message}") + encoded_wf_message = dumps(job_status_message).encode(encoding="utf-8") + logger.debug(f"Pushing to the RabbitMQ queue for job statuses: {RABBITMQ_QUEUE_JOB_STATUSES}") + rmq_publisher.publish_to_queue(queue_name=RABBITMQ_QUEUE_JOB_STATUSES, message=encoded_wf_message) + except Exception as error: + message = "Failed to push status request to RabbitMQ" + logger.error(f"{message}, error: {error}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) + async def get_user_workflow_jobs( user_id: str, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None ) -> List[WorkflowJobRsrc]: db_workflow_jobs = await db_get_all_workflow_jobs_by_user(user_id=user_id, start_date=start_date, end_date=end_date) response = [] for db_workflow_job in db_workflow_jobs: + job_state = db_workflow_job.job_state + if job_state != StateJob.SUCCESS and job_state != StateJob.FAILED: + # TODO: Call here the 'push_status_request_to_rabbitmq' once + # that method is also refactored to be rmq_publisher independent + pass db_workflow = await db_get_workflow(db_workflow_job.workflow_id) db_workspace = await db_get_workspace(db_workflow_job.workspace_id) response.append(WorkflowJobRsrc.from_db_workflow_job(db_workflow_job, db_workflow, db_workspace)) From 90ec3676f6559dfb2b8238cae88b77fb1dc2721f Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 17:45:39 +0100 Subject: [PATCH 12/15] refactor: remove versioning of nf scripts from tests --- tests/assets/oton/test_output_nextflow1_apptainer.nf | 2 +- tests/assets/oton/test_output_nextflow1_apptainer_with_MS.nf | 2 +- tests/assets/oton/test_output_nextflow1_docker.nf | 2 +- tests/assets/oton/test_output_nextflow1_docker_with_MS.nf | 2 +- tests/assets/oton/test_output_nextflow1_local.nf | 2 +- tests/assets/oton/test_output_nextflow1_local_with_MS.nf | 2 +- tests/assets/oton/test_output_nextflow2.nf | 2 +- tests/assets/oton/test_output_nextflow3.nf | 2 +- tests/assets/oton/test_output_nextflow4.nf | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/assets/oton/test_output_nextflow1_apptainer.nf b/tests/assets/oton/test_output_nextflow1_apptainer.nf index 4e9c069..0b63c78 100644 --- a/tests/assets/oton/test_output_nextflow1_apptainer.nf +++ b/tests/assets/oton/test_output_nextflow1_apptainer.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/tests/assets/oton/test_output_nextflow1_apptainer_with_MS.nf b/tests/assets/oton/test_output_nextflow1_apptainer_with_MS.nf index ba9bf52..dbdf345 100644 --- a/tests/assets/oton/test_output_nextflow1_apptainer_with_MS.nf +++ b/tests/assets/oton/test_output_nextflow1_apptainer_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/tests/assets/oton/test_output_nextflow1_docker.nf b/tests/assets/oton/test_output_nextflow1_docker.nf index fdf03c0..fd63a5d 100644 --- a/tests/assets/oton/test_output_nextflow1_docker.nf +++ b/tests/assets/oton/test_output_nextflow1_docker.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/tests/assets/oton/test_output_nextflow1_docker_with_MS.nf b/tests/assets/oton/test_output_nextflow1_docker_with_MS.nf index 05e8966..5fa87c0 100644 --- a/tests/assets/oton/test_output_nextflow1_docker_with_MS.nf +++ b/tests/assets/oton/test_output_nextflow1_docker_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/tests/assets/oton/test_output_nextflow1_local.nf b/tests/assets/oton/test_output_nextflow1_local.nf index ced217c..541a93b 100644 --- a/tests/assets/oton/test_output_nextflow1_local.nf +++ b/tests/assets/oton/test_output_nextflow1_local.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/tests/assets/oton/test_output_nextflow1_local_with_MS.nf b/tests/assets/oton/test_output_nextflow1_local_with_MS.nf index e11f434..4080332 100644 --- a/tests/assets/oton/test_output_nextflow1_local_with_MS.nf +++ b/tests/assets/oton/test_output_nextflow1_local_with_MS.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/tests/assets/oton/test_output_nextflow2.nf b/tests/assets/oton/test_output_nextflow2.nf index a2798bd..7850c98 100644 --- a/tests/assets/oton/test_output_nextflow2.nf +++ b/tests/assets/oton/test_output_nextflow2.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" diff --git a/tests/assets/oton/test_output_nextflow3.nf b/tests/assets/oton/test_output_nextflow3.nf index 8e8df4a..19ad338 100644 --- a/tests/assets/oton/test_output_nextflow3.nf +++ b/tests/assets/oton/test_output_nextflow3.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-GT-SEG-BLOCK,OCR-D-OCR" diff --git a/tests/assets/oton/test_output_nextflow4.nf b/tests/assets/oton/test_output_nextflow4.nf index 7f5c313..a72010d 100644 --- a/tests/assets/oton/test_output_nextflow4.nf +++ b/tests/assets/oton/test_output_nextflow4.nf @@ -1,4 +1,4 @@ -// This workflow was automatically generated by the v2.18.0 operandi_utils.oton module +// This workflow was automatically generated by the operandi_utils.oton module nextflow.enable.dsl = 2 params.input_file_group = "OCR-D-IMG" From 5857c4354213d6eb48657505c86e331eb2b01893 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 17:45:59 +0100 Subject: [PATCH 13/15] fix: job status states of checking data in batch --- src/server/operandi_server/routers/admin_panel.py | 11 +++++++++-- src/server/operandi_server/routers/user.py | 9 ++++++++- src/server/operandi_server/routers/workflow_utils.py | 6 ++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/server/operandi_server/routers/admin_panel.py b/src/server/operandi_server/routers/admin_panel.py index 4974359..61dae3b 100644 --- a/src/server/operandi_server/routers/admin_panel.py +++ b/src/server/operandi_server/routers/admin_panel.py @@ -7,6 +7,7 @@ from operandi_server.models import PYUserInfo, WorkflowJobRsrc, WorkspaceRsrc, WorkflowRsrc from operandi_utils.constants import AccountType, ServerApiTag from operandi_utils.utils import send_bag_to_ola_hd +from operandi_utils.rabbitmq import get_connection_publisher from .user_utils import get_user_accounts, get_user_processing_stats_with_handling, user_auth_with_handling from .workflow_utils import get_user_workflows, get_user_workflow_jobs from .workspace_utils import ( @@ -15,7 +16,12 @@ class RouterAdminPanel: def __init__(self): - self.logger = getLogger("operandi_server.routers.user") + self.logger = getLogger("operandi_server.routers.admin_panel") + + self.logger.info(f"Trying to connect RMQ Publisher") + self.rmq_publisher = get_connection_publisher(enable_acks=True) + self.logger.info(f"RMQPublisher connected") + self.router = APIRouter(tags=[ServerApiTag.ADMIN]) self.router.add_api_route( path="/admin/users", @@ -95,7 +101,8 @@ async def user_workflow_jobs( The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ await self.auth_admin_with_handling(auth) - return await get_user_workflow_jobs(user_id=user_id, start_date=start_date, end_date=end_date) + return await get_user_workflow_jobs( + self.logger, self.rmq_publisher, user_id, start_date, end_date) async def user_workspaces( self, user_id: str, auth: HTTPBasicCredentials = Depends(HTTPBasic()), diff --git a/src/server/operandi_server/routers/user.py b/src/server/operandi_server/routers/user.py index 87cb93f..e2b4a41 100644 --- a/src/server/operandi_server/routers/user.py +++ b/src/server/operandi_server/routers/user.py @@ -7,6 +7,7 @@ from operandi_utils.constants import AccountType, ServerApiTag from operandi_server.models import PYUserAction, WorkflowJobRsrc, WorkspaceRsrc, WorkflowRsrc from operandi_utils.database.models import DBProcessingStatistics +from operandi_utils.rabbitmq import get_connection_publisher from .workflow_utils import get_user_workflows, get_user_workflow_jobs from .workspace_utils import get_user_workspaces from .user_utils import get_user_processing_stats_with_handling, user_auth_with_handling, user_register_with_handling @@ -15,6 +16,11 @@ class RouterUser: def __init__(self): self.logger = getLogger("operandi_server.routers.user") + + self.logger.info(f"Trying to connect RMQ Publisher") + self.rmq_publisher = get_connection_publisher(enable_acks=True) + self.logger.info(f"RMQPublisher connected") + self.router = APIRouter(tags=[ServerApiTag.USER]) self.router.add_api_route( path="/user/register", @@ -97,7 +103,8 @@ async def user_workflow_jobs( The expected datetime format: YYYY-MM-DDTHH:MM:SS, for example, 2024-12-01T18:17:15 """ py_user_action = await user_auth_with_handling(self.logger, auth) - return await get_user_workflow_jobs(user_id=py_user_action.user_id, start_date=start_date, end_date=end_date) + return await get_user_workflow_jobs( + self.logger, self.rmq_publisher, py_user_action.user_id, start_date, end_date) async def user_workspaces( self, auth: HTTPBasicCredentials = Depends(HTTPBasic()), diff --git a/src/server/operandi_server/routers/workflow_utils.py b/src/server/operandi_server/routers/workflow_utils.py index 75cc81b..9d04bc5 100644 --- a/src/server/operandi_server/routers/workflow_utils.py +++ b/src/server/operandi_server/routers/workflow_utils.py @@ -141,16 +141,14 @@ async def push_status_request_to_rabbitmq(logger, rmq_publisher, job_id: str): raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=message) async def get_user_workflow_jobs( - user_id: str, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None + logger, rmq_publisher, user_id: str, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None ) -> List[WorkflowJobRsrc]: db_workflow_jobs = await db_get_all_workflow_jobs_by_user(user_id=user_id, start_date=start_date, end_date=end_date) response = [] for db_workflow_job in db_workflow_jobs: job_state = db_workflow_job.job_state if job_state != StateJob.SUCCESS and job_state != StateJob.FAILED: - # TODO: Call here the 'push_status_request_to_rabbitmq' once - # that method is also refactored to be rmq_publisher independent - pass + await push_status_request_to_rabbitmq(logger, rmq_publisher, db_workflow_job.job_id) db_workflow = await db_get_workflow(db_workflow_job.workflow_id) db_workspace = await db_get_workspace(db_workflow_job.workspace_id) response.append(WorkflowJobRsrc.from_db_workflow_job(db_workflow_job, db_workflow, db_workspace)) From b9d0772c4b59cceca2fd597d1ae13c81e870ed8d Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 17:46:44 +0100 Subject: [PATCH 14/15] release: v2.18.3 --- src/utils/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/setup.py b/src/utils/setup.py index 858b942..a896d42 100644 --- a/src/utils/setup.py +++ b/src/utils/setup.py @@ -5,7 +5,7 @@ setup( name='operandi_utils', - version='2.18.2', + version='2.18.3', description='OPERANDI - Utils', long_description=open('README.md').read(), long_description_content_type='text/markdown', From b6b96de8189c9a77f75a90300e03d6b09accbf1a Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 6 Dec 2024 17:48:18 +0100 Subject: [PATCH 15/15] fix: disconnect rmq in destructor --- src/server/operandi_server/routers/admin_panel.py | 4 ++++ src/server/operandi_server/routers/user.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/server/operandi_server/routers/admin_panel.py b/src/server/operandi_server/routers/admin_panel.py index 61dae3b..ae07de7 100644 --- a/src/server/operandi_server/routers/admin_panel.py +++ b/src/server/operandi_server/routers/admin_panel.py @@ -54,6 +54,10 @@ def __init__(self): summary="Push a workspace to Ola-HD service" ) + def __del__(self): + if self.rmq_publisher: + self.rmq_publisher.disconnect() + async def auth_admin_with_handling(self, auth: HTTPBasicCredentials): py_user_action = await user_auth_with_handling(self.logger, auth) if py_user_action.account_type != AccountType.ADMIN: diff --git a/src/server/operandi_server/routers/user.py b/src/server/operandi_server/routers/user.py index e2b4a41..12a518e 100644 --- a/src/server/operandi_server/routers/user.py +++ b/src/server/operandi_server/routers/user.py @@ -59,6 +59,10 @@ def __init__(self): response_model=List, response_model_exclude_unset=True, response_model_exclude_none=True ) + def __del__(self): + if self.rmq_publisher: + self.rmq_publisher.disconnect() + async def user_login(self, auth: HTTPBasicCredentials = Depends(HTTPBasic())) -> PYUserAction: """ Used for user authentication.