From f782e61ff3bc332a8e6ee09ed450e4c62c01048d Mon Sep 17 00:00:00 2001 From: Bukosabino Date: Wed, 22 May 2024 14:24:34 +0200 Subject: [PATCH 1/2] Handling exceptions for scheduled jobs --- src/etls/boa/load.py | 2 ++ src/etls/bocm/load.py | 2 ++ src/etls/boe/load.py | 2 ++ src/etls/boja/load.py | 2 ++ src/etls/bopv/load.py | 2 ++ src/etls/bopz/load.py | 2 ++ src/etls/utils.py | 16 ++++++++++++++++ 7 files changed, 28 insertions(+) create mode 100644 src/etls/utils.py diff --git a/src/etls/boa/load.py b/src/etls/boa/load.py index 7ad298c..83dc127 100644 --- a/src/etls/boa/load.py +++ b/src/etls/boa/load.py @@ -5,6 +5,7 @@ from src.email.send_email import send_email from src.etls.boa.scrapper import BOAScrapper +from src.etls.utils import catch_exceptions from src.etls.boa.defs import COLLECTION_NAME from src.etls.common.etl import ETL from src.initialize import initialize_app @@ -12,6 +13,7 @@ app = typer.Typer() @app.command() +@catch_exceptions(cancel_on_failure=True) def today(init_objects=None): if init_objects is None: init_objects = initialize_app() diff --git a/src/etls/bocm/load.py b/src/etls/bocm/load.py index 7326503..7ac96e5 100644 --- a/src/etls/bocm/load.py +++ b/src/etls/bocm/load.py @@ -5,6 +5,7 @@ from src.email.send_email import send_email from src.etls.bocm.scrapper import BOCMScrapper from src.etls.common.etl import ETL +from src.etls.utils import catch_exceptions from src.etls.bocm.defs import COLLECTION_NAME from src.initialize import initialize_app @@ -13,6 +14,7 @@ @app.command() +@catch_exceptions(cancel_on_failure=True) def today(init_objects=None): if init_objects is None: init_objects = initialize_app() diff --git a/src/etls/boe/load.py b/src/etls/boe/load.py index 9282e75..ed149db 100644 --- a/src/etls/boe/load.py +++ b/src/etls/boe/load.py @@ -5,6 +5,7 @@ from src.email.send_email import send_email from src.etls.boe.scrapper import BOEScrapper from src.etls.common.etl import ETL +from src.etls.utils import catch_exceptions from src.etls.boe.defs import COLLECTION_NAME from src.initialize import initialize_app @@ -13,6 +14,7 @@ @app.command() +@catch_exceptions(cancel_on_failure=True) def today(init_objects=None): if init_objects is None: init_objects = initialize_app() diff --git a/src/etls/boja/load.py b/src/etls/boja/load.py index 2e98d4c..98eaebe 100644 --- a/src/etls/boja/load.py +++ b/src/etls/boja/load.py @@ -4,6 +4,7 @@ from src.email.send_email import send_email from src.etls.boja.scrapper import BOJAScrapper +from src.etls.utils import catch_exceptions from src.etls.boja.defs import COLLECTION_NAME from src.etls.common.etl import ETL from src.initialize import initialize_app @@ -11,6 +12,7 @@ app = typer.Typer() @app.command() +@catch_exceptions(cancel_on_failure=True) def today(init_objects=None): if init_objects is None: init_objects = initialize_app() diff --git a/src/etls/bopv/load.py b/src/etls/bopv/load.py index e8f6305..a28976e 100644 --- a/src/etls/bopv/load.py +++ b/src/etls/bopv/load.py @@ -6,12 +6,14 @@ from src.email.send_email import send_email from src.etls.bopv.scrapper import BOPVScrapper from src.etls.bopv.defs import COLLECTION_NAME +from src.etls.utils import catch_exceptions from src.etls.common.etl import ETL from src.initialize import initialize_app app = typer.Typer() @app.command() +@catch_exceptions(cancel_on_failure=True) def today(init_objects=None): if init_objects is None: init_objects = initialize_app() diff --git a/src/etls/bopz/load.py b/src/etls/bopz/load.py index 63ab122..78a886c 100644 --- a/src/etls/bopz/load.py +++ b/src/etls/bopz/load.py @@ -5,6 +5,7 @@ from src.email.send_email import send_email from src.etls.bopz.scrapper import BOPZScrapper from src.etls.bopz.defs import COLLECTION_NAME +from src.etls.utils import catch_exceptions from src.etls.common.etl import ETL from src.initialize import initialize_app @@ -13,6 +14,7 @@ @app.command() +@catch_exceptions(cancel_on_failure=True) def today(init_objects=None): if init_objects is None: init_objects = initialize_app() diff --git a/src/etls/utils.py b/src/etls/utils.py new file mode 100644 index 0000000..6a52c85 --- /dev/null +++ b/src/etls/utils.py @@ -0,0 +1,16 @@ +import functools + + +def catch_exceptions(cancel_on_failure=False): + def catch_exceptions_decorator(job_func): + @functools.wraps(job_func) + def wrapper(*args, **kwargs): + try: + return job_func(*args, **kwargs) + except: + import traceback + print(traceback.format_exc()) + if cancel_on_failure: + return schedule.CancelJob + return wrapper + return catch_exceptions_decorator From 5102130f5306901e345d15a763f22be4db158741 Mon Sep 17 00:00:00 2001 From: Bukosabino Date: Wed, 22 May 2024 14:27:14 +0200 Subject: [PATCH 2/2] Handling exceptions for scheduled jobs --- src/etls/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/etls/utils.py b/src/etls/utils.py index 6a52c85..718160a 100644 --- a/src/etls/utils.py +++ b/src/etls/utils.py @@ -1,5 +1,7 @@ import functools +import schedule + def catch_exceptions(cancel_on_failure=False): def catch_exceptions_decorator(job_func):