Skip to content

Commit

Permalink
Add traceback log output when sigterm was sent
Browse files Browse the repository at this point in the history
  • Loading branch information
Ulada Zakharava committed Dec 12, 2024
1 parent 42dfa7e commit 42d3d71
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 0 deletions.
2 changes: 2 additions & 0 deletions airflow/models/taskinstance.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import operator
import os
import signal
import traceback
from collections import defaultdict
from collections.abc import Collection, Generator, Iterable, Mapping
from datetime import timedelta
Expand Down Expand Up @@ -2810,6 +2811,7 @@ def signal_handler(signum, frame):
os._exit(1)
return
self.log.error("Received SIGTERM. Terminating subprocesses.")
self.log.error("Stacktrace: \n%s", "".join(traceback.format_stack()))
self.task.on_kill()
raise AirflowTaskTerminated("Task received SIGTERM signal")

Expand Down
21 changes: 21 additions & 0 deletions tests/models/test_taskinstance.py
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,27 @@ def task_function(ti):
ti.run()
assert "on_failure_callback called" in caplog.text

def test_task_sigterm_calls_with_traceback_in_logs(self, dag_maker, caplog):
"""
Test that ensures that tasks print traceback to the logs when they receive sigterm
"""

def task_function(ti):
os.kill(ti.pid, signal.SIGTERM)

with dag_maker():
task_ = PythonOperator(
task_id="test_on_failure",
python_callable=task_function,
)

dr = dag_maker.create_dagrun()
ti = dr.task_instances[0]
ti.task = task_
with pytest.raises(AirflowTaskTerminated):
ti.run()
assert "Stacktrace: " in caplog.text

def test_task_sigterm_works_with_retries(self, dag_maker):
"""
Test that ensures that tasks are retried when they receive sigterm
Expand Down

0 comments on commit 42d3d71

Please sign in to comment.