From b1bf81c7cdf514a0b596e66e2f1d497b970a515e Mon Sep 17 00:00:00 2001 From: vvanglro Date: Wed, 14 Aug 2024 12:26:12 +0800 Subject: [PATCH 1/8] test(supervisors): ensure cleanup in test_multiprocess by using 'finally' Ensure proper cleanup in 'test_multiprocess' by wrapping the supervisor shutdown logic in a 'finally' block. This guarantees that the supervisor is properly terminated and all processes are joined, avoiding potential resource leaks or state residuebetween test runs. --- tests/supervisors/test_multiprocess.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/supervisors/test_multiprocess.py b/tests/supervisors/test_multiprocess.py index e1f594efe..022243388 100644 --- a/tests/supervisors/test_multiprocess.py +++ b/tests/supervisors/test_multiprocess.py @@ -86,12 +86,14 @@ def test_multiprocess_health_check() -> None: time.sleep(1) process = supervisor.processes[0] process.kill() - assert not process.is_alive() - time.sleep(1) - for p in supervisor.processes: - assert p.is_alive() - supervisor.signal_queue.append(signal.SIGINT) - supervisor.join_all() + try: + assert not process.is_alive() + time.sleep(1) + for p in supervisor.processes: + assert p.is_alive() + finally: + supervisor.signal_queue.append(signal.SIGINT) + supervisor.join_all() @new_console_in_windows From b4d8b09217650e80afd1ca16cf37711cf8023f1d Mon Sep 17 00:00:00 2001 From: vvanglro Date: Wed, 14 Aug 2024 17:18:08 +0800 Subject: [PATCH 2/8] test(supervisor): add timeout to multiprocess test subprocess A10-second timeout has been added to the subprocess call within the test_multiprocess.py to ensure it does not hang indefinitely in case of failures or slow systems. --- tests/supervisors/test_multiprocess.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/supervisors/test_multiprocess.py b/tests/supervisors/test_multiprocess.py index 022243388..2b642282b 100644 --- a/tests/supervisors/test_multiprocess.py +++ b/tests/supervisors/test_multiprocess.py @@ -34,6 +34,7 @@ def new_function(): "-c", f"from {module} import {name}; {name}.__wrapped__()", ], + timeout=10, creationflags=subprocess.CREATE_NO_WINDOW, # type: ignore[attr-defined] ) From a65c61ad1b61a0eb6a2e3fc692a20fe1993f9a18 Mon Sep 17 00:00:00 2001 From: vvanglro Date: Wed, 14 Aug 2024 17:24:43 +0800 Subject: [PATCH 3/8] try to solve --- tests/supervisors/test_multiprocess.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/supervisors/test_multiprocess.py b/tests/supervisors/test_multiprocess.py index 2b642282b..93953772d 100644 --- a/tests/supervisors/test_multiprocess.py +++ b/tests/supervisors/test_multiprocess.py @@ -84,12 +84,13 @@ def test_multiprocess_health_check() -> None: config = Config(app=app, workers=2) supervisor = Multiprocess(config, target=run, sockets=[]) threading.Thread(target=supervisor.run, daemon=True).start() - time.sleep(1) + time.sleep(0.5) process = supervisor.processes[0] process.kill() + time.sleep(0.5) try: assert not process.is_alive() - time.sleep(1) + time.sleep(0.5) for p in supervisor.processes: assert p.is_alive() finally: From d832a5a156f73898ec2e887ea96f9cd4e502f6af Mon Sep 17 00:00:00 2001 From: vvanglro Date: Wed, 14 Aug 2024 17:33:16 +0800 Subject: [PATCH 4/8] try to solve --- tests/supervisors/test_multiprocess.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/supervisors/test_multiprocess.py b/tests/supervisors/test_multiprocess.py index 93953772d..2f2a99540 100644 --- a/tests/supervisors/test_multiprocess.py +++ b/tests/supervisors/test_multiprocess.py @@ -84,13 +84,13 @@ def test_multiprocess_health_check() -> None: config = Config(app=app, workers=2) supervisor = Multiprocess(config, target=run, sockets=[]) threading.Thread(target=supervisor.run, daemon=True).start() - time.sleep(0.5) + time.sleep(1) process = supervisor.processes[0] process.kill() - time.sleep(0.5) + time.sleep(1) try: assert not process.is_alive() - time.sleep(0.5) + time.sleep(1) for p in supervisor.processes: assert p.is_alive() finally: From 27c696879038d7b0cbef86f68c183f6520a97189 Mon Sep 17 00:00:00 2001 From: vvanglro Date: Thu, 15 Aug 2024 14:43:48 +0800 Subject: [PATCH 5/8] feat: add retry for unstable tests --- requirements.txt | 1 + tests/supervisors/test_multiprocess.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index b3aba41c8..761ae30a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ twine==5.1.1 ruff==0.5.0 pytest==8.2.2 pytest-mock==3.14.0 +pytest-rerunfailures==14.0 mypy==1.10.1 types-click==7.1.8 types-pyyaml==6.0.12.20240311 diff --git a/tests/supervisors/test_multiprocess.py b/tests/supervisors/test_multiprocess.py index 2f2a99540..55421dd54 100644 --- a/tests/supervisors/test_multiprocess.py +++ b/tests/supervisors/test_multiprocess.py @@ -76,6 +76,7 @@ def test_multiprocess_run() -> None: supervisor.join_all() +@pytest.mark.flaky(reruns=3, reruns_delay=2) @new_console_in_windows def test_multiprocess_health_check() -> None: """ @@ -84,13 +85,12 @@ def test_multiprocess_health_check() -> None: config = Config(app=app, workers=2) supervisor = Multiprocess(config, target=run, sockets=[]) threading.Thread(target=supervisor.run, daemon=True).start() - time.sleep(1) + time.sleep(1) # ensure server is up process = supervisor.processes[0] process.kill() - time.sleep(1) try: - assert not process.is_alive() - time.sleep(1) + assert not process.is_alive(1) + time.sleep(1) # release gil, ensure process restart. for p in supervisor.processes: assert p.is_alive() finally: From f0d496ec230964300c5cd75ace01b56b6fcff4aa Mon Sep 17 00:00:00 2001 From: vvanglro Date: Thu, 15 Aug 2024 16:55:40 +0800 Subject: [PATCH 6/8] test(supervisor): remove retry, ensure process close. --- requirements.txt | 1 - tests/supervisors/test_multiprocess.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 761ae30a1..b3aba41c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,6 @@ twine==5.1.1 ruff==0.5.0 pytest==8.2.2 pytest-mock==3.14.0 -pytest-rerunfailures==14.0 mypy==1.10.1 types-click==7.1.8 types-pyyaml==6.0.12.20240311 diff --git a/tests/supervisors/test_multiprocess.py b/tests/supervisors/test_multiprocess.py index 55421dd54..08433594f 100644 --- a/tests/supervisors/test_multiprocess.py +++ b/tests/supervisors/test_multiprocess.py @@ -76,7 +76,6 @@ def test_multiprocess_run() -> None: supervisor.join_all() -@pytest.mark.flaky(reruns=3, reruns_delay=2) @new_console_in_windows def test_multiprocess_health_check() -> None: """ @@ -88,9 +87,10 @@ def test_multiprocess_health_check() -> None: time.sleep(1) # ensure server is up process = supervisor.processes[0] process.kill() + process.join() try: - assert not process.is_alive(1) - time.sleep(1) # release gil, ensure process restart. + assert not process.is_alive(0.5) + time.sleep(1.5) # release gil, ensure process restart complete. for p in supervisor.processes: assert p.is_alive() finally: From ffe43cb02977e5e655ad4739d9b1a32c4c217a00 Mon Sep 17 00:00:00 2001 From: vvanglro Date: Tue, 20 Aug 2024 20:28:56 +0800 Subject: [PATCH 7/8] test case --- tests/supervisors/test_multiprocess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/supervisors/test_multiprocess.py b/tests/supervisors/test_multiprocess.py index 08433594f..d4f6a8002 100644 --- a/tests/supervisors/test_multiprocess.py +++ b/tests/supervisors/test_multiprocess.py @@ -34,7 +34,6 @@ def new_function(): "-c", f"from {module} import {name}; {name}.__wrapped__()", ], - timeout=10, creationflags=subprocess.CREATE_NO_WINDOW, # type: ignore[attr-defined] ) @@ -84,7 +83,8 @@ def test_multiprocess_health_check() -> None: config = Config(app=app, workers=2) supervisor = Multiprocess(config, target=run, sockets=[]) threading.Thread(target=supervisor.run, daemon=True).start() - time.sleep(1) # ensure server is up + # Ensure server is up. + time.sleep(1) process = supervisor.processes[0] process.kill() process.join() From 0b1b372157ca5a6208a5b4fae535d2a50e79608e Mon Sep 17 00:00:00 2001 From: vvanglro Date: Thu, 21 Nov 2024 16:12:15 +0800 Subject: [PATCH 8/8] test(supervisors): improve test_multiprocess reliability - Replace fixed sleep with retry mechanism for process status checks- Enhance test stability by allowing for variable process startup times --- tests/supervisors/test_multiprocess.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/supervisors/test_multiprocess.py b/tests/supervisors/test_multiprocess.py index d4f6a8002..b94ae4049 100644 --- a/tests/supervisors/test_multiprocess.py +++ b/tests/supervisors/test_multiprocess.py @@ -90,9 +90,15 @@ def test_multiprocess_health_check() -> None: process.join() try: assert not process.is_alive(0.5) - time.sleep(1.5) # release gil, ensure process restart complete. - for p in supervisor.processes: - assert p.is_alive() + while retry := 3: + try: + for p in supervisor.processes: + assert p.is_alive() + except AssertionError: + retry -= 1 + time.sleep(0.5) + else: + break finally: supervisor.signal_queue.append(signal.SIGINT) supervisor.join_all()