From 69a01ed5a5662fd3e3bf3a0baeaa8f1746d90b98 Mon Sep 17 00:00:00 2001 From: weilycoder Date: Fri, 29 Nov 2024 08:26:23 +0800 Subject: [PATCH 01/10] Resolve end-of-line sequence issues --- cyaron/io.py | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/cyaron/io.py b/cyaron/io.py index bbfa7f9..73363d1 100644 --- a/cyaron/io.py +++ b/cyaron/io.py @@ -243,7 +243,9 @@ def input_clear_content(self, pos: int = 0): self.__clear(self.input_file, pos) - def output_gen(self, shell_cmd, time_limit=None): + def output_gen( + self, shell_cmd: str, time_limit: float = None, replace_EOL: bool = True + ): """ Run the command `shell_cmd` (usually the std program) and send it the input file as stdin. Write its output to the output file. @@ -251,31 +253,34 @@ def output_gen(self, shell_cmd, time_limit=None): shell_cmd: the command to run, usually the std program. time_limit: the time limit (seconds) of the command to run. None means infinity. Defaults to None. + replace_EOL: Set whether to replace the end-of-line sequence with `'\\n'`. + Defaults to True. """ if self.output_file is None: raise ValueError("Output file is disabled") self.flush_buffer() origin_pos = self.input_file.tell() self.input_file.seek(0) - if time_limit is not None: - subprocess.check_call( - shell_cmd, - shell=True, - timeout=time_limit, - stdin=self.input_file.fileno(), - stdout=self.output_file.fileno(), - universal_newlines=True, - ) - else: - subprocess.check_call( - shell_cmd, - shell=True, - stdin=self.input_file.fileno(), - stdout=self.output_file.fileno(), - universal_newlines=True, - ) + + output_fileno = self.output_file.fileno() + if replace_EOL: + temp_outfile = tempfile.TemporaryFile("w+") + output_fileno = temp_outfile.fileno() + + subprocess.check_call( + shell_cmd, + shell=True, + timeout=time_limit, + stdin=self.input_file.fileno(), + stdout=output_fileno, + universal_newlines=True, + ) self.input_file.seek(origin_pos) + if replace_EOL: + temp_outfile.seek(0) + self.output_file.write(temp_outfile.read()) + log.debug(self.output_filename, " done") def output_write(self, *args, **kwargs): From 60f93ba905189d55f3ebf84421472c3183635b2f Mon Sep 17 00:00:00 2001 From: weilycoder Date: Mon, 2 Dec 2024 16:01:13 +0800 Subject: [PATCH 02/10] Add a buffer --- cyaron/io.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cyaron/io.py b/cyaron/io.py index 73363d1..0fafadc 100644 --- a/cyaron/io.py +++ b/cyaron/io.py @@ -244,7 +244,7 @@ def input_clear_content(self, pos: int = 0): self.__clear(self.input_file, pos) def output_gen( - self, shell_cmd: str, time_limit: float = None, replace_EOL: bool = True + self, shell_cmd: str, time_limit: float = None, *, replace_EOL: bool = True ): """ Run the command `shell_cmd` (usually the std program) and send it the input file as stdin. @@ -279,7 +279,11 @@ def output_gen( if replace_EOL: temp_outfile.seek(0) - self.output_file.write(temp_outfile.read()) + buf = temp_outfile.read(65536) + while buf != '': + self.output_file.write(buf) + buf = temp_outfile.read(65536) + temp_outfile.close() log.debug(self.output_filename, " done") From 9b2174efa3b3d73515af68de1605c23118995198 Mon Sep 17 00:00:00 2001 From: weilycoder Date: Mon, 2 Dec 2024 17:14:05 +0800 Subject: [PATCH 03/10] Add test --- cyaron/tests/io_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cyaron/tests/io_test.py b/cyaron/tests/io_test.py index 42cb996..485f29e 100644 --- a/cyaron/tests/io_test.py +++ b/cyaron/tests/io_test.py @@ -68,9 +68,9 @@ def test_output_gen(self): with IO("test_gen.in", "test_gen.out") as test: test.output_gen("echo 233") - with open("test_gen.out") as f: + with open("test_gen.out", "rb") as f: output = f.read() - self.assertEqual(output.strip("\n"), "233") + self.assertEqual(output.strip(b"\n"), b"233") def test_output_gen_time_limit_exceeded(self): time_limit_exceeded = False From d814892e42fc736340cc3ab9628a9b5914c137ee Mon Sep 17 00:00:00 2001 From: weilycoder Date: Mon, 2 Dec 2024 22:25:21 +0800 Subject: [PATCH 04/10] Fix type annotations --- cyaron/io.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cyaron/io.py b/cyaron/io.py index 0fafadc..f3f4558 100644 --- a/cyaron/io.py +++ b/cyaron/io.py @@ -8,7 +8,7 @@ import re import subprocess import tempfile -from typing import Union, overload, Optional +from typing import Union, overload, Optional, List from io import IOBase from . import log from .utils import list_like, make_unicode @@ -244,7 +244,11 @@ def input_clear_content(self, pos: int = 0): self.__clear(self.input_file, pos) def output_gen( - self, shell_cmd: str, time_limit: float = None, *, replace_EOL: bool = True + self, + shell_cmd: Union[str, List[str]], + time_limit: float = None, + *, + replace_EOL: bool = True ): """ Run the command `shell_cmd` (usually the std program) and send it the input file as stdin. @@ -280,7 +284,7 @@ def output_gen( if replace_EOL: temp_outfile.seek(0) buf = temp_outfile.read(65536) - while buf != '': + while buf != "": self.output_file.write(buf) buf = temp_outfile.read(65536) temp_outfile.close() From 0942353ab3b827c2929369452980fb594d577d05 Mon Sep 17 00:00:00 2001 From: weilycoder Date: Wed, 4 Dec 2024 22:25:20 +0800 Subject: [PATCH 05/10] Rewrite output_gen --- .gitignore | 1 + cyaron/io.py | 122 ++++++++++++++++++++++++++++----------------------- 2 files changed, 69 insertions(+), 54 deletions(-) diff --git a/.gitignore b/.gitignore index bf4997a..76f1197 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ config.py +*.cpp *.in *.out *.exe diff --git a/cyaron/io.py b/cyaron/io.py index f3f4558..49c1a08 100644 --- a/cyaron/io.py +++ b/cyaron/io.py @@ -3,6 +3,7 @@ Classes: IO: IO tool class. It will process the input and output files. """ + from __future__ import absolute_import import os import re @@ -18,34 +19,37 @@ class IO: """IO tool class. It will process the input and output files.""" @overload - def __init__(self, - input_file: Optional[Union[IOBase, str, int]] = None, - output_file: Optional[Union[IOBase, str, int]] = None, - data_id: Optional[int] = None, - disable_output: bool = False, - make_dirs: bool = False): - ... + def __init__( + self, + input_file: Optional[Union[IOBase, str, int]] = None, + output_file: Optional[Union[IOBase, str, int]] = None, + data_id: Optional[int] = None, + disable_output: bool = False, + make_dirs: bool = False, + ): ... @overload - def __init__(self, - data_id: Optional[int] = None, - file_prefix: Optional[str] = None, - input_suffix: str = '.in', - output_suffix: str = '.out', - disable_output: bool = False, - make_dirs: bool = False): - ... + def __init__( + self, + data_id: Optional[int] = None, + file_prefix: Optional[str] = None, + input_suffix: str = ".in", + output_suffix: str = ".out", + disable_output: bool = False, + make_dirs: bool = False, + ): ... def __init__( # type: ignore - self, - input_file: Optional[Union[IOBase, str, int]] = None, - output_file: Optional[Union[IOBase, str, int]] = None, - data_id: Optional[int] = None, - file_prefix: Optional[str] = None, - input_suffix: str = '.in', - output_suffix: str = '.out', - disable_output: bool = False, - make_dirs: bool = False): + self, + input_file: Optional[Union[IOBase, str, int]] = None, + output_file: Optional[Union[IOBase, str, int]] = None, + data_id: Optional[int] = None, + file_prefix: Optional[str] = None, + input_suffix: str = ".in", + output_suffix: str = ".out", + disable_output: bool = False, + make_dirs: bool = False, + ): """ Args: input_file (optional): input file object or filename or file descriptor. @@ -87,11 +91,12 @@ def __init__( # type: ignore self.input_file, self.output_file = None, None if file_prefix is not None: # legacy mode - input_file = '{}{{}}{}'.format(self.__escape_format(file_prefix), - self.__escape_format(input_suffix)) - output_file = '{}{{}}{}'.format( - self.__escape_format(file_prefix), - self.__escape_format(output_suffix)) + input_file = "{}{{}}{}".format( + self.__escape_format(file_prefix), self.__escape_format(input_suffix) + ) + output_file = "{}{{}}{}".format( + self.__escape_format(file_prefix), self.__escape_format(output_suffix) + ) self.input_filename, self.output_filename = None, None self.__input_temp, self.__output_temp = False, False self.__init_file(input_file, data_id, "i", make_dirs) @@ -101,9 +106,13 @@ def __init__( # type: ignore self.output_file = None self.is_first_char = {} - def __init_file(self, f: Union[IOBase, str, int, - None], data_id: Union[int, None], - file_type: str, make_dirs: bool): + def __init_file( + self, + f: Union[IOBase, str, int, None], + data_id: Union[int, None], + file_type: str, + make_dirs: bool, + ): if isinstance(f, IOBase): # consider ``f`` as a file object if file_type == "i": @@ -112,8 +121,12 @@ def __init_file(self, f: Union[IOBase, str, int, self.output_file = f elif isinstance(f, int): # consider ``f`` as a file descor - self.__init_file(open(f, 'w+', encoding="utf-8", newline='\n'), - data_id, file_type, make_dirs) + self.__init_file( + open(f, "w+", encoding="utf-8", newline="\n"), + data_id, + file_type, + make_dirs, + ) elif f is None: # consider wanna temp file fd, self.input_filename = tempfile.mkstemp() @@ -133,8 +146,11 @@ def __init_file(self, f: Union[IOBase, str, int, else: self.output_filename = filename self.__init_file( - open(filename, 'w+', newline='\n', encoding='utf-8'), data_id, - file_type, make_dirs) + open(filename, "w+", newline="\n", encoding="utf-8"), + data_id, + file_type, + make_dirs, + ) def __escape_format(self, st: str): """replace "{}" to "{{}}" """ @@ -266,28 +282,26 @@ def output_gen( origin_pos = self.input_file.tell() self.input_file.seek(0) - output_fileno = self.output_file.fileno() - if replace_EOL: - temp_outfile = tempfile.TemporaryFile("w+") - output_fileno = temp_outfile.fileno() - - subprocess.check_call( + proc = subprocess.Popen( shell_cmd, shell=True, - timeout=time_limit, - stdin=self.input_file.fileno(), - stdout=output_fileno, - universal_newlines=True, + stdin=self.input_file, + stdout=subprocess.PIPE, + universal_newlines=replace_EOL, ) - self.input_file.seek(origin_pos) - if replace_EOL: - temp_outfile.seek(0) - buf = temp_outfile.read(65536) - while buf != "": - self.output_file.write(buf) - buf = temp_outfile.read(65536) - temp_outfile.close() + try: + output, _ = proc.communicate(timeout=time_limit) + except subprocess.TimeoutExpired: + proc.kill() + raise + else: + if replace_EOL: + self.output_file.write(output) + else: + os.write(self.output_file.fileno(), output) + finally: + self.input_file.seek(origin_pos) log.debug(self.output_filename, " done") From 55c57f46fca7360ce989ca1f7e0e5fb21a244b98 Mon Sep 17 00:00:00 2001 From: weilycoder Date: Wed, 4 Dec 2024 22:42:31 +0800 Subject: [PATCH 06/10] Ensure all child processes are terminated --- cyaron/io.py | 20 +++++++++++++++++++- poetry.lock | 34 ++++++++++++++++++++++++++++++++-- pyproject.toml | 1 + 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/cyaron/io.py b/cyaron/io.py index 49c1a08..6bb4cf4 100644 --- a/cyaron/io.py +++ b/cyaron/io.py @@ -9,6 +9,7 @@ import re import subprocess import tempfile +import psutil from typing import Union, overload, Optional, List from io import IOBase from . import log @@ -227,6 +228,22 @@ def __clear(self, file: IOBase, pos: int = 0): self.is_first_char[file] = True file.seek(pos) + @staticmethod + def _kill_process_and_children(pid: int): + try: + parent = psutil.Process(pid) + while True: + children = parent.children() + if not children: + break + for child in children: + IO._kill_process_and_children(child.pid) + parent.kill() + except psutil.NoSuchProcess: + pass + except psutil.AccessDenied: + pass + def input_write(self, *args, **kwargs): """ Write every element in *args into the input file. Splits with `separator`. @@ -293,7 +310,8 @@ def output_gen( try: output, _ = proc.communicate(timeout=time_limit) except subprocess.TimeoutExpired: - proc.kill() + # proc.kill() # didn't work because `shell=True`. + self._kill_process_and_children(proc.pid) raise else: if replace_EOL: diff --git a/poetry.lock b/poetry.lock index b1294ef..7b703db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "colorama" @@ -25,6 +25,36 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "psutil" +version = "6.1.0" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0"}, + {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:c0e0c00aa18ca2d3b2b991643b799a15fc8f0563d2ebb6040f64ce8dc027b942"}, + {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:000d1d1ebd634b4efb383f4034437384e44a6d455260aaee2eca1e9c1b55f047"}, + {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5cd2bcdc75b452ba2e10f0e8ecc0b57b827dd5d7aaffbc6821b2a9a242823a76"}, + {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:045f00a43c737f960d273a83973b2511430d61f283a44c96bf13a6e829ba8fdc"}, + {file = "psutil-6.1.0-cp27-none-win32.whl", hash = "sha256:9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e"}, + {file = "psutil-6.1.0-cp27-none-win_amd64.whl", hash = "sha256:a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85"}, + {file = "psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688"}, + {file = "psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a"}, + {file = "psutil-6.1.0-cp36-cp36m-win32.whl", hash = "sha256:6d3fbbc8d23fcdcb500d2c9f94e07b1342df8ed71b948a2649b5cb060a7c94ca"}, + {file = "psutil-6.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1209036fbd0421afde505a4879dee3b2fd7b1e14fee81c0069807adcbbcca747"}, + {file = "psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e"}, + {file = "psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"}, + {file = "psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"}, +] + +[package.extras] +dev = ["black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "wheel"] +test = ["pytest", "pytest-xdist", "setuptools"] + [[package]] name = "xeger" version = "0.4.0" @@ -38,4 +68,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.6" -content-hash = "f61b42d8bd0c6814638b0f4d9b5afa1b049f7ab03fb55dbdceaceba39936d21c" +content-hash = "3b7d7552aab0b4bfb77c96178a91816d9a53ba6eda49c7bd46f008583f7c3ae0" diff --git a/pyproject.toml b/pyproject.toml index 59ae302..6c8b023 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ readme = "README.md" python = ">=3.6" xeger = "^0.4.0" colorful = "^0.5.6" +psutil = "^6.1.0" [build-system] From 39a6dddc2132c17f7237b142deb82b1d3bfae9da Mon Sep 17 00:00:00 2001 From: weilycoder Date: Wed, 4 Dec 2024 22:58:52 +0800 Subject: [PATCH 07/10] Fix test --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 581148d..4e4efc5 100644 --- a/tox.ini +++ b/tox.ini @@ -7,5 +7,6 @@ isolated_build = true deps = xeger colorful + psutil commands = python unit_test.py allowlist_externals = poetry From cf182b9b584c461b6f56e818cb8ee79347962bc5 Mon Sep 17 00:00:00 2001 From: weilycoder Date: Fri, 6 Dec 2024 09:09:39 +0800 Subject: [PATCH 08/10] Rewrite output_gen test --- cyaron/tests/io_test.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/cyaron/tests/io_test.py b/cyaron/tests/io_test.py index 485f29e..7757db2 100644 --- a/cyaron/tests/io_test.py +++ b/cyaron/tests/io_test.py @@ -1,5 +1,7 @@ import unittest +import sys import os +import time import shutil import tempfile import subprocess @@ -73,33 +75,33 @@ def test_output_gen(self): self.assertEqual(output.strip(b"\n"), b"233") def test_output_gen_time_limit_exceeded(self): - time_limit_exceeded = False with captured_output() as (out, err): with open("long_time.py", "w") as f: - f.write("import time\ntime.sleep(10)\nprint(1)") + f.write("import time, os\nfn = input()\ntime.sleep(0.5)\nos.remove(fn)\n") - try: - with IO("test_gen.in", "test_gen.out") as test: - test.output_gen("python long_time.py", time_limit=1) - except subprocess.TimeoutExpired: - time_limit_exceeded = True - self.assertEqual(time_limit_exceeded, True) + with IO("test_gen.in", "test_gen.out") as test: + fd, input_filename = tempfile.mkstemp() + os.close(fd) + abs_input_filename: str = os.path.abspath(input_filename) + with self.assertRaises(subprocess.TimeoutExpired): + test.input_writeln(abs_input_filename) + test.output_gen(f'"{sys.executable}" long_time.py', time_limit=0.1) + time.sleep(0.5) + try: + os.remove(input_filename) + except FileNotFoundError: + raise RuntimeError("Child processes have not been terminated.") from None def test_output_gen_time_limit_not_exceeded(self): - time_limit_exceeded = False with captured_output() as (out, err): with open("short_time.py", "w") as f: - f.write("import time\ntime.sleep(0.2)\nprint(1)") + f.write("import time\ntime.sleep(0.1)\nprint(1)") - try: - with IO("test_gen.in", "test_gen.out") as test: - test.output_gen("python short_time.py", time_limit=1) - except subprocess.TimeoutExpired: - time_limit_exceeded = True + with IO("test_gen.in", "test_gen.out") as test: + test.output_gen(f'"{sys.executable}" short_time.py', time_limit=0.5) with open("test_gen.out") as f: output = f.read() self.assertEqual(output.strip("\n"), "1") - self.assertEqual(time_limit_exceeded, False) def test_init_overload(self): with IO(file_prefix="data{", data_id=5) as test: From f31496524f95ead46eb968ee3974e25005093634 Mon Sep 17 00:00:00 2001 From: "Mr. Python" <2789762371@qq.com> Date: Mon, 9 Dec 2024 16:48:59 +0800 Subject: [PATCH 09/10] Fix type hints; fix some io test --- .pylintrc | 2 +- cyaron/io.py | 33 +++++++++++++++++---------------- cyaron/tests/io_test.py | 33 ++++++++++++++++++++------------- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/.pylintrc b/.pylintrc index de2c0b1..61e0596 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,3 +1,3 @@ [MASTER] -py-version=3.5 +py-version=3.6 disable=R0902,R0903,R0913,R0917,R0912 \ No newline at end of file diff --git a/cyaron/io.py b/cyaron/io.py index 6bb4cf4..bf8fadc 100644 --- a/cyaron/io.py +++ b/cyaron/io.py @@ -27,7 +27,8 @@ def __init__( data_id: Optional[int] = None, disable_output: bool = False, make_dirs: bool = False, - ): ... + ): + ... @overload def __init__( @@ -38,7 +39,8 @@ def __init__( output_suffix: str = ".out", disable_output: bool = False, make_dirs: bool = False, - ): ... + ): + ... def __init__( # type: ignore self, @@ -89,15 +91,14 @@ def __init__( # type: ignore # if the dir "./io" not found it will be created """ self.__closed = False - self.input_file, self.output_file = None, None + self.output_file = None if file_prefix is not None: # legacy mode - input_file = "{}{{}}{}".format( - self.__escape_format(file_prefix), self.__escape_format(input_suffix) - ) + input_file = "{}{{}}{}".format(self.__escape_format(file_prefix), + self.__escape_format(input_suffix)) output_file = "{}{{}}{}".format( - self.__escape_format(file_prefix), self.__escape_format(output_suffix) - ) + self.__escape_format(file_prefix), + self.__escape_format(output_suffix)) self.input_filename, self.output_filename = None, None self.__input_temp, self.__output_temp = False, False self.__init_file(input_file, data_id, "i", make_dirs) @@ -276,13 +277,11 @@ def input_clear_content(self, pos: int = 0): self.__clear(self.input_file, pos) - def output_gen( - self, - shell_cmd: Union[str, List[str]], - time_limit: float = None, - *, - replace_EOL: bool = True - ): + def output_gen(self, + shell_cmd: Union[str, List[str]], + time_limit: Optional[float] = None, + *, + replace_EOL: bool = True): """ Run the command `shell_cmd` (usually the std program) and send it the input file as stdin. Write its output to the output file. @@ -302,7 +301,7 @@ def output_gen( proc = subprocess.Popen( shell_cmd, shell=True, - stdin=self.input_file, + stdin=self.input_file.fileno(), stdout=subprocess.PIPE, universal_newlines=replace_EOL, ) @@ -354,6 +353,8 @@ def output_clear_content(self, pos: int = 0): Args: pos: Where file will truncate """ + if self.output_file is None: + raise ValueError("Output file is disabled") self.__clear(self.output_file, pos) def flush_buffer(self): diff --git a/cyaron/tests/io_test.py b/cyaron/tests/io_test.py index 7757db2..86330dc 100644 --- a/cyaron/tests/io_test.py +++ b/cyaron/tests/io_test.py @@ -72,12 +72,15 @@ def test_output_gen(self): with open("test_gen.out", "rb") as f: output = f.read() - self.assertEqual(output.strip(b"\n"), b"233") + self.assertEqual(output, b"233\n") def test_output_gen_time_limit_exceeded(self): - with captured_output() as (out, err): - with open("long_time.py", "w") as f: - f.write("import time, os\nfn = input()\ntime.sleep(0.5)\nos.remove(fn)\n") + with captured_output(): + with open("long_time.py", "w", encoding="utf-8") as f: + f.write("import time, os\n" + "fn = input()\n" + "time.sleep(0.1)\n" + "os.remove(fn)\n") with IO("test_gen.in", "test_gen.out") as test: fd, input_filename = tempfile.mkstemp() @@ -85,23 +88,27 @@ def test_output_gen_time_limit_exceeded(self): abs_input_filename: str = os.path.abspath(input_filename) with self.assertRaises(subprocess.TimeoutExpired): test.input_writeln(abs_input_filename) - test.output_gen(f'"{sys.executable}" long_time.py', time_limit=0.1) - time.sleep(0.5) + test.output_gen(f'"{sys.executable}" long_time.py', + time_limit=0.05) + time.sleep(0.1) try: os.remove(input_filename) except FileNotFoundError: - raise RuntimeError("Child processes have not been terminated.") from None + self.fail("Child processes have not been terminated.") def test_output_gen_time_limit_not_exceeded(self): - with captured_output() as (out, err): - with open("short_time.py", "w") as f: - f.write("import time\ntime.sleep(0.1)\nprint(1)") + with captured_output(): + with open("short_time.py", "w", encoding="utf-8") as f: + f.write("import time\n" + "time.sleep(0.1)\n" + "print(1)") with IO("test_gen.in", "test_gen.out") as test: - test.output_gen(f'"{sys.executable}" short_time.py', time_limit=0.5) - with open("test_gen.out") as f: + test.output_gen(f'"{sys.executable}" short_time.py', + time_limit=0.5) + with open("test_gen.out", encoding="utf-8") as f: output = f.read() - self.assertEqual(output.strip("\n"), "1") + self.assertEqual(output, "1\n") def test_init_overload(self): with IO(file_prefix="data{", data_id=5) as test: From a459de99f21a019941385d7189e710e26d724d80 Mon Sep 17 00:00:00 2001 From: weilycoder Date: Mon, 9 Dec 2024 22:45:07 +0800 Subject: [PATCH 10/10] Use a simpler and more straightforward method to terminate the process tree --- cyaron/io.py | 31 +++++++++++++------------------ cyaron/tests/io_test.py | 13 ++++++------- poetry.lock | 32 +------------------------------- pyproject.toml | 1 - tox.ini | 1 - 5 files changed, 20 insertions(+), 58 deletions(-) diff --git a/cyaron/io.py b/cyaron/io.py index bf8fadc..3bee2ee 100644 --- a/cyaron/io.py +++ b/cyaron/io.py @@ -7,10 +7,10 @@ from __future__ import absolute_import import os import re +import signal import subprocess import tempfile -import psutil -from typing import Union, overload, Optional, List +from typing import Union, overload, Optional, List, cast from io import IOBase from . import log from .utils import list_like, make_unicode @@ -91,6 +91,7 @@ def __init__( # type: ignore # if the dir "./io" not found it will be created """ self.__closed = False + self.input_file = cast(IOBase, None) self.output_file = None if file_prefix is not None: # legacy mode @@ -230,20 +231,13 @@ def __clear(self, file: IOBase, pos: int = 0): file.seek(pos) @staticmethod - def _kill_process_and_children(pid: int): - try: - parent = psutil.Process(pid) - while True: - children = parent.children() - if not children: - break - for child in children: - IO._kill_process_and_children(child.pid) - parent.kill() - except psutil.NoSuchProcess: - pass - except psutil.AccessDenied: - pass + def _kill_process_and_children(proc: subprocess.Popen): + if os.name == "posix": + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + elif os.name == "nt": + os.system(f"TASKKILL /F /T /PID {proc.pid} > nul") + else: + proc.kill() # Not currently supported def input_write(self, *args, **kwargs): """ @@ -304,13 +298,14 @@ def output_gen(self, stdin=self.input_file.fileno(), stdout=subprocess.PIPE, universal_newlines=replace_EOL, + preexec_fn=os.setsid if os.name == "posix" else None, ) try: output, _ = proc.communicate(timeout=time_limit) except subprocess.TimeoutExpired: - # proc.kill() # didn't work because `shell=True`. - self._kill_process_and_children(proc.pid) + # proc.kill() # didn't work because `shell=True`. + self._kill_process_and_children(proc) raise else: if replace_EOL: diff --git a/cyaron/tests/io_test.py b/cyaron/tests/io_test.py index 86330dc..6a0032d 100644 --- a/cyaron/tests/io_test.py +++ b/cyaron/tests/io_test.py @@ -76,10 +76,12 @@ def test_output_gen(self): def test_output_gen_time_limit_exceeded(self): with captured_output(): + TIMEOUT = 0.02 + WAIT_TIME = 0.4 # If the wait time is too short, an error may occur with open("long_time.py", "w", encoding="utf-8") as f: f.write("import time, os\n" "fn = input()\n" - "time.sleep(0.1)\n" + f"time.sleep({WAIT_TIME})\n" "os.remove(fn)\n") with IO("test_gen.in", "test_gen.out") as test: @@ -89,8 +91,8 @@ def test_output_gen_time_limit_exceeded(self): with self.assertRaises(subprocess.TimeoutExpired): test.input_writeln(abs_input_filename) test.output_gen(f'"{sys.executable}" long_time.py', - time_limit=0.05) - time.sleep(0.1) + time_limit=TIMEOUT) + time.sleep(WAIT_TIME) try: os.remove(input_filename) except FileNotFoundError: @@ -133,10 +135,7 @@ def test_make_dirs(self): mkdir_false = False try: - with IO( - "./automkdir_false/data.in", - "./automkdir_false/data.out", - ): + with IO("./automkdir_false/data.in", "./automkdir_false/data.out"): pass except FileNotFoundError: mkdir_false = True diff --git a/poetry.lock b/poetry.lock index 7b703db..e9bf565 100644 --- a/poetry.lock +++ b/poetry.lock @@ -25,36 +25,6 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -[[package]] -name = "psutil" -version = "6.1.0" -description = "Cross-platform lib for process and system monitoring in Python." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" -files = [ - {file = "psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0"}, - {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:c0e0c00aa18ca2d3b2b991643b799a15fc8f0563d2ebb6040f64ce8dc027b942"}, - {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:000d1d1ebd634b4efb383f4034437384e44a6d455260aaee2eca1e9c1b55f047"}, - {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5cd2bcdc75b452ba2e10f0e8ecc0b57b827dd5d7aaffbc6821b2a9a242823a76"}, - {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:045f00a43c737f960d273a83973b2511430d61f283a44c96bf13a6e829ba8fdc"}, - {file = "psutil-6.1.0-cp27-none-win32.whl", hash = "sha256:9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e"}, - {file = "psutil-6.1.0-cp27-none-win_amd64.whl", hash = "sha256:a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85"}, - {file = "psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688"}, - {file = "psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e"}, - {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38"}, - {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b"}, - {file = "psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a"}, - {file = "psutil-6.1.0-cp36-cp36m-win32.whl", hash = "sha256:6d3fbbc8d23fcdcb500d2c9f94e07b1342df8ed71b948a2649b5cb060a7c94ca"}, - {file = "psutil-6.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1209036fbd0421afde505a4879dee3b2fd7b1e14fee81c0069807adcbbcca747"}, - {file = "psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e"}, - {file = "psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"}, - {file = "psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"}, -] - -[package.extras] -dev = ["black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "wheel"] -test = ["pytest", "pytest-xdist", "setuptools"] - [[package]] name = "xeger" version = "0.4.0" @@ -68,4 +38,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.6" -content-hash = "3b7d7552aab0b4bfb77c96178a91816d9a53ba6eda49c7bd46f008583f7c3ae0" +content-hash = "f61b42d8bd0c6814638b0f4d9b5afa1b049f7ab03fb55dbdceaceba39936d21c" diff --git a/pyproject.toml b/pyproject.toml index 6c8b023..59ae302 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ readme = "README.md" python = ">=3.6" xeger = "^0.4.0" colorful = "^0.5.6" -psutil = "^6.1.0" [build-system] diff --git a/tox.ini b/tox.ini index 4e4efc5..581148d 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,5 @@ isolated_build = true deps = xeger colorful - psutil commands = python unit_test.py allowlist_externals = poetry