diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 693a9017..93d972e7 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -13,6 +13,8 @@ jobs: - 3.8 - 3.9 - '3.10' + - 3.11 + - 3.12 test-dir: - client - server diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 14e81ed0..46badc1b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,9 @@ repos: - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.12.1 hooks: - id: black - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 \ No newline at end of file diff --git a/Changelog.md b/Changelog.md index edced725..d2ea60ac 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,13 @@ # CHANGELOG All notable changes to this project will be documented here. +## [v2.4.0] +- Fix bug that prevented test results from being returned when a feedback file could not be found (#458) +- Add support for Python 3.11 and 3.12 (#467) +- Track test environment setup status and report errors when running tests if environment setup is in progress or raised an error (#468) +- Update Haskell tester to use [Stack](https://docs.haskellstack.org/en/stable/) to install dependencies (#469) +- Improve default error message when a test group times out (#470) + ## [v2.3.3] - Updated python-ta to 2.6.2 (#454) diff --git a/README.md b/README.md index b20f5380..2486b0a7 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Both the autotester and the API are designed to be run on Ubuntu 20.04 (or suffi 6. [Configure the autotester](#autotester-configuration-options) 7. Optionally install additional python versions. - The `py` (python3) and `pyta` testers can be run using any version of python between versions 3.7 and 3.10. When + The `py` (python3) and `pyta` testers can be run using any version of python between versions 3.7 and 3.12. When these testers are installed the autotester will search the PATH for available python executables. If you want users to be able to run tests with a specific python version, ensure that it is visible in the PATH of both the user running the autotester and all users who run tests. diff --git a/client/.dockerfiles/Dockerfile b/client/.dockerfiles/Dockerfile index e6815ade..ed3a0839 100644 --- a/client/.dockerfiles/Dockerfile +++ b/client/.dockerfiles/Dockerfile @@ -2,11 +2,14 @@ ARG UBUNTU_VERSION FROM ubuntu:$UBUNTU_VERSION -RUN apt-get update -y && apt-get install -y python3 python3-venv +RUN apt-get update -y && \ + apt-get -y install software-properties-common && \ + add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get install -y python3.11 python3.11-venv COPY ./requirements.txt /requirements.txt -RUN python3 -m venv /markus_venv && \ +RUN python3.11 -m venv /markus_venv && \ /markus_venv/bin/pip install wheel && \ /markus_venv/bin/pip install -r /requirements.txt diff --git a/client/autotest_client/__init__.py b/client/autotest_client/__init__.py index 5ac845fe..dc2ea3a8 100644 --- a/client/autotest_client/__init__.py +++ b/client/autotest_client/__init__.py @@ -213,6 +213,17 @@ def update_settings(settings_id, user): @app.route("/settings//test", methods=["PUT"]) @authorize def run_tests(settings_id, user): + test_settings = json.loads(REDIS_CONNECTION.hget("autotest:settings", key=settings_id)) + env_status = test_settings.get("_env_status") + if env_status == "setup": + raise Exception("Setting up test environment. Please try again later.") + elif env_status == "error": + msg = "Settings Error" + settings_error = test_settings.get("_error", "") + if settings_error: + msg += f": {settings_error}" + raise Exception(msg) + test_data = request.json["test_data"] categories = request.json["categories"] high_priority = request.json.get("request_high_priority") @@ -221,7 +232,7 @@ def run_tests(settings_id, user): timeout = 0 - for settings_ in settings(settings_id)["testers"]: + for settings_ in test_settings["testers"]: for data in settings_["test_data"]: timeout += data["timeout"] diff --git a/client/requirements.txt b/client/requirements.txt index 2b044f9c..70902c1a 100644 --- a/client/requirements.txt +++ b/client/requirements.txt @@ -1,5 +1,10 @@ -flask==2.2.2 -python-dotenv==0.21.0 -rq==1.11.1 -redis==4.3.4 -jsonschema==4.16.0 +flask==2.2.5;python_version<"3.8" +flask==2.3.3;python_version>="3.8" +python-dotenv==0.21.1;python_version<"3.8" +python-dotenv==1.0.0;python_version>="3.8" +rq==1.15.1 +redis==5.0.1 +jsonschema==4.17.3;python_version<"3.8" +jsonschema==4.20.0;python_version>="3.8" +Werkzeug==2.2.3;python_version<"3.8" +Werkzeug==2.3.8;python_version>="3.8" diff --git a/docker-compose.yml b/docker-compose.yml index 88b76836..49ffa6b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,5 +77,5 @@ volumes: networks: markus_dev: - external: - name: markus_dev + name: markus_dev + external: true diff --git a/server/.dockerfiles/Dockerfile b/server/.dockerfiles/Dockerfile index 315b3aea..652ca6ac 100644 --- a/server/.dockerfiles/Dockerfile +++ b/server/.dockerfiles/Dockerfile @@ -18,6 +18,10 @@ RUN apt-get update -y && \ python3.9-venv \ python3.10 \ python3.10-venv \ + python3.11 \ + python3.11-venv \ + python3.12 \ + python3.12-venv \ redis-server \ postgresql-client \ libpq-dev \ @@ -34,7 +38,7 @@ RUN useradd -ms /bin/bash $LOGIN_USER && \ COPY . /app -RUN python3.10 -m venv /markus_venv && \ +RUN python3.11 -m venv /markus_venv && \ /markus_venv/bin/pip install wheel && \ /markus_venv/bin/pip install -r /app/requirements.txt && \ find /app/autotest_server/testers -name requirements.system -exec {} \; && \ diff --git a/server/autotest_server/__init__.py b/server/autotest_server/__init__.py index 7bf2a676..c3706fa5 100644 --- a/server/autotest_server/__init__.py +++ b/server/autotest_server/__init__.py @@ -1,3 +1,4 @@ +from __future__ import annotations import os import sys import shutil @@ -134,9 +135,9 @@ def _get_env_vars(test_username: str) -> Dict[str, str]: return env_vars -def _get_feedback(test_data, tests_path, test_id): +def _get_feedback(test_data, tests_path, test_id) -> tuple[dict, str]: feedback_files = test_data.get("feedback_file_names", []) - feedback = [] + feedback, feedback_errors = [], [] for feedback_file in feedback_files: feedback_path = os.path.join(tests_path, feedback_file) if os.path.isfile(feedback_path): @@ -155,8 +156,8 @@ def _get_feedback(test_data, tests_path, test_id): } ) else: - raise Exception(f"Cannot find feedback file at '{feedback_path}'.") - return feedback + feedback_errors.append(feedback_file) + return feedback, feedback_errors def _update_env_vars(base_env: Dict, test_env: Dict) -> Dict: @@ -203,7 +204,8 @@ def _run_test_specs( timeout_expired = None timeout = test_data.get("timeout") try: - env_vars = {**os.environ, **_get_env_vars(test_username), **settings["_env"]} + env = settings.get("_env", {}) + env_vars = {**os.environ, **_get_env_vars(test_username), **env} env_vars = _update_env_vars(env_vars, test_env_vars) proc = subprocess.Popen( args, @@ -215,7 +217,7 @@ def _run_test_specs( stdin=subprocess.PIPE, preexec_fn=set_rlimits_before_test, universal_newlines=True, - env={**os.environ, **env_vars, **settings["_env"]}, + env={**os.environ, **env_vars, **env}, ) try: settings_json = json.dumps({**settings, "test_data": test_data}) @@ -227,13 +229,22 @@ def _run_test_specs( else: _kill_user_processes(test_username) out, err = proc.communicate() + if err == "Killed\n": # Default message from shell + test_group_name = test_data.get("extra_info", {}).get("name", "").strip() + if test_group_name: + err = f"Tests for {test_group_name} did not complete within time limit ({timeout}s)\n" + else: + err = f"Tests did not complete within time limit ({timeout}s)\n" timeout_expired = timeout except Exception as e: err += "\n\n{}".format(e) finally: duration = int(round(time.time() - start, 3) * 1000) extra_info = test_data.get("extra_info", {}) - feedback = _get_feedback(test_data, tests_path, test_id) + feedback, feedback_errors = _get_feedback(test_data, tests_path, test_id) + if feedback_errors: + msg = "Cannot find feedback file(s): " + ", ".join(feedback_errors) + err = err + "\n\n" + msg if err else msg results.append(_create_test_group_result(out, err, duration, extra_info, feedback, timeout_expired)) return results @@ -355,6 +366,11 @@ def ignore_missing_dir_error( def update_test_settings(user, settings_id, test_settings, file_url): + test_settings["_user"] = user + test_settings["_last_access"] = int(time.time()) + test_settings["_env_status"] = "setup" + redis_connection().hset("autotest:settings", key=settings_id, value=json.dumps(test_settings)) + try: settings_dir = os.path.join(TEST_SCRIPT_DIR, str(settings_id)) @@ -387,8 +403,10 @@ def update_test_settings(user, settings_id, test_settings, file_url): test_settings["testers"][i] = tester_settings test_settings["_files"] = files_dir test_settings.pop("_error", None) + test_settings["_env_status"] = "ready" except Exception as e: test_settings["_error"] = str(e) + test_settings["_env_status"] = "error" raise finally: test_settings["_user"] = user diff --git a/server/autotest_server/config.py b/server/autotest_server/config.py index c6893cf8..6a4a6ee6 100644 --- a/server/autotest_server/config.py +++ b/server/autotest_server/config.py @@ -14,7 +14,6 @@ class _Config: - _replacement_pattern: ClassVar[Pattern] = re.compile(r".*?\${(\w+)}.*?") _not_found_key: ClassVar[str] = "" diff --git a/server/autotest_server/testers/haskell/haskell_tester.py b/server/autotest_server/testers/haskell/haskell_tester.py index 6ab9c7d3..572f2841 100644 --- a/server/autotest_server/testers/haskell/haskell_tester.py +++ b/server/autotest_server/testers/haskell/haskell_tester.py @@ -7,6 +7,8 @@ from ..tester import Tester, Test, TestError from ..specs import TestSpecs +STACK_OPTIONS = ["--resolver=lts-14.27", "--system-ghc", "--allow-different-user"] + class HaskellTest(Test): def __init__( @@ -104,11 +106,23 @@ def run_haskell_tests(self) -> Dict[str, List[Dict[str, Union[int, str]]]]: haskell_lib = os.path.join(os.path.dirname(os.path.realpath(__file__)), "lib") for test_file in self.specs["test_data", "script_files"]: with tempfile.NamedTemporaryFile(dir=this_dir) as f: - cmd = ["tasty-discover", ".", "_", f.name] + self._test_run_flags(test_file) + cmd = [ + "stack", + "exec", + *STACK_OPTIONS, + "--", + "tasty-discover", + ".", + "_", + f.name, + *self._test_run_flags(test_file), + ] subprocess.run(cmd, stdout=subprocess.DEVNULL, universal_newlines=True, check=True) with tempfile.NamedTemporaryFile(mode="w+", dir=this_dir) as sf: - cmd = ["runghc", "--", f"-i={haskell_lib}", f.name, f"--stats={sf.name}"] - subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, universal_newlines=True) + cmd = ["stack", "runghc", *STACK_OPTIONS, "--", f"-i={haskell_lib}", f.name, f"--stats={sf.name}"] + subprocess.run( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, universal_newlines=True, check=True + ) results[test_file] = self._parse_test_results(csv.reader(sf)) return results diff --git a/server/autotest_server/testers/haskell/requirements.system b/server/autotest_server/testers/haskell/requirements.system index b809b73a..341699d1 100755 --- a/server/autotest_server/testers/haskell/requirements.system +++ b/server/autotest_server/testers/haskell/requirements.system @@ -1,11 +1,8 @@ #!/usr/bin/env bash -if ! dpkg -l ghc cabal-install &> /dev/null; then +if ! dpkg -l ghc cabal-install haskell-stack &> /dev/null; then apt-get -y update - DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' ghc cabal-install + DEBIAN_FRONTEND=noninteractive apt-get install -y -o 'Dpkg::Options::=--force-confdef' -o 'Dpkg::Options::=--force-confold' ghc cabal-install haskell-stack fi -# TODO: install these without cabal so they can be properly isolated/uninstalled -cabal update -ghc-pkg describe tasty-discover &>/dev/null || cabal install tasty-discover --global -ghc-pkg describe tasty-quickcheck &>/dev/null || cabal install tasty-quickcheck --global +stack update diff --git a/server/autotest_server/testers/haskell/setup.py b/server/autotest_server/testers/haskell/setup.py index 72f1889d..2cbcdb65 100644 --- a/server/autotest_server/testers/haskell/setup.py +++ b/server/autotest_server/testers/haskell/setup.py @@ -3,7 +3,14 @@ import subprocess +HASKELL_TEST_DEPS = ["tasty-discover", "tasty-quickcheck"] + + def create_environment(_settings, _env_dir, default_env_dir): + resolver = "lts-14.27" + cmd = ["stack", "build", "--resolver", resolver, "--system-ghc", *HASKELL_TEST_DEPS] + subprocess.run(cmd, check=True) + return {"PYTHON": os.path.join(default_env_dir, "bin", "python3")} diff --git a/server/autotest_server/testers/java/java_tester.py b/server/autotest_server/testers/java/java_tester.py index 051c2f97..247fd919 100644 --- a/server/autotest_server/testers/java/java_tester.py +++ b/server/autotest_server/testers/java/java_tester.py @@ -30,7 +30,6 @@ def run(self): class JavaTester(Tester): - JUNIT_TESTER_JAR = os.path.join(os.path.dirname(__file__), "lib", "junit-platform-console-standalone.jar") JUNIT_JUPITER_RESULT = "TEST-junit-jupiter.xml" JUNIT_VINTAGE_RESULT = "TEST-junit-vintage.xml" diff --git a/server/autotest_server/testers/jupyter/setup.py b/server/autotest_server/testers/jupyter/setup.py index b95cf1e0..4df657f0 100644 --- a/server/autotest_server/testers/jupyter/setup.py +++ b/server/autotest_server/testers/jupyter/setup.py @@ -18,7 +18,7 @@ def create_environment(settings_, env_dir, _default_env_dir): def settings(): with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: settings_ = json.load(f) - py_versions = [f"3.{x}" for x in range(7, 11) if shutil.which(f"python3.{x}")] + py_versions = [f"3.{x}" for x in range(7, 13) if shutil.which(f"python3.{x}")] python_versions = settings_["properties"]["env_data"]["properties"]["python_version"] python_versions["enum"] = py_versions python_versions["default"] = py_versions[-1] diff --git a/server/autotest_server/testers/py/lib/sql_helper.py b/server/autotest_server/testers/py/lib/sql_helper.py index b664d029..70f69fe9 100644 --- a/server/autotest_server/testers/py/lib/sql_helper.py +++ b/server/autotest_server/testers/py/lib/sql_helper.py @@ -144,7 +144,6 @@ def execute_psql_file( class PSQLTest: - connection: ClassVar[Optional[ConnectionType]] = None SCHEMA_COPY_STR = """ diff --git a/server/autotest_server/testers/py/setup.py b/server/autotest_server/testers/py/setup.py index b95cf1e0..4df657f0 100644 --- a/server/autotest_server/testers/py/setup.py +++ b/server/autotest_server/testers/py/setup.py @@ -18,7 +18,7 @@ def create_environment(settings_, env_dir, _default_env_dir): def settings(): with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: settings_ = json.load(f) - py_versions = [f"3.{x}" for x in range(7, 11) if shutil.which(f"python3.{x}")] + py_versions = [f"3.{x}" for x in range(7, 13) if shutil.which(f"python3.{x}")] python_versions = settings_["properties"]["env_data"]["properties"]["python_version"] python_versions["enum"] = py_versions python_versions["default"] = py_versions[-1] diff --git a/server/autotest_server/testers/pyta/pyta_tester.py b/server/autotest_server/testers/pyta/pyta_tester.py index 584e5a4b..767f9f57 100644 --- a/server/autotest_server/testers/pyta/pyta_tester.py +++ b/server/autotest_server/testers/pyta/pyta_tester.py @@ -14,7 +14,6 @@ class PytaReporter(python_ta.reporters.json_reporter.JSONReporter, python_ta.rep class PytaTest(Test): - ERROR_MSGS = {"reported": "{} error(s)"} def __init__( diff --git a/server/autotest_server/testers/pyta/requirements.txt b/server/autotest_server/testers/pyta/requirements.txt index af37c046..b9e7f41c 100644 --- a/server/autotest_server/testers/pyta/requirements.txt +++ b/server/autotest_server/testers/pyta/requirements.txt @@ -1,3 +1,3 @@ python-ta==1.4.2;python_version<"3.8" -python-ta==2.6.2; python_version>="3.8" +python-ta==2.7.0; python_version>="3.8" isort<5;python_version<"3.8" diff --git a/server/autotest_server/testers/pyta/setup.py b/server/autotest_server/testers/pyta/setup.py index b95cf1e0..4df657f0 100644 --- a/server/autotest_server/testers/pyta/setup.py +++ b/server/autotest_server/testers/pyta/setup.py @@ -18,7 +18,7 @@ def create_environment(settings_, env_dir, _default_env_dir): def settings(): with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "settings_schema.json")) as f: settings_ = json.load(f) - py_versions = [f"3.{x}" for x in range(7, 11) if shutil.which(f"python3.{x}")] + py_versions = [f"3.{x}" for x in range(7, 13) if shutil.which(f"python3.{x}")] python_versions = settings_["properties"]["env_data"]["properties"]["python_version"] python_versions["enum"] = py_versions python_versions["default"] = py_versions[-1] diff --git a/server/autotest_server/testers/racket/racket_tester.py b/server/autotest_server/testers/racket/racket_tester.py index d579baa6..fc844923 100644 --- a/server/autotest_server/testers/racket/racket_tester.py +++ b/server/autotest_server/testers/racket/racket_tester.py @@ -36,7 +36,6 @@ def run(self) -> str: class RacketTester(Tester): - ERROR_MSGS = {"bad_json": "Unable to parse test results: {}"} def __init__(self, specs, test_class: Type[RacketTest] = RacketTest) -> None: diff --git a/server/requirements.txt b/server/requirements.txt index ddb1693a..ae91fdd0 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,8 +1,9 @@ -rq==1.11.1 -click==8.1.3 -redis==4.3.4 -pyyaml==6.0 -jsonschema==4.16.0 -requests==2.28.1 -psycopg2-binary==2.9.5 -supervisor==4.2.4 +rq==1.15.1 +click==8.1.7 +redis==5.0.1 +pyyaml==6.0.1 +jsonschema==4.17.3;python_version<"3.8" +jsonschema==4.20.0;python_version>="3.8" +requests==2.31.0 +psycopg2-binary==2.9.9 +supervisor==4.2.5