chore: remove deadlines and reruns

for historical reasons, pytest ran with `--allow-reruns` when tests failed because tests would usually fail due to deadline errors. deadline errors have never indicated any meaningful issue with the compiler, they are just a somewhat unavoidable byproduct of the fact that we are running in a CI environment which has a lot of jitter. this commit changes the hypothesis deadline to `None` for the whole test suite, and removes the `--allow-reruns` parameter in the CI, which should make the test suite much more efficient when there are failures.
vyperlang · Sep 28, 2023 · 83ab9ef · 83ab9ef
1 parent 4281780
commit 83ab9ef
Show file tree

Hide file tree

Showing 24 changed files with 48 additions and 48 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -104,7 +104,7 @@ jobs:
       run: pip install tox
 
     - name: Run Tox
-      run: TOXENV=py${{ matrix.python-version[1] }} tox -r -- --optimize ${{ matrix.opt-mode }} ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }} --reruns 10 --reruns-delay 1 -r aR tests/
+      run: TOXENV=py${{ matrix.python-version[1] }} tox -r -- --optimize ${{ matrix.opt-mode }} ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }} -r aR tests/
 
     - name: Upload Coverage
       uses: codecov/codecov-action@v1
@@ -148,12 +148,12 @@ jobs:
 
     # fetch test durations
     # NOTE: if the tests get poorly distributed, run this and commit the resulting `.test_durations` file to the `vyper-test-durations` repo.
-    # `TOXENV=fuzzing tox -r -- --store-durations --reruns 10 --reruns-delay 1 -r aR tests/`
+    # `TOXENV=fuzzing tox -r -- --store-durations -r aR tests/`
     - name: Fetch test-durations
       run: curl --location "https://raw.githubusercontent.com/vyperlang/vyper-test-durations/5982755ee8459f771f2e8622427c36494646e1dd/test_durations" -o .test_durations
 
     - name: Run Tox
-      run: TOXENV=fuzzing tox -r -- --splits 60 --group ${{ matrix.group }} --splitting-algorithm least_duration --reruns 10 --reruns-delay 1 -r aR tests/
+      run: TOXENV=fuzzing tox -r -- --splits 60 --group ${{ matrix.group }} --splitting-algorithm least_duration -r aR tests/
 
     - name: Upload Coverage
       uses: codecov/codecov-action@v1

diff --git a/tests/ast/nodes/test_evaluate_binop_decimal.py b/tests/ast/nodes/test_evaluate_binop_decimal.py
@@ -13,7 +13,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=None)
+@settings(max_examples=50)
 @given(left=st_decimals, right=st_decimals)
 @example(left=Decimal("0.9999999999"), right=Decimal("0.0000000001"))
 @example(left=Decimal("0.0000000001"), right=Decimal("0.9999999999"))
@@ -52,7 +52,7 @@ def test_binop_pow():
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=None)
+@settings(max_examples=50)
 @given(
     values=st.lists(st_decimals, min_size=2, max_size=10),
     ops=st.lists(st.sampled_from("+-*/%"), min_size=11, max_size=11),

diff --git a/tests/ast/nodes/test_evaluate_binop_int.py b/tests/ast/nodes/test_evaluate_binop_int.py
@@ -9,7 +9,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(left=st_int32, right=st_int32)
 @example(left=1, right=1)
 @example(left=1, right=-1)
@@ -42,7 +42,7 @@ def foo(a: int128, b: int128) -> int128:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(left=st_uint64, right=st_uint64)
 @pytest.mark.parametrize("op", "+-*/%")
 def test_binop_uint256(get_contract, assert_tx_failed, op, left, right):
@@ -69,7 +69,7 @@ def foo(a: uint256, b: uint256) -> uint256:
 
 @pytest.mark.xfail(reason="need to implement safe exponentiation logic")
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(left=st.integers(min_value=2, max_value=245), right=st.integers(min_value=0, max_value=16))
 @example(left=0, right=0)
 @example(left=0, right=1)
@@ -89,7 +89,7 @@ def foo(a: uint256, b: uint256) -> uint256:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(
     values=st.lists(st.integers(min_value=-256, max_value=256), min_size=2, max_size=10),
     ops=st.lists(st.sampled_from("+-*/%"), min_size=11, max_size=11),

diff --git a/tests/ast/nodes/test_evaluate_boolop.py b/tests/ast/nodes/test_evaluate_boolop.py
@@ -8,7 +8,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(values=st.lists(st.booleans(), min_size=2, max_size=10))
 @pytest.mark.parametrize("comparator", ["and", "or"])
 def test_boolop_simple(get_contract, values, comparator):
@@ -32,7 +32,7 @@ def foo({input_value}) -> bool:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(
     values=st.lists(st.booleans(), min_size=2, max_size=10),
     comparators=st.lists(st.sampled_from(["and", "or"]), min_size=11, max_size=11),

diff --git a/tests/ast/nodes/test_evaluate_compare.py b/tests/ast/nodes/test_evaluate_compare.py
@@ -8,7 +8,7 @@
 
 # TODO expand to all signed types
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(left=st.integers(), right=st.integers())
 @pytest.mark.parametrize("op", ["==", "!=", "<", "<=", ">=", ">"])
 def test_compare_eq_signed(get_contract, op, left, right):
@@ -28,7 +28,7 @@ def foo(a: int128, b: int128) -> bool:
 
 # TODO expand to all unsigned types
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(left=st.integers(min_value=0), right=st.integers(min_value=0))
 @pytest.mark.parametrize("op", ["==", "!=", "<", "<=", ">=", ">"])
 def test_compare_eq_unsigned(get_contract, op, left, right):
@@ -47,7 +47,7 @@ def foo(a: uint128, b: uint128) -> bool:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=20, deadline=1000)
+@settings(max_examples=20)
 @given(left=st.integers(), right=st.lists(st.integers(), min_size=1, max_size=16))
 def test_compare_in(left, right, get_contract):
     source = f"""
@@ -76,7 +76,7 @@ def bar(a: int128) -> bool:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=20, deadline=1000)
+@settings(max_examples=20)
 @given(left=st.integers(), right=st.lists(st.integers(), min_size=1, max_size=16))
 def test_compare_not_in(left, right, get_contract):
     source = f"""

diff --git a/tests/ast/nodes/test_evaluate_subscript.py b/tests/ast/nodes/test_evaluate_subscript.py
@@ -6,7 +6,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(
     idx=st.integers(min_value=0, max_value=9),
     array=st.lists(st.integers(), min_size=10, max_size=10),

diff --git a/tests/builtins/folding/test_abs.py b/tests/builtins/folding/test_abs.py
@@ -8,7 +8,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(a=st.integers(min_value=-(2**255) + 1, max_value=2**255 - 1))
 @example(a=0)
 def test_abs(get_contract, a):
@@ -27,7 +27,7 @@ def foo(a: int256) -> int256:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(a=st.integers(min_value=2**255, max_value=2**256 - 1))
 def test_abs_upper_bound_folding(get_contract, a):
     source = f"""

diff --git a/tests/builtins/folding/test_addmod_mulmod.py b/tests/builtins/folding/test_addmod_mulmod.py
@@ -9,7 +9,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(a=st_uint256, b=st_uint256, c=st_uint256)
 @pytest.mark.parametrize("fn_name", ["uint256_addmod", "uint256_mulmod"])
 def test_modmath(get_contract, a, b, c, fn_name):

diff --git a/tests/builtins/folding/test_bitwise.py b/tests/builtins/folding/test_bitwise.py
@@ -14,7 +14,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @pytest.mark.parametrize("op", ["&", "|", "^"])
 @given(a=st_uint256, b=st_uint256)
 def test_bitwise_ops(get_contract, a, b, op):
@@ -34,7 +34,7 @@ def foo(a: uint256, b: uint256) -> uint256:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @pytest.mark.parametrize("op", ["<<", ">>"])
 @given(a=st_uint256, b=st.integers(min_value=0, max_value=256))
 def test_bitwise_shift_unsigned(get_contract, a, b, op):
@@ -64,7 +64,7 @@ def foo(a: uint256, b: uint256) -> uint256:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @pytest.mark.parametrize("op", ["<<", ">>"])
 @given(a=st_sint256, b=st.integers(min_value=0, max_value=256))
 def test_bitwise_shift_signed(get_contract, a, b, op):
@@ -92,7 +92,7 @@ def foo(a: int256, b: uint256) -> int256:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(value=st_uint256)
 def test_bitwise_not(get_contract, value):
     source = """

diff --git a/tests/builtins/folding/test_floor_ceil.py b/tests/builtins/folding/test_floor_ceil.py
@@ -13,7 +13,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(value=st_decimals)
 @example(value=Decimal("0.9999999999"))
 @example(value=Decimal("0.0000000001"))

diff --git a/tests/builtins/folding/test_fold_as_wei_value.py b/tests/builtins/folding/test_fold_as_wei_value.py
@@ -19,7 +19,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=10, deadline=1000)
+@settings(max_examples=10)
 @given(value=st_decimals)
 @pytest.mark.parametrize("denom", denoms)
 def test_decimal(get_contract, value, denom):
@@ -38,7 +38,7 @@ def foo(a: decimal) -> uint256:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=10, deadline=1000)
+@settings(max_examples=10)
 @given(value=st.integers(min_value=0, max_value=2**128))
 @pytest.mark.parametrize("denom", denoms)
 def test_integer(get_contract, value, denom):

diff --git a/tests/builtins/folding/test_keccak_sha.py b/tests/builtins/folding/test_keccak_sha.py
@@ -10,7 +10,7 @@
 
 @pytest.mark.fuzzing
 @given(value=st.text(alphabet=alphabet, min_size=0, max_size=100))
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @pytest.mark.parametrize("fn_name", ["keccak256", "sha256"])
 def test_string(get_contract, value, fn_name):
     source = f"""
@@ -29,7 +29,7 @@ def foo(a: String[100]) -> bytes32:
 
 @pytest.mark.fuzzing
 @given(value=st.binary(min_size=0, max_size=100))
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @pytest.mark.parametrize("fn_name", ["keccak256", "sha256"])
 def test_bytes(get_contract, value, fn_name):
     source = f"""
@@ -48,7 +48,7 @@ def foo(a: Bytes[100]) -> bytes32:
 
 @pytest.mark.fuzzing
 @given(value=st.binary(min_size=1, max_size=100))
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @pytest.mark.parametrize("fn_name", ["keccak256", "sha256"])
 def test_hex(get_contract, value, fn_name):
     source = f"""

diff --git a/tests/builtins/folding/test_min_max.py b/tests/builtins/folding/test_min_max.py
@@ -18,7 +18,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(left=st_decimals, right=st_decimals)
 @pytest.mark.parametrize("fn_name", ["min", "max"])
 def test_decimal(get_contract, left, right, fn_name):
@@ -37,7 +37,7 @@ def foo(a: decimal, b: decimal) -> decimal:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(left=st_int128, right=st_int128)
 @pytest.mark.parametrize("fn_name", ["min", "max"])
 def test_int128(get_contract, left, right, fn_name):
@@ -56,7 +56,7 @@ def foo(a: int128, b: int128) -> int128:
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=50, deadline=1000)
+@settings(max_examples=50)
 @given(left=st_uint256, right=st_uint256)
 @pytest.mark.parametrize("fn_name", ["min", "max"])
 def test_min_uint256(get_contract, left, right, fn_name):

diff --git a/tests/builtins/folding/test_powmod.py b/tests/builtins/folding/test_powmod.py
@@ -9,7 +9,7 @@
 
 
 @pytest.mark.fuzzing
-@settings(max_examples=100, deadline=1000)
+@settings(max_examples=100)
 @given(a=st_uint256, b=st_uint256)
 def test_powmod_uint256(get_contract, a, b):
     source = """

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,6 +1,7 @@
 import logging
 from functools import wraps
 
+import hypothesis
 import pytest
 from eth_tester import EthereumTester, PyEVMBackend
 from eth_utils import setup_DEBUG2_logging
@@ -46,6 +47,9 @@ def pytest_addoption(parser):
     parser.addoption("--enable-compiler-debug-mode", action="store_true")
 
 
+hypothesis.settings.default.deadline = None
+
+
 @pytest.fixture(scope="module")
 def optimize(pytestconfig):
     flag = pytestconfig.getoption("optimize")

diff --git a/tests/fuzzing/test_exponents.py b/tests/fuzzing/test_exponents.py
@@ -92,7 +92,7 @@ def foo(a: int16) -> int16:
 @example(a=2**127 - 1)
 # 256 bits
 @example(a=2**256 - 1)
-@settings(max_examples=200, deadline=1000)
+@settings(max_examples=200)
 def test_max_exp(get_contract, assert_tx_failed, a):
     code = f"""
 @external
@@ -127,7 +127,7 @@ def foo(b: uint256) -> uint256:
 @example(a=2**63 - 1)
 # 128 bits
 @example(a=2**127 - 1)
-@settings(max_examples=200, deadline=1000)
+@settings(max_examples=200)
 def test_max_exp_int128(get_contract, assert_tx_failed, a):
     code = f"""
 @external

diff --git a/tests/grammar/test_grammar.py b/tests/grammar/test_grammar.py
@@ -4,7 +4,7 @@
 import hypothesis
 import hypothesis.strategies as st
 import pytest
-from hypothesis import HealthCheck, assume, given
+from hypothesis import assume, given
 from hypothesis.extra.lark import LarkStrategy
 
 from vyper.ast import Module, parse_to_ast
@@ -103,7 +103,7 @@ def has_no_docstrings(c):
 
 @pytest.mark.fuzzing
 @given(code=from_grammar().filter(lambda c: utf8_encodable(c)))
-@hypothesis.settings(deadline=400, max_examples=500, suppress_health_check=(HealthCheck.too_slow,))
+@hypothesis.settings(max_examples=500)
 def test_grammar_bruteforce(code):
     if utf8_encodable(code):
         _, _, reformatted_code = pre_parse(code + "\n")

diff --git a/tests/parser/features/test_internal_call.py b/tests/parser/features/test_internal_call.py
@@ -669,7 +669,7 @@ def test_internal_call_kwargs(get_contract, typ1, strategy1, typ2, strategy2):
     # GHSA-ph9x-4vc9-m39g
 
     @given(kwarg1=strategy1, default1=strategy1, kwarg2=strategy2, default2=strategy2)
-    @settings(deadline=None, max_examples=5)  # len(cases) * len(cases) * 5 * 5
+    @settings(max_examples=5)  # len(cases) * len(cases) * 5 * 5
     def fuzz(kwarg1, kwarg2, default1, default2):
         code = f"""
 @internal

diff --git a/tests/parser/functions/test_slice.py b/tests/parser/functions/test_slice.py
@@ -36,7 +36,7 @@ def slice_tower_test(inp1: Bytes[50]) -> Bytes[50]:
 @pytest.mark.parametrize("literal_length", (True, False))
 @pytest.mark.parametrize("opt_level", list(OptimizationLevel))
 @given(start=_draw_1024, length=_draw_1024, length_bound=_draw_1024_1, bytesdata=_bytes_1024)
-@settings(max_examples=100, deadline=None)
+@settings(max_examples=100)
 @pytest.mark.fuzzing
 def test_slice_immutable(
     get_contract,
@@ -90,7 +90,7 @@ def _get_contract():
 @pytest.mark.parametrize("literal_length", (True, False))
 @pytest.mark.parametrize("opt_level", list(OptimizationLevel))
 @given(start=_draw_1024, length=_draw_1024, length_bound=_draw_1024_1, bytesdata=_bytes_1024)
-@settings(max_examples=100, deadline=None)
+@settings(max_examples=100)
 @pytest.mark.fuzzing
 def test_slice_bytes(
     get_contract,

diff --git a/tests/parser/test_call_graph_stability.py b/tests/parser/test_call_graph_stability.py
@@ -15,7 +15,7 @@ def _valid_identifier(attr):
 
 
 # random names for functions
-@settings(max_examples=20, deadline=None)
+@settings(max_examples=20)
 @given(
     st.lists(
         st.tuples(