feat[ir]: add make_ssa pass to venom pipeline (vyperlang#3825)

this commit updates the venom pipeline to be capable of translating 100%
of the original vyper IR, and successfully passes the entire test suite.

to accomplish this, the translation pass from the old IR to Venom is
simplified, moving several optimization and analysis steps to separate
passes. the most significant of these is the `make_ssa` pass, which
converts any Venom code into SSA form, therefore letting us write
non-SSA code in the translation pass, simplifying the translation.

to support the `make_ssa` pass, this commit also adds a dominator tree
implementation, along with implementations of dominance frontier and
other utility functions. these should also be useful for additional
passes that will be contributed in the future.

to facilitate the development process, this commit also adds two more
output formats: `cfg` and `cfg_runtime`, which provide a graph
representation of the Venom code.

---------

Co-authored-by: Charles Cooper <[email protected]>

.github/workflows/test.yml

@@ -67,8 +67,9 @@ jobs:
       matrix:
         python-version: [["3.11", "311"]]
         opt-mode: ["gas", "none", "codesize"]
-        evm-version: [shanghai]
         debug: [true, false]
+        evm-version: [shanghai]
+        experimental-codegen: [false]
         memorymock: [false]
 
         # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#expanding-or-adding-matrix-configurations
@@ -94,6 +95,14 @@ jobs:
             opt-mode: gas
             evm-version: cancun
 
+          # test experimental pipeline
+          - python-version: ["3.11", "311"]
+            opt-mode: gas
+            debug: false
+            evm-version: shanghai
+            experimental-codegen: true
+          # TODO: test experimental_codegen + -Ocodesize
+
           # run with `--memorymock`, but only need to do it one configuration
           # TODO: consider removing the memorymock tests
           - python-version: ["3.11", "311"]
@@ -108,12 +117,14 @@ jobs:
             opt-mode: gas
             debug: false
             evm-version: shanghai
+
           - python-version: ["3.12", "312"]
             opt-mode: gas
             debug: false
             evm-version: shanghai
 
-    name: py${{ matrix.python-version[1] }}-opt-${{ matrix.opt-mode }}${{ matrix.debug && '-debug' || '' }}${{ matrix.memorymock && '-memorymock' || '' }}-${{ matrix.evm-version }}
+
+    name: py${{ matrix.python-version[1] }}-opt-${{ matrix.opt-mode }}${{ matrix.debug && '-debug' || '' }}${{ matrix.memorymock && '-memorymock' || '' }}${{ matrix.experimental-codegen && '-experimental' || '' }}-${{ matrix.evm-version }}
 
     steps:
     - uses: actions/checkout@v4
@@ -141,6 +152,7 @@ jobs:
           --evm-version ${{ matrix.evm-version }} \
           ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }} \
           ${{ matrix.memorymock && '--memorymock' || '' }} \
+          ${{ matrix.experimental-codegen && '--experimental-codegen' || '' }} \
           --cov-branch \
           --cov-report xml:coverage.xml \
           --cov=vyper \
@@ -193,8 +205,7 @@ jobs:
     # NOTE: if the tests get poorly distributed, run this and commit the resulting `.test_durations` file to the `vyper-test-durations` repo.
     # `pytest -m "fuzzing" --store-durations -r aR tests/`
     - name: Fetch test-durations
-      run: |
-        curl --location "https://raw.githubusercontent.com/vyperlang/vyper-test-durations/master/test_durations" -o .test_durations
+      run: curl --location "https://raw.githubusercontent.com/vyperlang/vyper-test-durations/master/test_durations" -o .test_durations
 
     - name: Run tests
       run: |

setup.cfg

@@ -31,3 +31,4 @@ testpaths = tests
 xfail_strict = true
 markers =
 	fuzzing: Run Hypothesis fuzz test suite (deselect with '-m "not fuzzing"')
+	venom_xfail: mark a test case as a regression (expected to fail) under the venom pipeline

tests/conftest.py

@@ -59,6 +59,7 @@ def pytest_addoption(parser):
         help="change optimization mode",
     )
     parser.addoption("--enable-compiler-debug-mode", action="store_true")
+    parser.addoption("--experimental-codegen", action="store_true")
 
     parser.addoption(
         "--evm-version",
@@ -73,6 +74,8 @@ def output_formats():
     output_formats = compiler.OUTPUT_FORMATS.copy()
     del output_formats["bb"]
     del output_formats["bb_runtime"]
+    del output_formats["cfg"]
+    del output_formats["cfg_runtime"]
     return output_formats
 
 
@@ -89,6 +92,36 @@ def debug(pytestconfig):
     _set_debug_mode(debug)
 
 
+@pytest.fixture(scope="session")
+def experimental_codegen(pytestconfig):
+    ret = pytestconfig.getoption("experimental_codegen")
+    assert isinstance(ret, bool)
+    return ret
+
+
+@pytest.fixture(autouse=True)
+def check_venom_xfail(request, experimental_codegen):
+    if not experimental_codegen:
+        return
+
+    marker = request.node.get_closest_marker("venom_xfail")
+    if marker is None:
+        return
+
+    # https://github.com/okken/pytest-runtime-xfail?tab=readme-ov-file#alternatives
+    request.node.add_marker(pytest.mark.xfail(strict=True, **marker.kwargs))
+
+
+@pytest.fixture
+def venom_xfail(request, experimental_codegen):
+    def _xfail(*args, **kwargs):
+        if not experimental_codegen:
+            return
+        request.node.add_marker(pytest.mark.xfail(*args, strict=True, **kwargs))
+
+    return _xfail
+
+
 @pytest.fixture(scope="session", autouse=True)
 def evm_version(pytestconfig):
     # note: we configure the evm version that we emit code for,
@@ -108,6 +141,7 @@ def chdir_tmp_path(tmp_path):
         yield
 
 
+# CMC 2024-03-01 this doesn't need to be a fixture
 @pytest.fixture
 def keccak():
     return Web3.keccak
@@ -321,6 +355,7 @@ def _get_contract(
     w3,
     source_code,
     optimize,
+    experimental_codegen,
     output_formats,
     *args,
     override_opt_level=None,
@@ -329,6 +364,7 @@ def _get_contract(
 ):
     settings = Settings()
     settings.optimize = override_opt_level or optimize
+    settings.experimental_codegen = experimental_codegen
     out = compiler.compile_code(
         source_code,
         # test that all output formats can get generated
@@ -352,17 +388,21 @@ def _get_contract(
 
 
 @pytest.fixture(scope="module")
-def get_contract(w3, optimize, output_formats):
+def get_contract(w3, optimize, experimental_codegen, output_formats):
     def fn(source_code, *args, **kwargs):
-        return _get_contract(w3, source_code, optimize, output_formats, *args, **kwargs)
+        return _get_contract(
+            w3, source_code, optimize, experimental_codegen, output_formats, *args, **kwargs
+        )
 
     return fn
 
 
 @pytest.fixture
-def get_contract_with_gas_estimation(tester, w3, optimize, output_formats):
+def get_contract_with_gas_estimation(tester, w3, optimize, experimental_codegen, output_formats):
     def get_contract_with_gas_estimation(source_code, *args, **kwargs):
-        contract = _get_contract(w3, source_code, optimize, output_formats, *args, **kwargs)
+        contract = _get_contract(
+            w3, source_code, optimize, experimental_codegen, output_formats, *args, **kwargs
+        )
         for abi_ in contract._classic_contract.functions.abi:
             if abi_["type"] == "function":
                 set_decorator_to_contract_function(w3, tester, contract, source_code, abi_["name"])
@@ -372,15 +412,19 @@ def get_contract_with_gas_estimation(source_code, *args, **kwargs):
 
 
 @pytest.fixture
-def get_contract_with_gas_estimation_for_constants(w3, optimize, output_formats):
+def get_contract_with_gas_estimation_for_constants(
+    w3, optimize, experimental_codegen, output_formats
+):
     def get_contract_with_gas_estimation_for_constants(source_code, *args, **kwargs):
-        return _get_contract(w3, source_code, optimize, output_formats, *args, **kwargs)
+        return _get_contract(
+            w3, source_code, optimize, experimental_codegen, output_formats, *args, **kwargs
+        )
 
     return get_contract_with_gas_estimation_for_constants
 
 
 @pytest.fixture(scope="module")
-def get_contract_module(optimize, output_formats):
+def get_contract_module(optimize, experimental_codegen, output_formats):
     """
     This fixture is used for Hypothesis tests to ensure that
     the same contract is called over multiple runs of the test.
@@ -393,16 +437,25 @@ def get_contract_module(optimize, output_formats):
     w3.eth.set_gas_price_strategy(zero_gas_price_strategy)
 
     def get_contract_module(source_code, *args, **kwargs):
-        return _get_contract(w3, source_code, optimize, output_formats, *args, **kwargs)
+        return _get_contract(
+            w3, source_code, optimize, experimental_codegen, output_formats, *args, **kwargs
+        )
 
     return get_contract_module
 
 
 def _deploy_blueprint_for(
-    w3, source_code, optimize, output_formats, initcode_prefix=ERC5202_PREFIX, **kwargs
+    w3,
+    source_code,
+    optimize,
+    experimental_codegen,
+    output_formats,
+    initcode_prefix=ERC5202_PREFIX,
+    **kwargs,
 ):
     settings = Settings()
     settings.optimize = optimize
+    settings.experimental_codegen = experimental_codegen
     out = compiler.compile_code(
         source_code,
         output_formats=output_formats,
@@ -438,9 +491,11 @@ def factory(address):
 
 
 @pytest.fixture(scope="module")
-def deploy_blueprint_for(w3, optimize, output_formats):
+def deploy_blueprint_for(w3, optimize, experimental_codegen, output_formats):
     def deploy_blueprint_for(source_code, *args, **kwargs):
-        return _deploy_blueprint_for(w3, source_code, optimize, output_formats, *args, **kwargs)
+        return _deploy_blueprint_for(
+            w3, source_code, optimize, experimental_codegen, output_formats, *args, **kwargs
+        )
 
     return deploy_blueprint_for
 

tests/functional/builtins/codegen/test_abi_decode.py

@@ -3,7 +3,7 @@
 import pytest
 from eth.codecs import abi
 
-from vyper.exceptions import ArgumentException, StructureException
+from vyper.exceptions import ArgumentException, StackTooDeep, StructureException
 
 TEST_ADDR = "0x" + b"".join(chr(i).encode("utf-8") for i in range(20)).hex()
 
@@ -196,6 +196,7 @@ def abi_decode(x: Bytes[{len}]) -> DynArray[DynArray[uint256, 3], 3]:
 
 @pytest.mark.parametrize("args", nested_3d_array_args)
 @pytest.mark.parametrize("unwrap_tuple", (True, False))
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_abi_decode_nested_dynarray2(get_contract, args, unwrap_tuple):
     if unwrap_tuple is True:
         encoded = abi.encode("(uint256[][][])", (args,))
@@ -273,6 +274,7 @@ def foo(bs: Bytes[160]) -> (uint256, DynArray[uint256, 3]):
     assert c.foo(encoded) == [2**256 - 1, bs]
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_abi_decode_private_nested_dynarray(get_contract):
     code = """
 bytez: DynArray[DynArray[DynArray[uint256, 3], 3], 3]

tests/functional/builtins/codegen/test_abi_encode.py

@@ -3,6 +3,8 @@
 import pytest
 from eth.codecs import abi
 
+from vyper.exceptions import StackTooDeep
+
 
 # @pytest.mark.parametrize("string", ["a", "abc", "abcde", "potato"])
 def test_abi_encode(get_contract):
@@ -226,6 +228,7 @@ def abi_encode(
 
 
 @pytest.mark.parametrize("args", nested_3d_array_args)
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_abi_encode_nested_dynarray_2(get_contract, args):
     code = """
 @external
@@ -330,6 +333,7 @@ def foo(bs: DynArray[uint256, 3]) -> (uint256, Bytes[160]):
     assert c.foo(bs) == [2**256 - 1, abi.encode("(uint256[])", (bs,))]
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_abi_encode_private_nested_dynarray(get_contract):
     code = """
 bytez: Bytes[1696]

tests/functional/codegen/features/test_clampers.py

@@ -4,6 +4,7 @@
 from eth.codecs import abi
 from eth_utils import keccak
 
+from vyper.exceptions import StackTooDeep
 from vyper.utils import int_bounds
 
 
@@ -506,6 +507,7 @@ def foo(b: DynArray[int128, 10]) -> DynArray[int128, 10]:
 
 
 @pytest.mark.parametrize("value", [0, 1, -1, 2**127 - 1, -(2**127)])
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_multidimension_dynarray_clamper_passing(w3, get_contract, value):
     code = """
 @external

tests/functional/codegen/features/test_constructor.py

@@ -1,6 +1,8 @@
 import pytest
 from web3.exceptions import ValidationError
 
+from vyper.exceptions import StackTooDeep
+
 
 def test_init_argument_test(get_contract_with_gas_estimation):
     init_argument_test = """
@@ -163,6 +165,7 @@ def get_foo() -> uint256:
     assert c.get_foo() == 39
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_nested_dynamic_array_constructor_arg_2(w3, get_contract_with_gas_estimation):
     code = """
 foo: int128
@@ -208,6 +211,7 @@ def get_foo() -> DynArray[DynArray[uint256, 3], 3]:
     assert c.get_foo() == [[37, 41, 73], [37041, 41073, 73037], [146, 123, 148]]
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_initialise_nested_dynamic_array_2(w3, get_contract_with_gas_estimation):
     code = """
 foo: DynArray[DynArray[DynArray[int128, 3], 3], 3]

tests/functional/codegen/features/test_immutable.py

@@ -1,6 +1,7 @@
 import pytest
 
 from vyper.compiler.settings import OptimizationLevel
+from vyper.exceptions import StackTooDeep
 
 
 @pytest.mark.parametrize(
@@ -198,6 +199,7 @@ def get_idx_two() -> uint256:
     assert c.get_idx_two() == expected_values[2][2]
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_nested_dynarray_immutable(get_contract):
     code = """
 my_list: immutable(DynArray[DynArray[DynArray[int128, 3], 3], 3])

tests/functional/codegen/types/test_dynamic_array.py

@@ -8,6 +8,7 @@
     ArrayIndexException,
     ImmutableViolation,
     OverflowException,
+    StackTooDeep,
     StateAccessViolation,
     TypeMismatch,
 )
@@ -60,6 +61,7 @@ def loo(x: DynArray[DynArray[int128, 2], 2]) -> int128:
     print("Passed list tests")
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_string_list(get_contract):
     code = """
 @external
@@ -732,6 +734,7 @@ def test_array_decimal_return3() -> DynArray[DynArray[decimal, 2], 2]:
     assert c.test_array_decimal_return3() == [[1.0, 2.0], [3.0]]
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_mult_list(get_contract_with_gas_estimation):
     code = """
 nest3: DynArray[DynArray[DynArray[uint256, 2], 2], 2]
@@ -1478,6 +1481,7 @@ def foo(x: int128) -> int128:
     assert c.foo(7) == 392
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_struct_of_lists(get_contract):
     code = """
 struct Foo:
@@ -1566,6 +1570,7 @@ def bar(x: int128) -> DynArray[int128, 3]:
     assert c.bar(7) == [7, 14]
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_nested_struct_of_lists(get_contract, assert_compile_failed, optimize):
     code = """
 struct nestedFoo:
@@ -1695,7 +1700,9 @@ def __init__():
         ("DynArray[DynArray[DynArray[uint256, 5], 5], 5]", [[[], []], []]),
     ],
 )
-def test_empty_nested_dynarray(get_contract, typ, val):
+def test_empty_nested_dynarray(get_contract, typ, val, venom_xfail):
+    if val == [[[], []], []]:
+        venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
     code = f"""
 @external
 def foo() -> {typ}: