From 919346bd36683a2aca4a8db4b7c6a82631e77336 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 11 Sep 2024 15:27:45 +0300 Subject: [PATCH 1/8] =?UTF-8?q?=CF=84=CE=B5=CE=BC=CF=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit temp more stack2mem exception and usage mem allocator tests and fixes to allocator fixes lint allocator more stack top method allocator fixes lint fixes and update tests plug memory allocator add types cleanup and lint exceptions handling --- .../codegen/features/test_clampers.py | 2 - .../codegen/types/test_dynamic_array.py | 2 - .../unit/compiler/venom/test_mem_allocator.py | 116 ++++++++++++++++++ vyper/exceptions.py | 13 +- vyper/semantics/analysis/module.py | 2 +- vyper/venom/__init__.py | 3 + vyper/venom/context.py | 5 + vyper/venom/mem_allocator.py | 58 +++++++++ vyper/venom/passes/simplify_cfg.py | 4 +- vyper/venom/passes/stack2mem.py | 70 +++++++++++ vyper/venom/stack_model.py | 12 +- vyper/venom/venom_to_assembly.py | 33 ++--- 12 files changed, 291 insertions(+), 29 deletions(-) create mode 100644 tests/unit/compiler/venom/test_mem_allocator.py create mode 100644 vyper/venom/mem_allocator.py create mode 100644 vyper/venom/passes/stack2mem.py diff --git a/tests/functional/codegen/features/test_clampers.py b/tests/functional/codegen/features/test_clampers.py index 1adffcf29a..d62b697fd5 100644 --- a/tests/functional/codegen/features/test_clampers.py +++ b/tests/functional/codegen/features/test_clampers.py @@ -5,7 +5,6 @@ from eth_utils import keccak from tests.utils import ZERO_ADDRESS, decimal_to_int -from vyper.exceptions import StackTooDeep from vyper.utils import int_bounds @@ -502,7 +501,6 @@ def foo(b: DynArray[int128, 10]) -> DynArray[int128, 10]: @pytest.mark.parametrize("value", [0, 1, -1, 2**127 - 1, -(2**127)]) -@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression") def test_multidimension_dynarray_clamper_passing(get_contract, value): code = """ @external diff --git a/tests/functional/codegen/types/test_dynamic_array.py b/tests/functional/codegen/types/test_dynamic_array.py index 2a0f4e77e5..2f647ac38c 100644 --- a/tests/functional/codegen/types/test_dynamic_array.py +++ b/tests/functional/codegen/types/test_dynamic_array.py @@ -11,7 +11,6 @@ CompilerPanic, ImmutableViolation, OverflowException, - StackTooDeep, StateAccessViolation, TypeMismatch, ) @@ -737,7 +736,6 @@ def test_array_decimal_return3() -> DynArray[DynArray[decimal, 2], 2]: ] -@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression") def test_mult_list(get_contract): code = """ nest3: DynArray[DynArray[DynArray[uint256, 2], 2], 2] diff --git a/tests/unit/compiler/venom/test_mem_allocator.py b/tests/unit/compiler/venom/test_mem_allocator.py new file mode 100644 index 0000000000..a5d88d3cd6 --- /dev/null +++ b/tests/unit/compiler/venom/test_mem_allocator.py @@ -0,0 +1,116 @@ +import pytest + +from vyper.venom.mem_allocator import MemoryAllocator + +MEM_BLOCK_ADDRESS = 0x1000 + + +@pytest.fixture +def allocator(): + return MemoryAllocator(1024, MEM_BLOCK_ADDRESS) + + +def test_initial_state(allocator): + assert allocator.get_free_memory() == 1024 + assert allocator.get_allocated_memory() == 0 + + +def test_single_allocation(allocator): + addr = allocator.allocate(256) + assert addr == MEM_BLOCK_ADDRESS + assert allocator.get_free_memory() == 768 + assert allocator.get_allocated_memory() == 256 + + +def test_multiple_allocations(allocator): + addr1 = allocator.allocate(256) + addr2 = allocator.allocate(128) + addr3 = allocator.allocate(64) + + assert addr1 == MEM_BLOCK_ADDRESS + assert addr2 == MEM_BLOCK_ADDRESS + 256 + assert addr3 == MEM_BLOCK_ADDRESS + 384 + assert allocator.get_free_memory() == 576 + assert allocator.get_allocated_memory() == 448 + + +def test_deallocation(allocator): + addr1 = allocator.allocate(256) + addr2 = allocator.allocate(128) + + assert allocator.deallocate(addr1) is True + assert allocator.get_free_memory() == 896 + assert allocator.get_allocated_memory() == 128 + + assert allocator.deallocate(addr2) is True + assert allocator.get_free_memory() == 1024 + assert allocator.get_allocated_memory() == 0 + + +def test_allocation_after_deallocation(allocator): + addr1 = allocator.allocate(256) + allocator.deallocate(addr1) + addr2 = allocator.allocate(128) + + assert addr2 == MEM_BLOCK_ADDRESS + assert allocator.get_free_memory() == 896 + assert allocator.get_allocated_memory() == 128 + + +def test_out_of_memory(allocator): + allocator.allocate(1000) + with pytest.raises(MemoryError): + allocator.allocate(100) + + +def test_invalid_deallocation(allocator): + assert allocator.deallocate(0x2000) is False + + +def test_fragmentation_and_merging(allocator): + addr1 = allocator.allocate(256) + addr2 = allocator.allocate(256) + addr3 = allocator.allocate(256) + + assert allocator.get_free_memory() == 256 + assert allocator.get_allocated_memory() == 768 + + allocator.deallocate(addr1) + assert allocator.get_free_memory() == 512 + assert allocator.get_allocated_memory() == 512 + + allocator.deallocate(addr3) + assert allocator.get_free_memory() == 768 + assert allocator.get_allocated_memory() == 256 + + addr4 = allocator.allocate(512) + assert addr4 == MEM_BLOCK_ADDRESS + 512 + assert allocator.get_free_memory() == 256 + assert allocator.get_allocated_memory() == 768 + + allocator.deallocate(addr2) + assert allocator.get_free_memory() == 512 + assert allocator.get_allocated_memory() == 512 + + allocator.deallocate(addr4) + assert allocator.get_free_memory() == 1024 # All blocks merged + assert allocator.get_allocated_memory() == 0 + + # Test if we can now allocate the entire memory + addr5 = allocator.allocate(1024) + assert addr5 == MEM_BLOCK_ADDRESS + assert allocator.get_free_memory() == 0 + assert allocator.get_allocated_memory() == 1024 + + +def test_exact_fit_allocation(allocator): + addr1 = allocator.allocate(1024) + assert addr1 == MEM_BLOCK_ADDRESS + assert allocator.get_free_memory() == 0 + assert allocator.get_allocated_memory() == 1024 + + allocator.deallocate(addr1) + addr2 = allocator.allocate(1024) + assert addr2 == MEM_BLOCK_ADDRESS + assert allocator.get_free_memory() == 0 + assert allocator.get_allocated_memory() == 1024 diff --git a/vyper/exceptions.py b/vyper/exceptions.py index c69163b561..7b98818fcd 100644 --- a/vyper/exceptions.py +++ b/vyper/exceptions.py @@ -400,10 +400,6 @@ class CodegenPanic(VyperInternalException): """Invalid code generated during codegen phase""" -class StackTooDeep(CodegenPanic): - """Stack too deep""" # (should not happen) - - class UnexpectedNodeType(VyperInternalException): """Unexpected AST node type.""" @@ -424,6 +420,15 @@ class InvalidABIType(VyperInternalException): """An internal routine constructed an invalid ABI type""" +class UnreachableStackException(VyperException): + + """An unreachable stack operation was encountered.""" + + def __init__(self, message, op): + self.op = op + super().__init__(message) + + @contextlib.contextmanager def tag_exceptions(node, fallback_exception_type=CompilerPanic, note=None): try: diff --git a/vyper/semantics/analysis/module.py b/vyper/semantics/analysis/module.py index d05e494b80..2f4cffcf96 100644 --- a/vyper/semantics/analysis/module.py +++ b/vyper/semantics/analysis/module.py @@ -898,7 +898,7 @@ def _import_to_path(level: int, module_str: str) -> PurePath: base_path = "../" * (level - 1) elif level == 1: base_path = "./" - return PurePath(f"{base_path}{module_str.replace('.','/')}/") + return PurePath(f"{base_path}{module_str.replace('.', '/')}/") # can add more, e.g. "vyper.builtins.interfaces", etc. diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index afd79fc44f..add63d3764 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -6,6 +6,7 @@ from vyper.codegen.ir_node import IRnode from vyper.compiler.settings import OptimizationLevel from vyper.venom.analysis.analysis import IRAnalysesCache +from vyper.venom.basicblock import IRVariable from vyper.venom.context import IRContext from vyper.venom.function import IRFunction from vyper.venom.ir_node_to_venom import ir_node_to_venom @@ -18,6 +19,7 @@ from vyper.venom.passes.remove_unused_variables import RemoveUnusedVariablesPass from vyper.venom.passes.sccp import SCCP from vyper.venom.passes.simplify_cfg import SimplifyCFGPass +from vyper.venom.passes.stack2mem import Stack2Mem from vyper.venom.passes.store_elimination import StoreElimination from vyper.venom.venom_to_assembly import VenomCompiler @@ -57,6 +59,7 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: ExtractLiteralsPass(ac, fn).run_pass() RemoveUnusedVariablesPass(ac, fn).run_pass() DFTPass(ac, fn).run_pass() + Stack2Mem(ac, fn).run_pass() def generate_ir(ir: IRnode, optimize: OptimizationLevel) -> IRContext: diff --git a/vyper/venom/context.py b/vyper/venom/context.py index 0b0252d976..caca12f8ca 100644 --- a/vyper/venom/context.py +++ b/vyper/venom/context.py @@ -2,6 +2,7 @@ from vyper.venom.basicblock import IRInstruction, IRLabel, IROperand from vyper.venom.function import IRFunction +from vyper.venom.mem_allocator import MemoryAllocator class IRContext: @@ -10,6 +11,7 @@ class IRContext: immutables_len: Optional[int] data_segment: list[IRInstruction] last_label: int + mem_allocator: MemoryAllocator def __init__(self) -> None: self.functions = {} @@ -17,6 +19,9 @@ def __init__(self) -> None: self.immutables_len = None self.data_segment = [] self.last_label = 0 + self.mem_allocator = MemoryAllocator( + 4096, 0x100000 + ) # TODO: Should get this from the original IR def add_function(self, fn: IRFunction) -> None: fn.ctx = self diff --git a/vyper/venom/mem_allocator.py b/vyper/venom/mem_allocator.py new file mode 100644 index 0000000000..66f5ec6c2e --- /dev/null +++ b/vyper/venom/mem_allocator.py @@ -0,0 +1,58 @@ +from typing import List + + +class MemoryBlock: + size: int + address: int + is_free: bool + + def __init__(self, size: int, address: int): + self.size = size + self.address = address + self.is_free = True + + +class MemoryAllocator: + total_size: int + start_address: int + blocks: List[MemoryBlock] + + def __init__(self, total_size: int, start_address: int): + self.total_size = total_size + self.start_address = start_address + self.blocks = [MemoryBlock(total_size, 0)] + + def allocate(self, size: int) -> int: + for block in self.blocks: + if block.is_free and block.size >= size: + if block.size > size: + new_block = MemoryBlock(block.size - size, block.address + size) + self.blocks.insert(self.blocks.index(block) + 1, new_block) + block.size = size + block.is_free = False + return self.start_address + block.address + raise MemoryError("Memory allocation failed") + + def deallocate(self, address: int) -> bool: + relative_address = address - self.start_address + for block in self.blocks: + if block.address == relative_address: + block.is_free = True + self._merge_adjacent_free_blocks() + return True + return False # invalid address + + def _merge_adjacent_free_blocks(self) -> None: + i = 0 + while i < len(self.blocks) - 1: + if self.blocks[i].is_free and self.blocks[i + 1].is_free: + self.blocks[i].size += self.blocks[i + 1].size + self.blocks.pop(i + 1) + else: + i += 1 + + def get_free_memory(self) -> int: + return sum(block.size for block in self.blocks if block.is_free) + + def get_allocated_memory(self) -> int: + return sum(block.size for block in self.blocks if not block.is_free) diff --git a/vyper/venom/passes/simplify_cfg.py b/vyper/venom/passes/simplify_cfg.py index 08582fee96..0227192706 100644 --- a/vyper/venom/passes/simplify_cfg.py +++ b/vyper/venom/passes/simplify_cfg.py @@ -11,11 +11,11 @@ class SimplifyCFGPass(IRPass): def _merge_blocks(self, a: IRBasicBlock, b: IRBasicBlock): a.instructions.pop() for inst in b.instructions: - assert inst.opcode != "phi", "Not implemented yet" + inst.parent = a + if inst.opcode == "phi": a.instructions.insert(0, inst) else: - inst.parent = a a.instructions.append(inst) # Update CFG diff --git a/vyper/venom/passes/stack2mem.py b/vyper/venom/passes/stack2mem.py new file mode 100644 index 0000000000..31db891ee6 --- /dev/null +++ b/vyper/venom/passes/stack2mem.py @@ -0,0 +1,70 @@ +from vyper.exceptions import UnreachableStackException +from vyper.venom.analysis.cfg import CFGAnalysis +from vyper.venom.analysis.dfg import DFGAnalysis +from vyper.venom.analysis.liveness import LivenessAnalysis +from vyper.venom.basicblock import IRInstruction, IRLiteral, IRVariable +from vyper.venom.mem_allocator import MemoryAllocator +from vyper.venom.passes.base_pass import IRPass +from vyper.venom.venom_to_assembly import VenomCompiler + + +class Stack2Mem(IRPass): + mem_allocator: MemoryAllocator + + def run_pass(self): + fn = self.function + self.mem_allocator = self.function.ctx.mem_allocator + self.analyses_cache.request_analysis(CFGAnalysis) + dfg = self.analyses_cache.request_analysis(DFGAnalysis) + self.analyses_cache.request_analysis(LivenessAnalysis) + + while True: + compiler = VenomCompiler([fn.ctx]) + try: + compiler.generate_evm() + break + except Exception as e: + if isinstance(e, UnreachableStackException): + self._demote_variable(dfg, e.op) + self.analyses_cache.force_analysis(LivenessAnalysis) + else: + break + + self.analyses_cache.invalidate_analysis(DFGAnalysis) + + def _demote_variable(self, dfg: DFGAnalysis, var: IRVariable): + """ + Demote a stack variable to memory operations. + """ + uses = dfg.get_uses(var) + def_inst = dfg.get_producing_instruction(var) + + # Allocate memory for this variable + mem_addr = self.mem_allocator.allocate(32) + + if def_inst is not None: + self._insert_mstore_after(def_inst, mem_addr) + + for inst in uses: + self._insert_mload_before(inst, mem_addr, var) + + def _insert_mstore_after(self, inst: IRInstruction, mem_addr: int): + bb = inst.parent + idx = bb.instructions.index(inst) + assert inst.output is not None + # mem_var = IRVariable(f"mem_{mem_addr}") + # bb.insert_instruction( + # IRInstruction("alloca", [IRLiteral(mem_addr), 32], mem_var), idx + 1 + # ) + new_var = self.function.get_next_variable() + bb.insert_instruction(IRInstruction("mstore", [new_var, IRLiteral(mem_addr)]), idx + 1) + inst.output = new_var + + def _insert_mload_before(self, inst: IRInstruction, mem_addr: int, var: IRVariable): + bb = inst.parent + idx = bb.instructions.index(inst) + new_var = self.function.get_next_variable() + load_inst = IRInstruction("mload", [IRLiteral(mem_addr)]) + load_inst.output = new_var + bb.insert_instruction(load_inst, idx) + inst.replace_operands({var: new_var}) diff --git a/vyper/venom/stack_model.py b/vyper/venom/stack_model.py index a98e5bb25b..311b1a5e06 100644 --- a/vyper/venom/stack_model.py +++ b/vyper/venom/stack_model.py @@ -27,6 +27,12 @@ def push(self, op: IROperand) -> None: assert isinstance(op, IROperand), f"{type(op)}: {op}" self._stack.append(op) + def top(self) -> IROperand: + """ + Returns the top of the stack map. + """ + return self._stack[-1] + def pop(self, num: int = 1) -> None: del self._stack[len(self._stack) - num :] @@ -46,7 +52,7 @@ def get_depth(self, op: IROperand, n: int = 1) -> int: return StackModel.NOT_IN_STACK # type: ignore - def get_phi_depth(self, phis: list[IRVariable]) -> int: + def get_phi_depth(self, phis: list[IRVariable]) -> tuple[int, IROperand]: """ Returns the depth of the first matching phi variable in the stack map. If the none of the phi operands are in the stack, returns NOT_IN_STACK. @@ -55,14 +61,16 @@ def get_phi_depth(self, phis: list[IRVariable]) -> int: assert isinstance(phis, list) ret = StackModel.NOT_IN_STACK + ret_op = None for i, stack_item in enumerate(reversed(self._stack)): if stack_item in phis: assert ( ret is StackModel.NOT_IN_STACK ), f"phi argument is not unique! {phis}, {self._stack}" ret = -i + ret_op = stack_item - return ret # type: ignore + return ret, ret_op # type: ignore def peek(self, depth: int) -> IROperand: """ diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 07d63afc70..2773537c6f 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -1,7 +1,7 @@ from collections import Counter from typing import Any -from vyper.exceptions import CompilerPanic, StackTooDeep +from vyper.exceptions import CompilerPanic, UnreachableStackException from vyper.ir.compile_ir import ( PUSH, DataHeader, @@ -388,7 +388,7 @@ def _generate_evm_for_instruction( if opcode == "phi": ret = inst.get_outputs()[0] phis = list(inst.get_input_variables()) - depth = stack.get_phi_depth(phis) + depth, op = stack.get_phi_depth(phis) # collapse the arguments to the phi node in the stack. # example, for `%56 = %label1 %13 %label2 %14`, we will # find an instance of %13 *or* %14 in the stack and replace it with %56. @@ -553,39 +553,40 @@ def _generate_evm_for_instruction( return apply_line_numbers(inst, assembly) - def pop(self, assembly, stack, num=1): + def pop(self, assembly, stack: StackModel, num=1): stack.pop(num) assembly.extend(["POP"] * num) - def swap(self, assembly, stack, depth) -> int: + def swap(self, assembly, stack: StackModel, depth) -> int: # Swaps of the top is no op if depth == 0: return 0 - stack.swap(depth) - assembly.append(_evm_swap_for(depth)) + assembly.append(_evm_swap_for(depth, stack.top())) return 1 - def dup(self, assembly, stack, depth): + def dup(self, assembly, stack: StackModel, depth): stack.dup(depth) - assembly.append(_evm_dup_for(depth)) + assembly.append(_evm_dup_for(depth, stack.top())) - def swap_op(self, assembly, stack, op): - self.swap(assembly, stack, stack.get_depth(op)) + def swap_op(self, assembly, stack: StackModel, op): + depth = stack.get_depth(op) + self.swap(assembly, stack, depth) - def dup_op(self, assembly, stack, op): - self.dup(assembly, stack, stack.get_depth(op)) + def dup_op(self, assembly, stack: StackModel, op): + depth = stack.get_depth(op) + self.dup(assembly, stack, depth) -def _evm_swap_for(depth: int) -> str: +def _evm_swap_for(depth: int, op: IROperand) -> str: swap_idx = -depth if not (1 <= swap_idx <= 16): - raise StackTooDeep(f"Unsupported swap depth {swap_idx}") + raise UnreachableStackException(f"Unsupported swap depth {swap_idx} ({op})", op) return f"SWAP{swap_idx}" -def _evm_dup_for(depth: int) -> str: +def _evm_dup_for(depth: int, op: IROperand) -> str: dup_idx = 1 - depth if not (1 <= dup_idx <= 16): - raise StackTooDeep(f"Unsupported dup depth {dup_idx}") + raise UnreachableStackException(f"Unsupported dup depth {dup_idx} ({op})", op) return f"DUP{dup_idx}" From f4c7ac528bc1128fbea307f06da6656635121e85 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 18 Sep 2024 10:25:15 +0300 Subject: [PATCH 2/8] revert `get_phi_depth()` changes --- vyper/venom/stack_model.py | 6 ++---- vyper/venom/venom_to_assembly.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/vyper/venom/stack_model.py b/vyper/venom/stack_model.py index 311b1a5e06..2162fb171d 100644 --- a/vyper/venom/stack_model.py +++ b/vyper/venom/stack_model.py @@ -52,7 +52,7 @@ def get_depth(self, op: IROperand, n: int = 1) -> int: return StackModel.NOT_IN_STACK # type: ignore - def get_phi_depth(self, phis: list[IRVariable]) -> tuple[int, IROperand]: + def get_phi_depth(self, phis: list[IRVariable]) -> int: """ Returns the depth of the first matching phi variable in the stack map. If the none of the phi operands are in the stack, returns NOT_IN_STACK. @@ -61,16 +61,14 @@ def get_phi_depth(self, phis: list[IRVariable]) -> tuple[int, IROperand]: assert isinstance(phis, list) ret = StackModel.NOT_IN_STACK - ret_op = None for i, stack_item in enumerate(reversed(self._stack)): if stack_item in phis: assert ( ret is StackModel.NOT_IN_STACK ), f"phi argument is not unique! {phis}, {self._stack}" ret = -i - ret_op = stack_item - return ret, ret_op # type: ignore + return ret # type: ignore def peek(self, depth: int) -> IROperand: """ diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 2773537c6f..b4fd8e547b 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -388,7 +388,7 @@ def _generate_evm_for_instruction( if opcode == "phi": ret = inst.get_outputs()[0] phis = list(inst.get_input_variables()) - depth, op = stack.get_phi_depth(phis) + depth = stack.get_phi_depth(phis) # collapse the arguments to the phi node in the stack. # example, for `%56 = %label1 %13 %label2 %14`, we will # find an instance of %13 *or* %14 in the stack and replace it with %56. From cc50efe777a5bee0c563d87a11f26ad1f9d61f77 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 18 Sep 2024 10:27:37 +0300 Subject: [PATCH 3/8] revert change that slipped in from another branch --- vyper/venom/passes/simplify_cfg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vyper/venom/passes/simplify_cfg.py b/vyper/venom/passes/simplify_cfg.py index 0227192706..08582fee96 100644 --- a/vyper/venom/passes/simplify_cfg.py +++ b/vyper/venom/passes/simplify_cfg.py @@ -11,11 +11,11 @@ class SimplifyCFGPass(IRPass): def _merge_blocks(self, a: IRBasicBlock, b: IRBasicBlock): a.instructions.pop() for inst in b.instructions: - inst.parent = a - + assert inst.opcode != "phi", "Not implemented yet" if inst.opcode == "phi": a.instructions.insert(0, inst) else: + inst.parent = a a.instructions.append(inst) # Update CFG From 28eed40245dbee9fb1fe1b932f83b59ece168cf9 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Thu, 19 Sep 2024 09:08:27 +0300 Subject: [PATCH 4/8] `StackModel` refactor --- vyper/venom/stack_model.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/vyper/venom/stack_model.py b/vyper/venom/stack_model.py index 2162fb171d..0d35090140 100644 --- a/vyper/venom/stack_model.py +++ b/vyper/venom/stack_model.py @@ -27,12 +27,6 @@ def push(self, op: IROperand) -> None: assert isinstance(op, IROperand), f"{type(op)}: {op}" self._stack.append(op) - def top(self) -> IROperand: - """ - Returns the top of the stack map. - """ - return self._stack[-1] - def pop(self, num: int = 1) -> None: del self._stack[len(self._stack) - num :] @@ -72,23 +66,30 @@ def get_phi_depth(self, phis: list[IRVariable]) -> int: def peek(self, depth: int) -> IROperand: """ - Returns the top of the stack map. + Returns the depth-th element from the top of the stack. """ assert depth is not StackModel.NOT_IN_STACK, "Cannot peek non-in-stack depth" + assert depth <= 0, "Cannot peek positive depth" return self._stack[depth - 1] def poke(self, depth: int, op: IROperand) -> None: """ - Pokes an operand at the given depth in the stack map. + Pokes an operand at the given depth in the stack. """ assert depth is not StackModel.NOT_IN_STACK, "Cannot poke non-in-stack depth" - assert depth <= 0, "Bad depth" + assert depth <= 0, "Cannot poke positive depth" assert isinstance(op, IROperand), f"{type(op)}: {op}" self._stack[depth - 1] = op + def top(self) -> IROperand: + """ + Returns the top of the stack. + """ + return self.peek(0) + def dup(self, depth: int) -> None: """ - Duplicates the operand at the given depth in the stack map. + Duplicates the operand at the given depth in the stack. """ assert depth is not StackModel.NOT_IN_STACK, "Cannot dup non-existent operand" assert depth <= 0, "Cannot dup positive depth" @@ -96,7 +97,7 @@ def dup(self, depth: int) -> None: def swap(self, depth: int) -> None: """ - Swaps the operand at the given depth in the stack map with the top of the stack. + Swaps the operand at the given depth in the stack with the top of the stack. """ assert depth is not StackModel.NOT_IN_STACK, "Cannot swap non-existent operand" assert depth < 0, "Cannot swap positive depth" From 3836b8716fa663de56ea2074eec8338f2a860164 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Fri, 4 Oct 2024 12:19:19 +0300 Subject: [PATCH 5/8] wip --- vyper/venom/__init__.py | 6 ++++-- vyper/venom/function.py | 4 ++++ vyper/venom/mem_allocator.py | 3 ++- vyper/venom/passes/alloca_elimination.py | 22 ++++++++++++++++++++++ vyper/venom/venom_to_assembly.py | 3 +++ 5 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 vyper/venom/passes/alloca_elimination.py diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index add63d3764..5ac3af4cdf 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -11,6 +11,7 @@ from vyper.venom.function import IRFunction from vyper.venom.ir_node_to_venom import ir_node_to_venom from vyper.venom.passes.algebraic_optimization import AlgebraicOptimizationPass +from vyper.venom.passes.alloca_elimination import AllocaElimination from vyper.venom.passes.branch_optimization import BranchOptimizationPass from vyper.venom.passes.dft import DFTPass from vyper.venom.passes.extract_literals import ExtractLiteralsPass @@ -48,15 +49,16 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: ac = IRAnalysesCache(fn) SimplifyCFGPass(ac, fn).run_pass() + AllocaElimination(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() Mem2Var(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() - SCCP(ac, fn).run_pass() + #SCCP(ac, fn).run_pass() StoreElimination(ac, fn).run_pass() SimplifyCFGPass(ac, fn).run_pass() AlgebraicOptimizationPass(ac, fn).run_pass() BranchOptimizationPass(ac, fn).run_pass() - ExtractLiteralsPass(ac, fn).run_pass() + # ExtractLiteralsPass(ac, fn).run_pass() RemoveUnusedVariablesPass(ac, fn).run_pass() DFTPass(ac, fn).run_pass() Stack2Mem(ac, fn).run_pass() diff --git a/vyper/venom/function.py b/vyper/venom/function.py index fb0dabc99a..4bb61b6875 100644 --- a/vyper/venom/function.py +++ b/vyper/venom/function.py @@ -3,6 +3,7 @@ from vyper.codegen.ir_node import IRnode from vyper.utils import OrderedSet from vyper.venom.basicblock import CFG_ALTERING_INSTRUCTIONS, IRBasicBlock, IRLabel, IRVariable +from vyper.venom.mem_allocator import MemoryAllocator class IRFunction: @@ -16,6 +17,7 @@ class IRFunction: last_label: int last_variable: int _basic_block_dict: dict[str, IRBasicBlock] + _mem_allocator: MemoryAllocator # Used during code generation _ast_source_stack: list[IRnode] @@ -32,6 +34,8 @@ def __init__(self, name: IRLabel, ctx: "IRContext" = None) -> None: # type: ign self._ast_source_stack = [] self._error_msg_stack = [] + self._mem_allocator = MemoryAllocator(0xFFFFFFFFFFFFFFFF, 32) + self.append_basic_block(IRBasicBlock(name, self)) @property diff --git a/vyper/venom/mem_allocator.py b/vyper/venom/mem_allocator.py index 66f5ec6c2e..864a15fc1b 100644 --- a/vyper/venom/mem_allocator.py +++ b/vyper/venom/mem_allocator.py @@ -23,6 +23,7 @@ def __init__(self, total_size: int, start_address: int): self.blocks = [MemoryBlock(total_size, 0)] def allocate(self, size: int) -> int: + print(f"Allocating {size} bytes with free memory {self.get_free_memory()}") for block in self.blocks: if block.is_free and block.size >= size: if block.size > size: @@ -31,7 +32,7 @@ def allocate(self, size: int) -> int: block.size = size block.is_free = False return self.start_address + block.address - raise MemoryError("Memory allocation failed") + raise MemoryError(f"Memory allocation failed for size {size} with free memory {self.get_free_memory()}") def deallocate(self, address: int) -> bool: relative_address = address - self.start_address diff --git a/vyper/venom/passes/alloca_elimination.py b/vyper/venom/passes/alloca_elimination.py new file mode 100644 index 0000000000..191d1ed57d --- /dev/null +++ b/vyper/venom/passes/alloca_elimination.py @@ -0,0 +1,22 @@ +from vyper.venom.basicblock import IRInstruction, IRLiteral + +from vyper.venom.passes.base_pass import IRPass + + +class AllocaElimination(IRPass): + """ + This pass eliminates alloca instructions by allocating memory for them + """ + + def run_pass(self): + for bb in self.function.get_basic_blocks(): + for inst in bb.instructions: + if inst.opcode == "alloca": + self._process_alloca(inst) + + def _process_alloca(self, inst: IRInstruction): + offset, _size = inst.operands + address = inst.parent.parent._mem_allocator.allocate(_size.value) + inst.opcode = "store" + inst.operands = [IRLiteral(address)] + # print(f"Allocated address {address} for alloca {_size.value}") diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index c0fefa907b..ab9d8017a3 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -346,7 +346,10 @@ def _generate_evm_for_instruction( if opcode in ["jmp", "djmp", "jnz", "invoke"]: operands = list(inst.get_non_label_operands()) elif opcode == "alloca": + raise Exception("Alloca at assembly generation is not valid") offset, _size = inst.operands + offset = inst.parent.parent._mem_allocator.allocate(_size) + print(f"Allocated {offset} for alloca {_size}") operands = [offset] # iload and istore are special cases because they can take a literal From 52efbafa1fd5e403d4b3495a164067d905a001e5 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 16 Oct 2024 22:25:46 +0300 Subject: [PATCH 6/8] refactor to the new pass importing --- vyper/venom/__init__.py | 1 + vyper/venom/passes/__init__.py | 2 ++ vyper/venom/venom_to_assembly.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index 989e6d653e..52f7552504 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -21,6 +21,7 @@ StoreElimination, StoreExpansionPass, AllocaElimination, + Stack2Mem, ) from vyper.venom.venom_to_assembly import VenomCompiler diff --git a/vyper/venom/passes/__init__.py b/vyper/venom/passes/__init__.py index 83098234c1..444cbdbe45 100644 --- a/vyper/venom/passes/__init__.py +++ b/vyper/venom/passes/__init__.py @@ -9,3 +9,5 @@ from .simplify_cfg import SimplifyCFGPass from .store_elimination import StoreElimination from .store_expansion import StoreExpansionPass +from .alloca_elimination import AllocaElimination +from .stack2mem import Stack2Mem \ No newline at end of file diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 936070e2fe..011e6438de 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -359,7 +359,7 @@ def _generate_evm_for_instruction( if opcode in ["jmp", "djmp", "jnz", "invoke"]: operands = list(inst.get_non_label_operands()) elif opcode in ["alloca", "palloca"]: - raise Exception("Alloca at assembly generation is not valid") + #raise Exception("Alloca at assembly generation is not valid") offset, _size = inst.operands offset = inst.parent.parent._mem_allocator.allocate(_size) print(f"Allocated {offset} for alloca {_size}") From 9212c86d548ffd33b11808dc60f70fe869bebfb0 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 16 Oct 2024 22:32:49 +0300 Subject: [PATCH 7/8] add back SCCP --- vyper/venom/__init__.py | 3 ++- vyper/venom/mem_allocator.py | 2 +- vyper/venom/venom_to_assembly.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index 52f7552504..41e47cd7db 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -11,6 +11,7 @@ from vyper.venom.function import IRFunction from vyper.venom.ir_node_to_venom import ir_node_to_venom from vyper.venom.passes import ( + SCCP, AlgebraicOptimizationPass, BranchOptimizationPass, DFTPass, @@ -54,7 +55,7 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: MakeSSA(ac, fn).run_pass() Mem2Var(ac, fn).run_pass() MakeSSA(ac, fn).run_pass() - #SCCP(ac, fn).run_pass() + SCCP(ac, fn).run_pass() StoreElimination(ac, fn).run_pass() SimplifyCFGPass(ac, fn).run_pass() AlgebraicOptimizationPass(ac, fn).run_pass() diff --git a/vyper/venom/mem_allocator.py b/vyper/venom/mem_allocator.py index 864a15fc1b..6323561bd5 100644 --- a/vyper/venom/mem_allocator.py +++ b/vyper/venom/mem_allocator.py @@ -23,7 +23,7 @@ def __init__(self, total_size: int, start_address: int): self.blocks = [MemoryBlock(total_size, 0)] def allocate(self, size: int) -> int: - print(f"Allocating {size} bytes with free memory {self.get_free_memory()}") + #print(f"Allocating {size} bytes with free memory {self.get_free_memory()}") for block in self.blocks: if block.is_free and block.size >= size: if block.size > size: diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 011e6438de..482a1085ec 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -361,8 +361,8 @@ def _generate_evm_for_instruction( elif opcode in ["alloca", "palloca"]: #raise Exception("Alloca at assembly generation is not valid") offset, _size = inst.operands - offset = inst.parent.parent._mem_allocator.allocate(_size) - print(f"Allocated {offset} for alloca {_size}") + offset = inst.parent.parent._mem_allocator.allocate(_size.value) + # print(f"Allocated {offset} for alloca {_size}") operands = [offset] # iload and istore are special cases because they can take a literal From 414ca33236acd4160c5c053ff98dbe553d0a40e9 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Wed, 16 Oct 2024 22:52:33 +0300 Subject: [PATCH 8/8] fixes and cleanup --- vyper/venom/__init__.py | 4 ++-- vyper/venom/mem_allocator.py | 6 ++++-- vyper/venom/passes/__init__.py | 4 ++-- vyper/venom/passes/alloca_elimination.py | 1 - vyper/venom/venom_to_assembly.py | 7 +++++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index 41e47cd7db..3c3e932300 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -13,16 +13,16 @@ from vyper.venom.passes import ( SCCP, AlgebraicOptimizationPass, + AllocaElimination, BranchOptimizationPass, DFTPass, MakeSSA, Mem2Var, RemoveUnusedVariablesPass, SimplifyCFGPass, + Stack2Mem, StoreElimination, StoreExpansionPass, - AllocaElimination, - Stack2Mem, ) from vyper.venom.venom_to_assembly import VenomCompiler diff --git a/vyper/venom/mem_allocator.py b/vyper/venom/mem_allocator.py index 6323561bd5..625f6746b9 100644 --- a/vyper/venom/mem_allocator.py +++ b/vyper/venom/mem_allocator.py @@ -23,7 +23,7 @@ def __init__(self, total_size: int, start_address: int): self.blocks = [MemoryBlock(total_size, 0)] def allocate(self, size: int) -> int: - #print(f"Allocating {size} bytes with free memory {self.get_free_memory()}") + # print(f"Allocating {size} bytes with free memory {self.get_free_memory()}") for block in self.blocks: if block.is_free and block.size >= size: if block.size > size: @@ -32,7 +32,9 @@ def allocate(self, size: int) -> int: block.size = size block.is_free = False return self.start_address + block.address - raise MemoryError(f"Memory allocation failed for size {size} with free memory {self.get_free_memory()}") + raise MemoryError( + f"Memory allocation failed for size {size} with free memory {self.get_free_memory()}" + ) def deallocate(self, address: int) -> bool: relative_address = address - self.start_address diff --git a/vyper/venom/passes/__init__.py b/vyper/venom/passes/__init__.py index 444cbdbe45..0a4b7ab0bf 100644 --- a/vyper/venom/passes/__init__.py +++ b/vyper/venom/passes/__init__.py @@ -1,4 +1,5 @@ from .algebraic_optimization import AlgebraicOptimizationPass +from .alloca_elimination import AllocaElimination from .branch_optimization import BranchOptimizationPass from .dft import DFTPass from .make_ssa import MakeSSA @@ -7,7 +8,6 @@ from .remove_unused_variables import RemoveUnusedVariablesPass from .sccp import SCCP from .simplify_cfg import SimplifyCFGPass +from .stack2mem import Stack2Mem from .store_elimination import StoreElimination from .store_expansion import StoreExpansionPass -from .alloca_elimination import AllocaElimination -from .stack2mem import Stack2Mem \ No newline at end of file diff --git a/vyper/venom/passes/alloca_elimination.py b/vyper/venom/passes/alloca_elimination.py index 191d1ed57d..45fe8a9c7e 100644 --- a/vyper/venom/passes/alloca_elimination.py +++ b/vyper/venom/passes/alloca_elimination.py @@ -1,5 +1,4 @@ from vyper.venom.basicblock import IRInstruction, IRLiteral - from vyper.venom.passes.base_pass import IRPass diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 482a1085ec..d0aae5f1c1 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -358,12 +358,15 @@ def _generate_evm_for_instruction( if opcode in ["jmp", "djmp", "jnz", "invoke"]: operands = list(inst.get_non_label_operands()) - elif opcode in ["alloca", "palloca"]: - #raise Exception("Alloca at assembly generation is not valid") + elif opcode == "alloca": + raise Exception("Alloca at assembly generation is not valid") offset, _size = inst.operands offset = inst.parent.parent._mem_allocator.allocate(_size.value) # print(f"Allocated {offset} for alloca {_size}") operands = [offset] + elif opcode == "palloca": + offset, _size = inst.operands + operands = [offset] # iload and istore are special cases because they can take a literal # that is handled specialy with the _OFST macro. Look below, after the