Merge branch 'main' into initial_struct_support

decompiler/backend/cexpressiongenerator.py

@@ -239,6 +239,9 @@ def visit_constant_composition(self, expr: expressions.ConstantComposition):
             case CustomType(text="wchar16") | CustomType(text="wchar32"):
                 val = "".join([x.value for x in expr.value])
                 return f'L"{val}"' if len(val) <= MAX_GLOBAL_INIT_LENGTH else f'L"{val[:MAX_GLOBAL_INIT_LENGTH]}..."'
+            case Integer(size=8, signed=False):
+                val = "".join([f"\\x{x.value:02X}" for x in expr.value][:MAX_GLOBAL_INIT_LENGTH])
+                return f'"{val}"' if len(val) <= MAX_GLOBAL_INIT_LENGTH else f'"{val[:MAX_GLOBAL_INIT_LENGTH]}..."'
             case Integer(8):
                 val = "".join([x.value for x in expr.value][:MAX_GLOBAL_INIT_LENGTH])
                 return f'"{val}"' if len(val) <= MAX_GLOBAL_INIT_LENGTH else f'"{val[:MAX_GLOBAL_INIT_LENGTH]}..."'

decompiler/backend/variabledeclarations.py

@@ -70,7 +70,7 @@ def _generate_definitions(global_variables: set[GlobalVariable]) -> Iterator[str
             match variable.type:
                 case ArrayType():
                     br, bl = "", ""
-                    if not variable.type.type in [Integer.char(), CustomType.wchar16(), CustomType.wchar32()]:
+                    if not variable.type.type in [Integer.char(), Integer.uint8_t(), CustomType.wchar16(), CustomType.wchar32()]:
                         br, bl = "{", "}"
                     yield f"{base}{variable.type.type} {variable.name}[{hex(variable.type.elements)}] = {br}{CExpressionGenerator().visit(variable.initial_value)}{bl};"
                 case Struct():

decompiler/frontend/binaryninja/handlers/constants.py

@@ -72,9 +72,6 @@ def lift_constant_pointer(self, pointer: mediumlevelil.MediumLevelILConstPtr, **
         if isinstance(res, Constant):  # BNinja Error case handling
             return res
 
-        if isinstance(res.type, Pointer) and res.type.type == CustomType.void():
-            return res
-
         if isinstance(pointer, mediumlevelil.MediumLevelILImport):  # Temp fix for '&'
             return res
 

decompiler/frontend/binaryninja/handlers/globals.py

@@ -308,7 +308,9 @@ def _get_unknown_value(self, variable: DataVariable):
             type = PseudoArrayType(self._lifter.lift(data[1]), len(data[0]))
             data = ConstantComposition([Constant(x, type.type) for x in data[0]], type)
         else:
-            data, type = get_raw_bytes(variable.address, self._view), Pointer(CustomType.void(), self._view.address_size * BYTE_SIZE)
+            rbytes = get_raw_bytes(variable.address, self._view)
+            type = PseudoArrayType(Integer.uint8_t(), len(rbytes))
+            data = ConstantComposition([Constant(b, type.type) for b in rbytes], type)
         return data, type
 
     def _get_unknown_pointer_value(self, variable: DataVariable, callers: list[int] = None):

decompiler/pipeline/commons/expressionpropagationcommons.py

@@ -225,8 +225,9 @@ def _is_address_into_dereference(self, definition: Assignment, target: Instructi
         if self._is_address(definition.value):
             for subexpr in target:
                 for sub in self._find_subexpressions(subexpr):
-                    if self._is_dereference(sub) and sub.operand == definition.destination:
+                    if self._is_dereference(sub) and sub.operand in definition.definitions:
                         return True
+        return False
 
     def _contains_aliased_variables(self, definition: Assignment) -> bool:
         """
@@ -326,14 +327,13 @@ def _has_any_of_dangerous_uses_between_definition_and_target(
     def _get_dangerous_uses_of_variable_address(self, var: Variable) -> Set[Instruction]:
         """
         Dangerous use of & of x is func(&x) cause it can potentially modify x.
-        *(&x) could also do the job but I consider it to be too exotic so that we could get such instruction from Binary Ninja
-        If it happens we can handle it later.
+        Another case is an Assignment where the left side is *(&).
         :param var: aliased variable
         :return: set of function call assignments that take &var as parameter
         """
         dangerous_uses = set()
         for use in self._use_map.get(var):
-            if not self._is_call_assignment(use):
+            if not self._is_call_assignment(use) and not (isinstance(use, Assignment) and self._is_dereference(use.destination)):
                 continue
             for subexpr in self._find_subexpressions(use):
                 if self._is_address(subexpr):

decompiler/pipeline/controlflowanalysis/expression_simplification/constant_folding.py

@@ -65,7 +65,12 @@ def constant_fold(operation: OperationType, constants: list[Constant], result_ty
     )
 
 
-def _constant_fold_arithmetic_binary(constants: list[Constant], fun: Callable[[int, int], int], norm_sign: Optional[bool] = None) -> int:
+def _constant_fold_arithmetic_binary(
+    constants: list[Constant],
+    fun: Callable[[int, int], int],
+    norm_sign: Optional[bool] = None,
+    allow_mismatched_sizes: bool = False,
+) -> int:
     """
     Fold an arithmetic binary operation with constants as operands.
 
@@ -84,7 +89,7 @@ def _constant_fold_arithmetic_binary(constants: list[Constant], fun: Callable[[i
 
     if len(constants) != 2:
         raise IncompatibleOperandCount(f"Expected exactly 2 constants to fold, got {len(constants)}.")
-    if not all(constant.type.size == constants[0].type.size for constant in constants):
+    if not allow_mismatched_sizes and not all(constant.type.size == constants[0].type.size for constant in constants):
         raise UnsupportedMismatchedSizes(f"Can not fold constants with different sizes: {[constant.type for constant in constants]}")
 
     left, right = constants
@@ -137,13 +142,19 @@ def _constant_fold_shift(constants: list[Constant], fun: Callable[[int, int], in
     return fun(normalize_int(left.value, left.type.size, norm_signed), right.value)
 
 
+def remainder(n, d):
+    return (-1 if n < 0 else 1) * (n % d)
+
+
 _OPERATION_TO_FOLD_FUNCTION: dict[OperationType, Callable[[list[Constant]], int]] = {
     OperationType.minus: partial(_constant_fold_arithmetic_binary, fun=operator.sub),
     OperationType.plus: partial(_constant_fold_arithmetic_binary, fun=operator.add),
     OperationType.multiply: partial(_constant_fold_arithmetic_binary, fun=operator.mul, norm_sign=True),
     OperationType.multiply_us: partial(_constant_fold_arithmetic_binary, fun=operator.mul, norm_sign=False),
     OperationType.divide: partial(_constant_fold_arithmetic_binary, fun=operator.floordiv, norm_sign=True),
     OperationType.divide_us: partial(_constant_fold_arithmetic_binary, fun=operator.floordiv, norm_sign=False),
+    OperationType.modulo: partial(_constant_fold_arithmetic_binary, fun=remainder, norm_sign=True, allow_mismatched_sizes=True),
+    OperationType.modulo_us: partial(_constant_fold_arithmetic_binary, fun=operator.mod, norm_sign=False, allow_mismatched_sizes=True),
     OperationType.negate: partial(_constant_fold_arithmetic_unary, fun=operator.neg),
     OperationType.left_shift: partial(_constant_fold_shift, fun=operator.lshift, signed=True),
     OperationType.right_shift: partial(_constant_fold_shift, fun=operator.rshift, signed=True),

decompiler/pipeline/controlflowanalysis/restructuring_commons/acyclic_restructuring.py

@@ -94,7 +94,10 @@ def _construct_refined_ast(self, seq_node_root: SeqNode) -> AbstractSyntaxTreeNo
         ConditionBasedRefinement.refine(self.asforest)
         acyclic_processor.preprocess_condition_aware_refinement()
         if self.options.reconstruct_switch:
-            ConditionAwareRefinement.refine(self.asforest, self.options)
+            updated_switch_nodes = ConditionAwareRefinement.refine(self.asforest, self.options)
+            for switch_node in updated_switch_nodes:
+                for sequence_case in (c for c in switch_node.cases if isinstance(c.child, SeqNode)):
+                    ConditionBasedRefinement.refine(self.asforest, sequence_case.child)
         acyclic_processor.postprocess_condition_refinement()
         root = self.asforest.current_root
         self.asforest.remove_current_root()

decompiler/pipeline/controlflowanalysis/restructuring_commons/ast_processor.py

@@ -232,7 +232,7 @@ def _group_by_reaching_conditions(self, nodes: Tuple[AbstractSyntaxTreeNode]) ->
 
         :param nodes: The AST nodes that we want to group.
         :return: A dictionary that assigns to a reaching condition the list of AST code nodes with this reaching condition,
-                 if it are at least two with the same.
+                 if there are at least two with the same.
         """
         initial_groups: Dict[LogicCondition, List[AbstractSyntaxTreeNode]] = dict()
         for node in nodes:

decompiler/pipeline/controlflowanalysis/restructuring_commons/condition_aware_refinement.py

@@ -2,6 +2,8 @@
 Module for Condition Aware Refinement
 """
 
+from typing import Set
+
 from decompiler.pipeline.controlflowanalysis.restructuring_commons.condition_aware_refinement_commons.base_class_car import (
     BaseClassConditionAwareRefinement,
 )
@@ -21,6 +23,7 @@
     SwitchExtractor,
 )
 from decompiler.pipeline.controlflowanalysis.restructuring_options import RestructuringOptions
+from decompiler.structures.ast.ast_nodes import SwitchNode
 from decompiler.structures.ast.syntaxforest import AbstractSyntaxForest
 
 
@@ -35,13 +38,14 @@ class ConditionAwareRefinement(BaseClassConditionAwareRefinement):
     ]
 
     @classmethod
-    def refine(cls, asforest: AbstractSyntaxForest, options: RestructuringOptions):
+    def refine(cls, asforest: AbstractSyntaxForest, options: RestructuringOptions) -> Set[SwitchNode]:
         condition_aware_refinement = cls(asforest, options)
         for stage in condition_aware_refinement.REFINEMENT_PIPELINE:
             asforest.clean_up(asforest.current_root)
-            stage(asforest, options)
+            condition_aware_refinement.updated_switch_nodes.update(stage(asforest, options))
             condition_aware_refinement._remove_redundant_reaching_condition_from_switch_nodes()
         asforest.clean_up(asforest.current_root)
+        return set(switch for switch in condition_aware_refinement.updated_switch_nodes if switch in asforest)
 
     def _remove_redundant_reaching_condition_from_switch_nodes(self):
         """Remove the reaching condition from all switch nodes if it is redundant."""

decompiler/pipeline/controlflowanalysis/restructuring_commons/condition_aware_refinement_commons/base_class_car.py

@@ -1,10 +1,9 @@
 from dataclasses import dataclass
-from typing import Iterator, Optional, Tuple
+from typing import Iterator, Optional, Set, Tuple
 
 from decompiler.pipeline.controlflowanalysis.restructuring_options import LoopBreakOptions, RestructuringOptions
 from decompiler.structures.ast.ast_nodes import AbstractSyntaxTreeNode, CaseNode, FalseNode, SwitchNode, TrueNode
-from decompiler.structures.ast.condition_symbol import ConditionHandler
-from decompiler.structures.ast.switch_node_handler import ExpressionUsages
+from decompiler.structures.ast.condition_symbol import ConditionHandler, ExpressionUsages
 from decompiler.structures.ast.syntaxforest import AbstractSyntaxForest
 from decompiler.structures.logic.logic_condition import LogicCondition, PseudoLogicCondition
 from decompiler.structures.pseudo import Condition, Constant, Expression, OperationType
@@ -63,6 +62,7 @@ def __init__(self, asforest: AbstractSyntaxForest, options: RestructuringOptions
         self.asforest: AbstractSyntaxForest = asforest
         self.condition_handler: ConditionHandler = asforest.condition_handler
         self.options: RestructuringOptions = options
+        self.updated_switch_nodes: Set[SwitchNode] = set()
 
     def _get_constant_equality_check_expressions_and_conditions(
         self, condition: LogicCondition
@@ -109,14 +109,14 @@ def _get_expression_compared_with_constant(self, reaching_condition: LogicCondit
         Check whether the given reaching condition, which is a literal, i.e., a z3-symbol or its negation is of the form `expr == const`.
         If this is the case, then we return the expression `expr`.
         """
-        return self.asforest.switch_node_handler.get_potential_switch_expression(reaching_condition)
+        return self.asforest.condition_handler.get_potential_switch_expression_of(reaching_condition)
 
     def _get_constant_compared_with_expression(self, reaching_condition: LogicCondition) -> Optional[Constant]:
         """
         Check whether the given reaching condition, which is a literal, i.e., a z3-symbol or its negation is of the form `expr == const`.
         If this is the case, then we return the constant `const`.
         """
-        return self.asforest.switch_node_handler.get_potential_switch_constant(reaching_condition)
+        return self.asforest.condition_handler.get_potential_switch_constant_of(reaching_condition)
 
     def _convert_to_z3_condition(self, condition: LogicCondition) -> PseudoLogicCondition:
         return PseudoLogicCondition.initialize_from_formula(condition, self.condition_handler.get_z3_condition_map())

decompiler/pipeline/controlflowanalysis/restructuring_commons/condition_aware_refinement_commons/initial_switch_node_constructer.py

@@ -11,8 +11,8 @@
 )
 from decompiler.pipeline.controlflowanalysis.restructuring_options import RestructuringOptions
 from decompiler.structures.ast.ast_nodes import AbstractSyntaxTreeNode, CaseNode, CodeNode, ConditionNode, SeqNode, SwitchNode, TrueNode
+from decompiler.structures.ast.condition_symbol import ExpressionUsages
 from decompiler.structures.ast.reachability_graph import CaseDependencyGraph, LinearOrderDependency, SiblingReachability
-from decompiler.structures.ast.switch_node_handler import ExpressionUsages
 from decompiler.structures.ast.syntaxforest import AbstractSyntaxForest
 from decompiler.structures.logic.logic_condition import LogicCondition
 from decompiler.structures.pseudo import Constant, Expression
@@ -90,8 +90,8 @@ def _clean_up_reachability(self):
         """
         for candidate_1, candidate_2 in permutations(self.switch_candidate.cases, 2):
             if self.sibling_reachability.reaches(candidate_1.node, candidate_2.node) and not (
-                set(self.asforest.switch_node_handler.get_constants_for(candidate_1.condition))
-                & set(self.asforest.switch_node_handler.get_constants_for(candidate_2.condition))
+                set(self.asforest.condition_handler.get_constants_of(candidate_1.condition))
+                & set(self.asforest.condition_handler.get_constants_of(candidate_2.condition))
             ):
                 self.asforest._code_node_reachability_graph.remove_reachability_between([candidate_1.node, candidate_2.node])
                 self.sibling_reachability.remove_reachability_between([candidate_1.node, candidate_2.node])
@@ -214,13 +214,14 @@ class InitialSwitchNodeConstructor(BaseClassConditionAwareRefinement):
     """Class that constructs switch nodes."""
 
     @classmethod
-    def construct(cls, asforest: AbstractSyntaxForest, options: RestructuringOptions):
+    def construct(cls, asforest: AbstractSyntaxForest, options: RestructuringOptions) -> Set[SwitchNode]:
         """Constructs initial switch nodes if possible."""
         initial_switch_constructor = cls(asforest, options)
         for cond_node in asforest.get_condition_nodes_post_order(asforest.current_root):
             initial_switch_constructor._extract_case_nodes_from_nested_condition(cond_node)
         for seq_node in asforest.get_sequence_nodes_post_order(asforest.current_root):
             initial_switch_constructor._try_to_construct_initial_switch_node_for(seq_node)
+        return initial_switch_constructor.updated_switch_nodes
 
     def _extract_case_nodes_from_nested_condition(self, cond_node: ConditionNode) -> None:
         """
@@ -336,6 +337,7 @@ def _try_to_construct_initial_switch_node_for(self, seq_node: SeqNode) -> None:
             sibling_reachability = self.asforest.get_sibling_reachability_of_children_of(seq_node)
             switch_cases = list(possible_switch_node.construct_switch_cases())
             switch_node = self.asforest.create_switch_node_with(possible_switch_node.expression, switch_cases)
+            self.updated_switch_nodes.add(switch_node)
             case_dependency = CaseDependencyGraph.construct_case_dependency_for(self.asforest.children(switch_node), sibling_reachability)
             self._update_reaching_condition_for_case_node_children(switch_node)
             self._add_constants_to_cases(switch_node, case_dependency)
@@ -393,7 +395,7 @@ def _update_reaching_condition_for_case_node_children(self, switch_node: SwitchN
                 case_node.reaching_condition.is_disjunction_of_literals
             ), f"The condition of a case node should be a disjunction, but it is {case_node.reaching_condition}!"
 
-            if isinstance(cond_node := case_node.child, ConditionNode) and cond_node.false_branch is None:
+            if (cond_node := case_node.child).is_single_branch:
                 self._update_condition_for(cond_node, case_node)
 
             case_node.child.reaching_condition = case_node.child.reaching_condition.substitute_by_true(case_node.reaching_condition)
@@ -519,7 +521,7 @@ def _add_constants_to_cases_for(
                 case_node.constant = Constant("add_to_previous_case")
             else:
                 considered_conditions.update(
-                    (c, l) for l, c in self.asforest.switch_node_handler.get_literal_and_constant_for(case_node.reaching_condition)
+                    (c, l) for l, c in self.asforest.condition_handler.get_literal_and_constant_of(case_node.reaching_condition)
                 )
 
     def _update_reaching_condition_of(self, case_node: CaseNode, considered_conditions: Dict[Constant, LogicCondition]) -> None:
@@ -535,8 +537,7 @@ def _update_reaching_condition_of(self, case_node: CaseNode, considered_conditio
         :param considered_conditions: The conditions (literals) that are already fulfilled when we reach the given case node.
         """
         constant_of_case_node_literal = {
-            const: literal
-            for literal, const in self.asforest.switch_node_handler.get_literal_and_constant_for(case_node.reaching_condition)
+            const: literal for literal, const in self.asforest.condition_handler.get_literal_and_constant_of(case_node.reaching_condition)
         }
         exception_condition: LogicCondition = self.condition_handler.get_true_value()
 
@@ -576,7 +577,7 @@ def prepend_empty_cases_to_case_with_or_condition(self, case: CaseNode) -> List[
         the list of new case nodes.
         """
         condition_for_constant: Dict[Constant, LogicCondition] = dict()
-        for l, c in self.asforest.switch_node_handler.get_literal_and_constant_for(case.reaching_condition):
+        for l, c in self.asforest.condition_handler.get_literal_and_constant_of(case.reaching_condition):
             if c is None:
                 raise ValueError(
                     f"The case node should have a reaching-condition that is a disjunction of literals, but it has the clause {l}."

decompiler/pipeline/controlflowanalysis/restructuring_commons/condition_aware_refinement_commons/missing_case_finder.py

@@ -33,6 +33,7 @@ def _insert_case_node(self, new_case_node: AbstractSyntaxTreeNode, case_constant
         if default_case := switch_node.default:
             new_children.append(default_case)
         switch_node._sorted_cases = tuple(new_children)
+        self.updated_switch_nodes.add(switch_node)
 
     def _new_case_nodes_for(
         self, new_case_node: AbstractSyntaxTreeNode, switch_node: SwitchNode, sorted_case_constants: List[Constant]

decompiler/pipeline/controlflowanalysis/restructuring_commons/condition_aware_refinement_commons/missing_case_finder_condition.py

@@ -28,7 +28,7 @@ class MissingCaseFinderCondition(MissingCaseFinder):
     """
 
     @classmethod
-    def find(cls, asforest: AbstractSyntaxForest, options: RestructuringOptions):
+    def find(cls, asforest: AbstractSyntaxForest, options: RestructuringOptions) -> Set[SwitchNode]:
         """Try to find missing cases that are branches of condition nodes."""
         missing_case_finder = cls(asforest, options)
         for condition_node in asforest.get_condition_nodes_post_order(asforest.current_root):
@@ -37,9 +37,10 @@ def find(cls, asforest: AbstractSyntaxForest, options: RestructuringOptions):
                     case_candidate_information.case_node, case_candidate_information.case_constants, case_candidate_information.switch_node
                 )
                 if case_candidate_information.in_sequence:
-                    asforest.extract_switch_from_condition_sequence(case_candidate_information.switch_node, condition_node)
+                    asforest.extract_switch_from_sequence(case_candidate_information.switch_node)
                 else:
                     asforest.replace_condition_node_by_single_branch(condition_node)
+        return missing_case_finder.updated_switch_nodes
 
     def _can_insert_missing_case_node(self, condition_node: ConditionNode) -> Optional[CaseCandidateInformation]:
         """