From 12852b4699cb3b2db48136e19385bb05e7cd0046 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 31 Jul 2024 19:08:18 +0200 Subject: [PATCH 01/19] Make copy function of types return self --- decompiler/structures/pseudo/complextypes.py | 3 --- decompiler/structures/pseudo/typing.py | 14 +------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/decompiler/structures/pseudo/complextypes.py b/decompiler/structures/pseudo/complextypes.py index 7beb3b326..0c03d2142 100644 --- a/decompiler/structures/pseudo/complextypes.py +++ b/decompiler/structures/pseudo/complextypes.py @@ -22,9 +22,6 @@ class ComplexType(Type): def __str__(self): return self.name - def copy(self, **kwargs) -> Type: - return copy.deepcopy(self) - def declaration(self) -> str: raise NotImplementedError diff --git a/decompiler/structures/pseudo/typing.py b/decompiler/structures/pseudo/typing.py index cb2d2404e..31aca5a40 100644 --- a/decompiler/structures/pseudo/typing.py +++ b/decompiler/structures/pseudo/typing.py @@ -20,7 +20,7 @@ def is_boolean(self) -> bool: def copy(self, **kwargs) -> Type: """Generate a copy of the current type.""" - return replace(self, **kwargs) + return self def resize(self, new_size: int) -> Type: """Create an object of the type with a different size.""" @@ -173,10 +173,6 @@ def __str__(self) -> str: return f"{self.type}*" return f"{self.type} *" - def copy(self, **kwargs) -> Pointer: - """Generate a copy of the current pointer.""" - return Pointer(self.type.copy(), self.size) - @dataclass(frozen=True, order=True) class ArrayType(Type): @@ -195,10 +191,6 @@ def __str__(self) -> str: """Return a nice string representation.""" return f"{self.type} [{self.elements}]" - def copy(self, **kwargs) -> Pointer: - """Generate a copy of the current pointer.""" - return ArrayType(self.type.copy(), self.elements) - @dataclass(frozen=True, order=True) class CustomType(Type): @@ -235,10 +227,6 @@ def __str__(self) -> str: """Return the given string representation.""" return self.text - def copy(self, **kwargs) -> CustomType: - """Generate a copy of the current custom type.""" - return CustomType(self.text, self.size) - @dataclass(frozen=True, order=True) class FunctionTypeDef(Type): From 22ac914499d0034910fd3e160c0d8827b637a7f8 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Thu, 1 Aug 2024 09:38:13 +0200 Subject: [PATCH 02/19] Revert "Make copy function of types return self" This reverts commit 12852b4699cb3b2db48136e19385bb05e7cd0046. --- decompiler/structures/pseudo/complextypes.py | 3 +++ decompiler/structures/pseudo/typing.py | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/decompiler/structures/pseudo/complextypes.py b/decompiler/structures/pseudo/complextypes.py index 0c03d2142..7beb3b326 100644 --- a/decompiler/structures/pseudo/complextypes.py +++ b/decompiler/structures/pseudo/complextypes.py @@ -22,6 +22,9 @@ class ComplexType(Type): def __str__(self): return self.name + def copy(self, **kwargs) -> Type: + return copy.deepcopy(self) + def declaration(self) -> str: raise NotImplementedError diff --git a/decompiler/structures/pseudo/typing.py b/decompiler/structures/pseudo/typing.py index 31aca5a40..cb2d2404e 100644 --- a/decompiler/structures/pseudo/typing.py +++ b/decompiler/structures/pseudo/typing.py @@ -20,7 +20,7 @@ def is_boolean(self) -> bool: def copy(self, **kwargs) -> Type: """Generate a copy of the current type.""" - return self + return replace(self, **kwargs) def resize(self, new_size: int) -> Type: """Create an object of the type with a different size.""" @@ -173,6 +173,10 @@ def __str__(self) -> str: return f"{self.type}*" return f"{self.type} *" + def copy(self, **kwargs) -> Pointer: + """Generate a copy of the current pointer.""" + return Pointer(self.type.copy(), self.size) + @dataclass(frozen=True, order=True) class ArrayType(Type): @@ -191,6 +195,10 @@ def __str__(self) -> str: """Return a nice string representation.""" return f"{self.type} [{self.elements}]" + def copy(self, **kwargs) -> Pointer: + """Generate a copy of the current pointer.""" + return ArrayType(self.type.copy(), self.elements) + @dataclass(frozen=True, order=True) class CustomType(Type): @@ -227,6 +235,10 @@ def __str__(self) -> str: """Return the given string representation.""" return self.text + def copy(self, **kwargs) -> CustomType: + """Generate a copy of the current custom type.""" + return CustomType(self.text, self.size) + @dataclass(frozen=True, order=True) class FunctionTypeDef(Type): From 982ff40112e12a10414413b2c60d035cdaa6b8f9 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Thu, 1 Aug 2024 09:58:05 +0200 Subject: [PATCH 03/19] Change copy constructor of typing --- decompiler/structures/pseudo/complextypes.py | 6 +----- decompiler/structures/pseudo/typing.py | 21 ++++++-------------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/decompiler/structures/pseudo/complextypes.py b/decompiler/structures/pseudo/complextypes.py index 7beb3b326..dbe81d22a 100644 --- a/decompiler/structures/pseudo/complextypes.py +++ b/decompiler/structures/pseudo/complextypes.py @@ -1,8 +1,7 @@ -import copy import logging from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Dict, List, Optional from decompiler.structures.pseudo.typing import Type @@ -22,9 +21,6 @@ class ComplexType(Type): def __str__(self): return self.name - def copy(self, **kwargs) -> Type: - return copy.deepcopy(self) - def declaration(self) -> str: raise NotImplementedError diff --git a/decompiler/structures/pseudo/typing.py b/decompiler/structures/pseudo/typing.py index cb2d2404e..65884c618 100644 --- a/decompiler/structures/pseudo/typing.py +++ b/decompiler/structures/pseudo/typing.py @@ -4,7 +4,9 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, replace -from typing import Tuple +from typing import Tuple, TypeVar, final + +_T = TypeVar("_T", bound="Type") @dataclass(frozen=True, order=True) @@ -18,11 +20,12 @@ def is_boolean(self) -> bool: """Check whether the given value is a boolean.""" return str(self) == "bool" - def copy(self, **kwargs) -> Type: + @final + def copy(self: _T, **kwargs) -> _T: """Generate a copy of the current type.""" return replace(self, **kwargs) - def resize(self, new_size: int) -> Type: + def resize(self: _T, new_size: int) -> _T: """Create an object of the type with a different size.""" return self.copy(size=new_size) @@ -173,10 +176,6 @@ def __str__(self) -> str: return f"{self.type}*" return f"{self.type} *" - def copy(self, **kwargs) -> Pointer: - """Generate a copy of the current pointer.""" - return Pointer(self.type.copy(), self.size) - @dataclass(frozen=True, order=True) class ArrayType(Type): @@ -195,10 +194,6 @@ def __str__(self) -> str: """Return a nice string representation.""" return f"{self.type} [{self.elements}]" - def copy(self, **kwargs) -> Pointer: - """Generate a copy of the current pointer.""" - return ArrayType(self.type.copy(), self.elements) - @dataclass(frozen=True, order=True) class CustomType(Type): @@ -235,10 +230,6 @@ def __str__(self) -> str: """Return the given string representation.""" return self.text - def copy(self, **kwargs) -> CustomType: - """Generate a copy of the current custom type.""" - return CustomType(self.text, self.size) - @dataclass(frozen=True, order=True) class FunctionTypeDef(Type): From 64f3c8bbf23ba2dc42af170e4d63f9fe3e7e8882 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 7 Aug 2024 16:48:11 +0200 Subject: [PATCH 04/19] tmp --- .../frontend/binaryninja/handlers/types.py | 30 +++++++++++-------- decompiler/structures/pseudo/complextypes.py | 12 ++------ tests/structures/pseudo/test_complextypes.py | 17 +++++++---- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/decompiler/frontend/binaryninja/handlers/types.py b/decompiler/frontend/binaryninja/handlers/types.py index daf2902fd..a5651bde4 100644 --- a/decompiler/frontend/binaryninja/handlers/types.py +++ b/decompiler/frontend/binaryninja/handlers/types.py @@ -1,6 +1,6 @@ import logging from abc import abstractmethod -from typing import Optional, Union +from typing import Union from binaryninja import BinaryView, StructureVariant from binaryninja.types import ( @@ -22,7 +22,7 @@ ) from decompiler.frontend.lifter import Handler from decompiler.structures.pseudo import ArrayType as PseudoArrayType -from decompiler.structures.pseudo import CustomType, Float, FunctionTypeDef, Integer, Pointer, UnknownType, Variable +from decompiler.structures.pseudo import CustomType, Float, FunctionTypeDef, Integer, Pointer, UnknownType from decompiler.structures.pseudo.complextypes import Class, ComplexTypeMember, ComplexTypeName, Enum, Struct from decompiler.structures.pseudo.complextypes import Union as Union_ @@ -95,21 +95,25 @@ def lift_struct(self, struct: StructureType, name: str = None, **kwargs) -> Unio return cached_type """Lift struct or union type.""" - if struct.type == StructureVariant.StructStructureType: - keyword, type, members = "struct", Struct, {} - elif struct.type == StructureVariant.UnionStructureType: - keyword, type, members = "union", Union_, [] - elif struct.type == StructureVariant.ClassStructureType: - keyword, type, members = "class", Class, {} - else: - raise RuntimeError(f"Unknown struct type {struct.type.name}") + match struct.type: + case StructureVariant.StructStructureType: + keyword, type, to_members = "struct", Struct, lambda members: {m.offset: m for m in members} + case StructureVariant.UnionStructureType: + keyword, type, to_members = "union", Union_, lambda members: members + case StructureVariant.ClassStructureType: + keyword, type, to_members = "class", Class, lambda members: members + case _: + raise RuntimeError(f"Unknown struct type {struct.type.name}") type_name = self._get_data_type_name(struct, keyword=keyword, provided_name=name) - lifted_struct = type(struct.width * self.BYTE_SIZE, type_name, members) - self._lifter.complex_types.add(lifted_struct, type_id) + members: list[ComplexTypeMember] = [] for member in struct.members: - lifted_struct.add_member(self.lift_struct_member(member, type_name)) + members.append(self.lift_struct_member(member, type_name)) + + lifted_struct = type(struct.width * self.BYTE_SIZE, type_name, to_members(members)) + + self._lifter.complex_types.add(lifted_struct, type_id) return lifted_struct @abstractmethod diff --git a/decompiler/structures/pseudo/complextypes.py b/decompiler/structures/pseudo/complextypes.py index dbe81d22a..119e5d5ff 100644 --- a/decompiler/structures/pseudo/complextypes.py +++ b/decompiler/structures/pseudo/complextypes.py @@ -4,6 +4,7 @@ from typing import Dict, List, Optional from decompiler.structures.pseudo.typing import Type +from pydot import frozendict class ComplexTypeSpecifier(Enum): @@ -57,12 +58,9 @@ def declaration(self) -> str: class _BaseStruct(ComplexType): """Class representing a struct type.""" - members: Dict[int, ComplexTypeMember] = field(compare=False) + members: frozendict[int, ComplexTypeMember] = field(compare=False) type_specifier: ComplexTypeSpecifier - def add_member(self, member: ComplexTypeMember): - self.members[member.offset] = member - def get_member_by_offset(self, offset: int) -> Optional[ComplexTypeMember]: return self.members.get(offset) @@ -95,9 +93,6 @@ class Union(ComplexType): members: List[ComplexTypeMember] = field(compare=False) type_specifier = ComplexTypeSpecifier.UNION - def add_member(self, member: ComplexTypeMember): - self.members.append(member) - def declaration(self) -> str: members = ";\n\t".join(x.declaration() for x in self.members) + ";" return f"{self.type_specifier.value} {self.name} {{\n\t{members}\n}}" @@ -123,9 +118,6 @@ class Enum(ComplexType): members: Dict[int, ComplexTypeMember] = field(compare=False) type_specifier = ComplexTypeSpecifier.ENUM - def add_member(self, member: ComplexTypeMember): - self.members[member.value] = member - def get_name_by_value(self, value: int) -> Optional[str]: member = self.members.get(value) return member.name if member is not None else None diff --git a/tests/structures/pseudo/test_complextypes.py b/tests/structures/pseudo/test_complextypes.py index 7b09e82b3..3fc0b9702 100644 --- a/tests/structures/pseudo/test_complextypes.py +++ b/tests/structures/pseudo/test_complextypes.py @@ -11,6 +11,7 @@ Union, UniqueNameProvider, ) +from pydot import frozendict class TestStruct: @@ -60,11 +61,17 @@ def test_get_complex_type_name(self, book): class TestClass: - def test_declaration(self, class_book: Struct, record_id: Union): - assert class_book.declaration() == "class ClassBook {\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n}" - # nest complex type - class_book.add_member( - m := ComplexTypeMember(size=64, name="id", offset=12, type=record_id), + def test_declaration(self, record_id: Union): + m = ComplexTypeMember(size=64, name="id", offset=12, type=record_id) + class_book = Struct( + name="Book", + members=frozendict({ + 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), + 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), + 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), + 12: m + }), + size=96, ) result = f"class ClassBook {{\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n\t{m.declaration()};\n}}" assert class_book.declaration() == result From 3cb4b089167b8244c9d6c95aea760113426bb291 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 7 Aug 2024 17:27:34 +0200 Subject: [PATCH 05/19] Fix tests and stuff --- .../frontend/binaryninja/handlers/types.py | 3 +- decompiler/structures/pseudo/complextypes.py | 4 +- decompiler/util/frozen_dict.py | 27 +++++ tests/structures/pseudo/test_complextypes.py | 102 +++--------------- 4 files changed, 43 insertions(+), 93 deletions(-) create mode 100644 decompiler/util/frozen_dict.py diff --git a/decompiler/frontend/binaryninja/handlers/types.py b/decompiler/frontend/binaryninja/handlers/types.py index a5651bde4..332fcc6fc 100644 --- a/decompiler/frontend/binaryninja/handlers/types.py +++ b/decompiler/frontend/binaryninja/handlers/types.py @@ -25,6 +25,7 @@ from decompiler.structures.pseudo import CustomType, Float, FunctionTypeDef, Integer, Pointer, UnknownType from decompiler.structures.pseudo.complextypes import Class, ComplexTypeMember, ComplexTypeName, Enum, Struct from decompiler.structures.pseudo.complextypes import Union as Union_ +from decompiler.util.frozen_dict import FrozenDict class TypeHandler(Handler): @@ -97,7 +98,7 @@ def lift_struct(self, struct: StructureType, name: str = None, **kwargs) -> Unio """Lift struct or union type.""" match struct.type: case StructureVariant.StructStructureType: - keyword, type, to_members = "struct", Struct, lambda members: {m.offset: m for m in members} + keyword, type, to_members = "struct", Struct, lambda members: FrozenDict({m.offset: m for m in members}) case StructureVariant.UnionStructureType: keyword, type, to_members = "union", Union_, lambda members: members case StructureVariant.ClassStructureType: diff --git a/decompiler/structures/pseudo/complextypes.py b/decompiler/structures/pseudo/complextypes.py index 119e5d5ff..228969e83 100644 --- a/decompiler/structures/pseudo/complextypes.py +++ b/decompiler/structures/pseudo/complextypes.py @@ -4,7 +4,7 @@ from typing import Dict, List, Optional from decompiler.structures.pseudo.typing import Type -from pydot import frozendict +from decompiler.util.frozen_dict import FrozenDict class ComplexTypeSpecifier(Enum): @@ -58,7 +58,7 @@ def declaration(self) -> str: class _BaseStruct(ComplexType): """Class representing a struct type.""" - members: frozendict[int, ComplexTypeMember] = field(compare=False) + members: FrozenDict[int, ComplexTypeMember] = field(compare=False) type_specifier: ComplexTypeSpecifier def get_member_by_offset(self, offset: int) -> Optional[ComplexTypeMember]: diff --git a/decompiler/util/frozen_dict.py b/decompiler/util/frozen_dict.py new file mode 100644 index 000000000..97dcffb4a --- /dev/null +++ b/decompiler/util/frozen_dict.py @@ -0,0 +1,27 @@ +from collections.abc import Mapping + + +class FrozenDict(Mapping): + """Same as python dicts, but immutable""" + + def __init__(self, *args, **kwargs): + self._d = dict(*args, **kwargs) + self._hash = None + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + def __getitem__(self, key): + return self._d[key] + + def __hash__(self): + if self._hash is None: + hash_ = 0 + for pair in self.items(): + hash_ ^= hash(pair) + self._hash = hash_ + + return self._hash \ No newline at end of file diff --git a/tests/structures/pseudo/test_complextypes.py b/tests/structures/pseudo/test_complextypes.py index 3fc0b9702..346ed6c4b 100644 --- a/tests/structures/pseudo/test_complextypes.py +++ b/tests/structures/pseudo/test_complextypes.py @@ -5,45 +5,21 @@ ComplexTypeMap, ComplexTypeMember, ComplexTypeName, - ComplexTypeSpecifier, Enum, Struct, Union, UniqueNameProvider, ) -from pydot import frozendict +from decompiler.util.frozen_dict import FrozenDict class TestStruct: - def test_declaration(self, book: Struct, record_id: Union): + def test_declaration(self, book: Struct): assert book.declaration() == "struct Book {\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n}" - # nest complex type - book.add_member( - m := ComplexTypeMember(size=64, name="id", offset=12, type=record_id), - ) - result = f"struct Book {{\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n\t{m.declaration()};\n}}" - assert book.declaration() == result def test_str(self, book: Struct): assert str(book) == "Book" - def test_copy(self, book: Struct): - new_book: Struct = book.copy() - assert id(new_book) != id(book) - assert new_book.size == book.size - assert new_book.type_specifier == book.type_specifier == ComplexTypeSpecifier.STRUCT - assert id(new_book.members) != id(book.members) - assert new_book.get_member_by_offset(0) == book.get_member_by_offset(0) - assert id(new_book.get_member_by_offset(0)) != id(book.get_member_by_offset(0)) - assert len(new_book.members) == len(book.members) - - def test_add_members(self, book, title, num_pages, author): - empty_book = Struct(name="Book", members={}, size=96) - empty_book.add_member(title) - empty_book.add_member(author) - empty_book.add_member(num_pages) - assert empty_book == book - def test_get_member_by_offset(self, book, title, num_pages, author): assert book.get_member_by_offset(0) == title assert book.get_member_by_offset(4) == num_pages @@ -61,41 +37,12 @@ def test_get_complex_type_name(self, book): class TestClass: - def test_declaration(self, record_id: Union): - m = ComplexTypeMember(size=64, name="id", offset=12, type=record_id) - class_book = Struct( - name="Book", - members=frozendict({ - 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), - 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), - 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), - 12: m - }), - size=96, - ) - result = f"class ClassBook {{\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n\t{m.declaration()};\n}}" - assert class_book.declaration() == result + def test_declaration(self, class_book): + assert class_book.declaration() == "class ClassBook {\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n}" def test_str(self, class_book: Struct): assert str(class_book) == "ClassBook" - def test_copy(self, class_book: Struct): - new_class_book: Struct = class_book.copy() - assert id(new_class_book) != id(class_book) - assert new_class_book.size == class_book.size - assert new_class_book.type_specifier == class_book.type_specifier == ComplexTypeSpecifier.CLASS - assert id(new_class_book.members) != id(class_book.members) - assert new_class_book.get_member_by_offset(0) == class_book.get_member_by_offset(0) - assert id(new_class_book.get_member_by_offset(0)) != id(class_book.get_member_by_offset(0)) - assert len(new_class_book.members) == len(class_book.members) - - def test_add_members(self, class_book, title, num_pages, author): - empty_class_book = Class(name="ClassBook", members={}, size=96) - empty_class_book.add_member(title) - empty_class_book.add_member(author) - empty_class_book.add_member(num_pages) - assert empty_class_book == class_book - def test_get_member_by_offset(self, class_book, title, num_pages, author): assert class_book.get_member_by_offset(0) == title assert class_book.get_member_by_offset(4) == num_pages @@ -119,11 +66,11 @@ def test_class_not_struct(self, class_book, book): def book() -> Struct: return Struct( name="Book", - members={ + members=FrozenDict({ 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), - }, + }), size=96, ) @@ -132,11 +79,11 @@ def book() -> Struct: def class_book() -> Class: return Class( name="ClassBook", - members={ + members=FrozenDict({ 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), - }, + }), size=96, ) @@ -163,20 +110,6 @@ def test_declaration(self, record_id): def test_str(self, record_id): assert str(record_id) == "RecordID" - def test_copy(self, record_id): - new_record_id: Union = record_id.copy() - assert new_record_id == record_id - assert id(new_record_id) != id(record_id) - assert id(new_record_id.members) != id(record_id.members) - assert new_record_id.get_member_by_type(Float.float()) == record_id.get_member_by_type(Float.float()) - assert id(new_record_id.get_member_by_type(Float.float())) != id(record_id.get_member_by_type(Float.float())) - - def test_add_members(self, empty_record_id, record_id, float_id, int_id, double_id): - empty_record_id.add_member(float_id) - empty_record_id.add_member(int_id) - empty_record_id.add_member(double_id) - assert empty_record_id == record_id - def test_get_member_by_type(self, record_id, float_id, int_id, double_id): assert record_id.get_member_by_type(Float.float()) == float_id assert record_id.get_member_by_type(Integer.int32_t()) == int_id @@ -232,17 +165,6 @@ def test_declaration(self, color): def test_str(self, color): assert str(color) == "Color" - def test_copy(self, color): - new_color = color.copy() - assert new_color == color - assert id(new_color) != color - - def test_add_members(self, empty_color, color, red, green, blue): - empty_color.add_member(red) - empty_color.add_member(green) - empty_color.add_member(blue) - assert empty_color == color - def test_get_complex_type_name(self, color): assert color.complex_type_name == (ComplexTypeName(0, "Color")) @@ -261,22 +183,22 @@ def color(): @pytest.fixture -def empty_color(): +def empty_color() -> Enum: return Enum(0, "Color", {}) @pytest.fixture -def red(): +def red() -> ComplexTypeMember: return ComplexTypeMember(0, "red", value=0, offset=0, type=Integer.int32_t()) @pytest.fixture -def green(): +def green() -> ComplexTypeMember: return ComplexTypeMember(0, "green", value=1, offset=0, type=Integer.int32_t()) @pytest.fixture -def blue(): +def blue() -> ComplexTypeMember: return ComplexTypeMember(0, "blue", value=2, offset=0, type=Integer.int32_t()) From eca61a64931278127e40a8b4b686beba3fef7d0f Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 7 Aug 2024 17:55:52 +0200 Subject: [PATCH 06/19] Remove copy method from types --- .../frontend/binaryninja/handlers/calls.py | 4 ++-- .../binaryninja/handlers/controlflow.py | 2 +- .../frontend/binaryninja/handlers/unary.py | 2 +- .../pipeline/preprocessing/coherence.py | 2 +- .../preprocessing/missing_definitions.py | 2 +- decompiler/structures/pseudo/expressions.py | 16 ++++++------- decompiler/structures/pseudo/operations.py | 8 +++---- decompiler/structures/pseudo/typing.py | 23 ++++++++++++------- .../test_redundant_cast_elimination.py | 2 +- .../pipeline/preprocessing/test-coherence.py | 18 +++++++-------- tests/structures/pseudo/test_typing.py | 1 - 11 files changed, 43 insertions(+), 37 deletions(-) diff --git a/decompiler/frontend/binaryninja/handlers/calls.py b/decompiler/frontend/binaryninja/handlers/calls.py index a7d9ec930..935f69b1a 100644 --- a/decompiler/frontend/binaryninja/handlers/calls.py +++ b/decompiler/frontend/binaryninja/handlers/calls.py @@ -39,7 +39,7 @@ def lift_call(self, call: mediumlevelil.MediumLevelILCall, ssa: bool = False, ** Call( dest := self._lifter.lift(call.dest, parent=call), [self._lifter.lift(parameter, parent=call) for parameter in call.params], - vartype=dest.type.copy(), + vartype=dest.type, writes_memory=call.output_dest_memory if ssa else None, meta_data={"param_names": self._lift_call_parameter_names(call), "is_tailcall": isinstance(call, Tailcall)}, ), @@ -52,7 +52,7 @@ def lift_syscall(self, call: mediumlevelil.MediumLevelILSyscall, ssa: bool = Fal Call( dest := ImportedFunctionSymbol("Syscall", value=-1), [self._lifter.lift(parameter, parent=call) for parameter in call.params], - vartype=dest.type.copy(), + vartype=dest.type, writes_memory=call.output_dest_memory if ssa else None, meta_data={"param_names": self._lift_syscall_parameter_names(call)}, ), diff --git a/decompiler/frontend/binaryninja/handlers/controlflow.py b/decompiler/frontend/binaryninja/handlers/controlflow.py index 6729dcea6..463d4288a 100644 --- a/decompiler/frontend/binaryninja/handlers/controlflow.py +++ b/decompiler/frontend/binaryninja/handlers/controlflow.py @@ -25,7 +25,7 @@ def lift_branch(self, branch: mediumlevelil.MediumLevelILIf, **kwargs) -> Branch """Lift a branch instruction by lifting its condition.""" condition = self._lifter.lift(branch.condition, parent=branch) if not isinstance(condition, Condition): - condition = Condition(OperationType.not_equal, [condition, Constant(0, condition.type.copy())]) + condition = Condition(OperationType.not_equal, [condition, Constant(0, condition.type)]) return Branch(condition) def lift_branch_indirect(self, branch: mediumlevelil.MediumLevelILJumpTo, **kwargs) -> IndirectBranch: diff --git a/decompiler/frontend/binaryninja/handlers/unary.py b/decompiler/frontend/binaryninja/handlers/unary.py index 873158a4f..950adf938 100644 --- a/decompiler/frontend/binaryninja/handlers/unary.py +++ b/decompiler/frontend/binaryninja/handlers/unary.py @@ -78,7 +78,7 @@ def lift_address_of_field(self, operation: mediumlevelil.MediumLevelILAddressOfF OperationType.plus, [ UnaryOperation(OperationType.address, [operand := self._lifter.lift(operation.src, parent=operation)]), - Constant(operation.offset, vartype=operand.type.copy()), + Constant(operation.offset, vartype=operand.type), ], vartype=self._lifter.lift(operation.expr_type), ) diff --git a/decompiler/pipeline/preprocessing/coherence.py b/decompiler/pipeline/preprocessing/coherence.py index 040e0c445..a72d65773 100644 --- a/decompiler/pipeline/preprocessing/coherence.py +++ b/decompiler/pipeline/preprocessing/coherence.py @@ -74,7 +74,7 @@ def _set_variables_type(self, variables: List[Variable]) -> None: """Harmonize the variable type of the given non-empty list of variables.""" group_type = variables[0].type for variable in variables: - variable._type = group_type.copy() + variable._type = group_type def _set_variables_aliased(self, variables: List) -> None: """Set all variables in the given list as aliased.""" diff --git a/decompiler/pipeline/preprocessing/missing_definitions.py b/decompiler/pipeline/preprocessing/missing_definitions.py index 3b2b1b0a6..f0ca5806b 100644 --- a/decompiler/pipeline/preprocessing/missing_definitions.py +++ b/decompiler/pipeline/preprocessing/missing_definitions.py @@ -59,7 +59,7 @@ def _insert_label_zero_for_aliased_if_missing(self, var_name: str, variable: Var """ first_copy = self.get_smallest_label_copy(var_name) if first_copy.ssa_label > 0 and first_copy.is_aliased: - first_copy = variable.copy(vartype=first_copy.type.copy(), is_aliased=True, ssa_label=0) + first_copy = variable.copy(vartype=first_copy.type, is_aliased=True, ssa_label=0) self._sorted_copies_of[var_name].insert(0, first_copy) def get_smallest_label_copy(self, variable: Union[str, Variable]): diff --git a/decompiler/structures/pseudo/expressions.py b/decompiler/structures/pseudo/expressions.py index 3ace5ae8d..2fa312fc1 100644 --- a/decompiler/structures/pseudo/expressions.py +++ b/decompiler/structures/pseudo/expressions.py @@ -227,7 +227,7 @@ def pointee(self) -> Optional[Constant]: def copy(self) -> Constant: """Generate a Constant with the same value and type.""" - return Constant(self.value, self._type.copy(), self._pointee.copy() if self._pointee else None, self.tags) + return Constant(self.value, self._type, self._pointee.copy() if self._pointee else None, self.tags) def accept(self, visitor: DataflowObjectVisitorInterface[T]) -> T: """Invoke the appropriate visitor for this Expression.""" @@ -289,7 +289,7 @@ def __repr__(self): raise ValueError(f"Unknown symbol type {type(self.value)}") def copy(self) -> Symbol: - return Symbol(self.name, self.value, self._type.copy(), self.tags) + return Symbol(self.name, self.value, self._type, self.tags) class FunctionSymbol(Symbol): @@ -302,7 +302,7 @@ def __hash__(self): return super().__hash__() def copy(self) -> FunctionSymbol: - return FunctionSymbol(self.name, self.value, self._type.copy(), self.tags) + return FunctionSymbol(self.name, self.value, self._type, self.tags) class ImportedFunctionSymbol(FunctionSymbol): @@ -315,7 +315,7 @@ def __hash__(self): return super().__hash__() def copy(self) -> ImportedFunctionSymbol: - return ImportedFunctionSymbol(self._name, self.value, self._type.copy(), self.tags) + return ImportedFunctionSymbol(self._name, self.value, self._type, self.tags) class IntrinsicSymbol(FunctionSymbol): @@ -410,7 +410,7 @@ def copy( """Provide a copy of the current Variable.""" return self.__class__( self._name[:] if name is None else name, - self._type.copy() if vartype is None else vartype, + self._type if vartype is None else vartype, self.ssa_label if ssa_label is None else ssa_label, self.is_aliased if is_aliased is None else is_aliased, self.ssa_name if ssa_name is None else ssa_name, @@ -467,7 +467,7 @@ def copy( return self.__class__( self._name[:] if name is None else name, - self._type.copy() if vartype is None else vartype, + self._type if vartype is None else vartype, self.initial_value.copy() if initial_value is None else initial_value.copy(), self.ssa_label if ssa_label is None else ssa_label, self.is_aliased if is_aliased is None else is_aliased, @@ -552,7 +552,7 @@ def substitute(self, replacee: Variable, replacement: Variable) -> None: def copy(self) -> RegisterPair: """Return a copy of the current register pair.""" - return RegisterPair(self._high.copy(), self._low.copy(), self._type.copy(), self.tags) + return RegisterPair(self._high.copy(), self._low.copy(), self._type, self.tags) def accept(self, visitor: DataflowObjectVisitorInterface[T]) -> T: """Invoke the appropriate visitor for this Expression.""" @@ -580,7 +580,7 @@ def __str__(self) -> str: def copy(self) -> ConstantComposition: """Generate a copy of the UnknownExpression with the same message.""" - return ConstantComposition([x.copy() for x in self.value], self._type.copy()) + return ConstantComposition([x.copy() for x in self.value], self._type) def accept(self, visitor: DataflowObjectVisitorInterface[T]) -> T: """Invoke the appropriate visitor for this Expression.""" diff --git a/decompiler/structures/pseudo/operations.py b/decompiler/structures/pseudo/operations.py index 127263214..d3feeea46 100644 --- a/decompiler/structures/pseudo/operations.py +++ b/decompiler/structures/pseudo/operations.py @@ -405,7 +405,7 @@ def copy(self) -> UnaryOperation: return UnaryOperation( self._operation, [operand.copy() for operand in self._operands], - self._type.copy(), + self._type, writes_memory=self._writes_memory, contraction=self.contraction, array_info=ArrayInfo(self.array_info.base, self.array_info.index, self.array_info.confidence) if self.array_info else None, @@ -457,7 +457,7 @@ def copy(self) -> MemberAccess: self.member_offset, self.member_name, [operand.copy() for operand in self._operands], - self._type.copy(), + self._type, writes_memory=self.writes_memory, ) @@ -499,7 +499,7 @@ def right(self) -> Expression: def copy(self) -> BinaryOperation: """Generate a deep copy of the current binary operation.""" - return self.__class__(self._operation, [operand.copy() for operand in self._operands], self._type.copy(), self.tags) + return self.__class__(self._operation, [operand.copy() for operand in self._operands], self._type, self.tags) def accept(self, visitor: DataflowObjectVisitorInterface[T]) -> T: """Invoke the appropriate visitor for this Operation.""" @@ -583,7 +583,7 @@ def copy(self) -> Call: return Call( self._function, [operand.copy() for operand in self._operands], - self._type.copy(), + self._type, self._writes_memory, self._meta_data.copy() if self._meta_data is not None else None, self.tags, diff --git a/decompiler/structures/pseudo/typing.py b/decompiler/structures/pseudo/typing.py index 65884c618..f6d843154 100644 --- a/decompiler/structures/pseudo/typing.py +++ b/decompiler/structures/pseudo/typing.py @@ -20,14 +20,9 @@ def is_boolean(self) -> bool: """Check whether the given value is a boolean.""" return str(self) == "bool" - @final - def copy(self: _T, **kwargs) -> _T: - """Generate a copy of the current type.""" - return replace(self, **kwargs) - - def resize(self: _T, new_size: int) -> _T: + def resize(self, new_size: int) -> Type: """Create an object of the type with a different size.""" - return self.copy(size=new_size) + raise NotImplementedError() @abstractmethod def __str__(self) -> str: @@ -35,7 +30,7 @@ def __str__(self) -> str: def __add__(self, other) -> Type: """Add two types to generate one type of bigger size.""" - return self.copy(size=self.size + other.size) + return self.resize(self.size + other.size) def __hash__(self) -> int: """Return a hash value for the given type.""" @@ -63,6 +58,9 @@ class Integer(Type): SIZE_TYPES = {8: "char", 16: "short", 32: "int", 64: "long"} + def resize(self, new_size: int) -> Integer: + return Integer(new_size, self.signed) + @classmethod def char(cls) -> Integer: """ @@ -144,6 +142,9 @@ def __init__(self, size: int): """Create a new float type with the given size.""" super().__init__(size) + def resize(self, new_size: int) -> Float: + return Float(new_size) + @classmethod def float(cls) -> Float: """Return a float type (IEEE 754).""" @@ -170,6 +171,9 @@ def __init__(self, basetype: Type, size: int = 32): object.__setattr__(self, "type", basetype) object.__setattr__(self, "size", size) + def resize(self, new_size: int) -> Pointer: + return Pointer(self.type, new_size) + def __str__(self) -> str: """Return a nice string representation.""" if isinstance(self.type, Pointer): @@ -236,6 +240,9 @@ class FunctionTypeDef(Type): return_type: Type parameters: Tuple[Type, ...] + def resize(self, new_size: int) -> FunctionTypeDef: + return FunctionTypeDef(new_size, self.return_type, self.parameters) + def __str__(self) -> str: """Return an anonymous string representation such as void*(int, int, char*).""" return f"{self.return_type}({', '.join(str(x) for x in self.parameters)})" diff --git a/tests/pipeline/dataflowanalysis/test_redundant_cast_elimination.py b/tests/pipeline/dataflowanalysis/test_redundant_cast_elimination.py index 98fb001c0..3a4e5899d 100644 --- a/tests/pipeline/dataflowanalysis/test_redundant_cast_elimination.py +++ b/tests/pipeline/dataflowanalysis/test_redundant_cast_elimination.py @@ -131,7 +131,7 @@ def branch(operation, *args): def contract(_type: Type, var): - t = _type.copy() + t = _type _field = UnaryOperation(OperationType.cast, [var], vartype=t, contraction=True) return _field diff --git a/tests/pipeline/preprocessing/test-coherence.py b/tests/pipeline/preprocessing/test-coherence.py index a0c2b8935..6ddee17fa 100644 --- a/tests/pipeline/preprocessing/test-coherence.py +++ b/tests/pipeline/preprocessing/test-coherence.py @@ -21,14 +21,14 @@ [ ( { - "a": {0: [Variable("a", i32.copy()), Variable("a", u32.copy())], 1: [Variable("a", u32.copy())]}, + "a": {0: [Variable("a", i32), Variable("a", u32)], 1: [Variable("a", u32)]}, "b": {42: [Variable("b")]}, - "c": {0: [Variable("c", u32.copy())], 2: [Variable("c", i64.copy())]}, + "c": {0: [Variable("c", u32)], 2: [Variable("c", i64)]}, }, { - "a": {0: [Variable("a", i32.copy()), Variable("a", i32.copy())], 1: [Variable("a", u32.copy())]}, + "a": {0: [Variable("a", i32), Variable("a", i32)], 1: [Variable("a", u32)]}, "b": {42: [Variable("b")]}, - "c": {0: [Variable("c", u32.copy())], 2: [Variable("c", i64.copy())]}, + "c": {0: [Variable("c", u32)], 2: [Variable("c", i64)]}, }, ) ], @@ -72,16 +72,16 @@ def test_acceptance(): BasicBlock( 0, instructions=[ - Assignment(x01 := Variable("x", i32.copy(), ssa_label=0), Constant(0x1337, i32.copy())), + Assignment(x01 := Variable("x", i32, ssa_label=0), Constant(0x1337, i32)), Assignment( - x10 := Variable("x", i32.copy(), is_aliased=True, ssa_label=1), - Call(FunctionSymbol("foo", 0x42), [x02 := Variable("x", u32.copy(), ssa_label=0)]), + x10 := Variable("x", i32, is_aliased=True, ssa_label=1), + Call(FunctionSymbol("foo", 0x42), [x02 := Variable("x", u32, ssa_label=0)]), ), - Return([x12 := Variable("x", i32.copy(), is_aliased=False, ssa_label=1)]), + Return([x12 := Variable("x", i32, is_aliased=False, ssa_label=1)]), ], ) ] ) Coherence().run(DecompilerTask(name="test", function_identifier="", cfg=cfg)) - assert {variable.type for variable in [x01, x02]} == {i32.copy()} + assert {variable.type for variable in [x01, x02]} == {i32} assert {variable.is_aliased for variable in [x01, x02, x10, x12]} == {True} diff --git a/tests/structures/pseudo/test_typing.py b/tests/structures/pseudo/test_typing.py index 29222ff5b..9fd5db8e0 100644 --- a/tests/structures/pseudo/test_typing.py +++ b/tests/structures/pseudo/test_typing.py @@ -56,7 +56,6 @@ def test_resize(): assert Integer.int32_t().resize(64) == Integer.int64_t() assert Float.float().resize(64) == Float.double() assert Integer.uint8_t() + Integer.int16_t() == Integer(24, signed=False) - assert CustomType.void() + CustomType.void() == CustomType.void() assert CustomType.bool().size + Float.float().size == CustomType("bool", 32 + SIZEOF_BOOL).size From b52987118b84eaea449633f59daa83667933b7cc Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 7 Aug 2024 18:05:21 +0200 Subject: [PATCH 07/19] Remove unused __add__ function from types --- decompiler/structures/pseudo/typing.py | 4 ---- tests/structures/pseudo/test_typing.py | 1 - 2 files changed, 5 deletions(-) diff --git a/decompiler/structures/pseudo/typing.py b/decompiler/structures/pseudo/typing.py index f6d843154..0477550f5 100644 --- a/decompiler/structures/pseudo/typing.py +++ b/decompiler/structures/pseudo/typing.py @@ -28,10 +28,6 @@ def resize(self, new_size: int) -> Type: def __str__(self) -> str: """Every type should provide a c-like string representation.""" - def __add__(self, other) -> Type: - """Add two types to generate one type of bigger size.""" - return self.resize(self.size + other.size) - def __hash__(self) -> int: """Return a hash value for the given type.""" return hash(repr(self)) diff --git a/tests/structures/pseudo/test_typing.py b/tests/structures/pseudo/test_typing.py index 9fd5db8e0..315651413 100644 --- a/tests/structures/pseudo/test_typing.py +++ b/tests/structures/pseudo/test_typing.py @@ -55,7 +55,6 @@ def test_resize(): """Test the resize system generating new types.""" assert Integer.int32_t().resize(64) == Integer.int64_t() assert Float.float().resize(64) == Float.double() - assert Integer.uint8_t() + Integer.int16_t() == Integer(24, signed=False) assert CustomType.bool().size + Float.float().size == CustomType("bool", 32 + SIZEOF_BOOL).size From 2414f066ea840e3bc3a2ab9b6815a2cc13af258d Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 7 Aug 2024 18:12:36 +0200 Subject: [PATCH 08/19] Remove custom hash implementation from types --- decompiler/structures/pseudo/typing.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/decompiler/structures/pseudo/typing.py b/decompiler/structures/pseudo/typing.py index 0477550f5..2a43f6c2d 100644 --- a/decompiler/structures/pseudo/typing.py +++ b/decompiler/structures/pseudo/typing.py @@ -28,10 +28,6 @@ def resize(self, new_size: int) -> Type: def __str__(self) -> str: """Every type should provide a c-like string representation.""" - def __hash__(self) -> int: - """Return a hash value for the given type.""" - return hash(repr(self)) - @dataclass(frozen=True, order=True) class UnknownType(Type): From 1a0f8e5271f68471ef3e0c68e38c0eaa4d6786ad Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 7 Aug 2024 23:13:14 +0200 Subject: [PATCH 09/19] tmp --- decompiler/structures/pseudo/complextypes.py | 2 +- decompiler/structures/pseudo/typing.py | 38 ++++++-------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/decompiler/structures/pseudo/complextypes.py b/decompiler/structures/pseudo/complextypes.py index 228969e83..8b5f6f709 100644 --- a/decompiler/structures/pseudo/complextypes.py +++ b/decompiler/structures/pseudo/complextypes.py @@ -14,7 +14,7 @@ class ComplexTypeSpecifier(Enum): CLASS = "class" -@dataclass(frozen=True, order=True) +@dataclass(frozen=True, order=True, slots=True) class ComplexType(Type): size = 0 name: str diff --git a/decompiler/structures/pseudo/typing.py b/decompiler/structures/pseudo/typing.py index 2a43f6c2d..aa6699adb 100644 --- a/decompiler/structures/pseudo/typing.py +++ b/decompiler/structures/pseudo/typing.py @@ -9,7 +9,7 @@ _T = TypeVar("_T", bound="Type") -@dataclass(frozen=True, order=True) +@dataclass(frozen=True, order=True, slots=False) class Type(ABC): """Base interface for all type classes.""" @@ -20,29 +20,29 @@ def is_boolean(self) -> bool: """Check whether the given value is a boolean.""" return str(self) == "bool" - def resize(self, new_size: int) -> Type: + def resize(self: _T, new_size: int) -> _T: """Create an object of the type with a different size.""" - raise NotImplementedError() + return replace(self, size=new_size) @abstractmethod def __str__(self) -> str: """Every type should provide a c-like string representation.""" -@dataclass(frozen=True, order=True) +@dataclass(frozen=True, order=True, slots=False) class UnknownType(Type): """Represent an unknown type, mostly utilized for testing purposes.""" def __init__(self, size: int = 0): """Create a type with size 0.""" - super().__init__(size) + Type.__init__(self, size=size) def __str__(self): """Return the representation of the unknown type.""" return "unknown type" -@dataclass(frozen=True, order=True) +@dataclass(frozen=True, order=True, slots=False) class Integer(Type): """Type for values representing numbers.""" @@ -50,9 +50,6 @@ class Integer(Type): SIZE_TYPES = {8: "char", 16: "short", 32: "int", 64: "long"} - def resize(self, new_size: int) -> Integer: - return Integer(new_size, self.signed) - @classmethod def char(cls) -> Integer: """ @@ -124,19 +121,12 @@ def __str__(self): return f"{'u' if not self.is_signed else ''}int{self.size}_t" -@dataclass(frozen=True, order=True) +@dataclass(frozen=True, order=True, slots=False) class Float(Type): """Class representing the type of a floating point number as defined in IEEE 754.""" SIZE_TYPES = {8: "quarter", 16: "half", 32: "float", 64: "double", 80: "long double", 128: "quadruple", 256: "octuple"} - def __init__(self, size: int): - """Create a new float type with the given size.""" - super().__init__(size) - - def resize(self, new_size: int) -> Float: - return Float(new_size) - @classmethod def float(cls) -> Float: """Return a float type (IEEE 754).""" @@ -152,7 +142,7 @@ def __str__(self) -> str: return self.SIZE_TYPES[self.size] -@dataclass(frozen=True, order=True) +@dataclass(frozen=True, order=True, slots=False) class Pointer(Type): """Class representing types based on being pointers on other types.""" @@ -163,9 +153,6 @@ def __init__(self, basetype: Type, size: int = 32): object.__setattr__(self, "type", basetype) object.__setattr__(self, "size", size) - def resize(self, new_size: int) -> Pointer: - return Pointer(self.type, new_size) - def __str__(self) -> str: """Return a nice string representation.""" if isinstance(self.type, Pointer): @@ -173,7 +160,7 @@ def __str__(self) -> str: return f"{self.type} *" -@dataclass(frozen=True, order=True) +@dataclass(frozen=True, order=True, slots=False) class ArrayType(Type): """Class representing arrays.""" @@ -191,7 +178,7 @@ def __str__(self) -> str: return f"{self.type} [{self.elements}]" -@dataclass(frozen=True, order=True) +@dataclass(frozen=True, order=True, slots=False) class CustomType(Type): """Class representing a non-basic type.""" @@ -227,14 +214,11 @@ def __str__(self) -> str: return self.text -@dataclass(frozen=True, order=True) +@dataclass(frozen=True, order=True, slots=False) class FunctionTypeDef(Type): return_type: Type parameters: Tuple[Type, ...] - def resize(self, new_size: int) -> FunctionTypeDef: - return FunctionTypeDef(new_size, self.return_type, self.parameters) - def __str__(self) -> str: """Return an anonymous string representation such as void*(int, int, char*).""" return f"{self.return_type}({', '.join(str(x) for x in self.parameters)})" From 86a4a77dfb110d7403e422c98bc8d264cf1c04d0 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Thu, 8 Aug 2024 09:34:46 +0200 Subject: [PATCH 10/19] Fix resize of ArrayType --- decompiler/structures/pseudo/typing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/decompiler/structures/pseudo/typing.py b/decompiler/structures/pseudo/typing.py index aa6699adb..e412b21d3 100644 --- a/decompiler/structures/pseudo/typing.py +++ b/decompiler/structures/pseudo/typing.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, replace -from typing import Tuple, TypeVar, final +from typing import Tuple, TypeVar _T = TypeVar("_T", bound="Type") @@ -173,6 +173,10 @@ def __init__(self, basetype: Type, elements: int): object.__setattr__(self, "size", basetype.size * elements) object.__setattr__(self, "elements", elements) + def resize(self, new_size: int) -> ArrayType: + # Overridden because of backwards compatibility... ArrayType was not able to be resized. + return self + def __str__(self) -> str: """Return a nice string representation.""" return f"{self.type} [{self.elements}]" From 0ca60587ba81c3bd92d9c2f15256e361f5d502a7 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:32:12 +0200 Subject: [PATCH 11/19] Fix some stuff --- decompiler/structures/pseudo/typing.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/decompiler/structures/pseudo/typing.py b/decompiler/structures/pseudo/typing.py index e412b21d3..c1fa32676 100644 --- a/decompiler/structures/pseudo/typing.py +++ b/decompiler/structures/pseudo/typing.py @@ -9,7 +9,7 @@ _T = TypeVar("_T", bound="Type") -@dataclass(frozen=True, order=True, slots=False) +@dataclass(frozen=True, order=True) class Type(ABC): """Base interface for all type classes.""" @@ -29,20 +29,20 @@ def __str__(self) -> str: """Every type should provide a c-like string representation.""" -@dataclass(frozen=True, order=True, slots=False) +@dataclass(frozen=True, order=True) class UnknownType(Type): """Represent an unknown type, mostly utilized for testing purposes.""" def __init__(self, size: int = 0): """Create a type with size 0.""" - Type.__init__(self, size=size) + object.__setattr__(self, "size", size) def __str__(self): """Return the representation of the unknown type.""" return "unknown type" -@dataclass(frozen=True, order=True, slots=False) +@dataclass(frozen=True, order=True) class Integer(Type): """Type for values representing numbers.""" @@ -121,7 +121,7 @@ def __str__(self): return f"{'u' if not self.is_signed else ''}int{self.size}_t" -@dataclass(frozen=True, order=True, slots=False) +@dataclass(frozen=True, order=True) class Float(Type): """Class representing the type of a floating point number as defined in IEEE 754.""" @@ -142,7 +142,7 @@ def __str__(self) -> str: return self.SIZE_TYPES[self.size] -@dataclass(frozen=True, order=True, slots=False) +@dataclass(frozen=True, order=True) class Pointer(Type): """Class representing types based on being pointers on other types.""" @@ -160,7 +160,7 @@ def __str__(self) -> str: return f"{self.type} *" -@dataclass(frozen=True, order=True, slots=False) +@dataclass(frozen=True, order=True) class ArrayType(Type): """Class representing arrays.""" @@ -182,7 +182,7 @@ def __str__(self) -> str: return f"{self.type} [{self.elements}]" -@dataclass(frozen=True, order=True, slots=False) +@dataclass(frozen=True, order=True) class CustomType(Type): """Class representing a non-basic type.""" @@ -218,7 +218,7 @@ def __str__(self) -> str: return self.text -@dataclass(frozen=True, order=True, slots=False) +@dataclass(frozen=True, order=True) class FunctionTypeDef(Type): return_type: Type parameters: Tuple[Type, ...] From df82b62ddfe22c1393bf8a94476c2657785910a2 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:40:50 +0200 Subject: [PATCH 12/19] black --- decompiler/util/frozen_dict.py | 2 +- tests/structures/pseudo/test_complextypes.py | 24 ++++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/decompiler/util/frozen_dict.py b/decompiler/util/frozen_dict.py index 97dcffb4a..64ec0d9ba 100644 --- a/decompiler/util/frozen_dict.py +++ b/decompiler/util/frozen_dict.py @@ -24,4 +24,4 @@ def __hash__(self): hash_ ^= hash(pair) self._hash = hash_ - return self._hash \ No newline at end of file + return self._hash diff --git a/tests/structures/pseudo/test_complextypes.py b/tests/structures/pseudo/test_complextypes.py index 346ed6c4b..c9eaa8471 100644 --- a/tests/structures/pseudo/test_complextypes.py +++ b/tests/structures/pseudo/test_complextypes.py @@ -66,11 +66,13 @@ def test_class_not_struct(self, class_book, book): def book() -> Struct: return Struct( name="Book", - members=FrozenDict({ - 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), - 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), - 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), - }), + members=FrozenDict( + { + 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), + 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), + 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), + } + ), size=96, ) @@ -79,11 +81,13 @@ def book() -> Struct: def class_book() -> Class: return Class( name="ClassBook", - members=FrozenDict({ - 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), - 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), - 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), - }), + members=FrozenDict( + { + 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), + 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), + 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), + } + ), size=96, ) From 51dee9165407bdf0ad571d189692589af8b70021 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 14 Aug 2024 16:43:37 +0200 Subject: [PATCH 13/19] Fix pointer resize method --- decompiler/structures/pseudo/typing.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/decompiler/structures/pseudo/typing.py b/decompiler/structures/pseudo/typing.py index c1fa32676..01eaf7fe3 100644 --- a/decompiler/structures/pseudo/typing.py +++ b/decompiler/structures/pseudo/typing.py @@ -153,6 +153,11 @@ def __init__(self, basetype: Type, size: int = 32): object.__setattr__(self, "type", basetype) object.__setattr__(self, "size", size) + def resize(self, new_size: int) -> Pointer: + # Needs custom implementation, because construction parameter 'basetype' differs in name to field 'type'. + # This causes dataclasses.replace to not work + return Pointer(self.type, new_size) + def __str__(self) -> str: """Return a nice string representation.""" if isinstance(self.type, Pointer): From 7770c33c48704f9ad922524084d914a9d04aa74c Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:11:05 +0200 Subject: [PATCH 14/19] Add __str__, __repr__ to FrozenDict --- decompiler/util/frozen_dict.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/decompiler/util/frozen_dict.py b/decompiler/util/frozen_dict.py index 64ec0d9ba..9c79a35b8 100644 --- a/decompiler/util/frozen_dict.py +++ b/decompiler/util/frozen_dict.py @@ -25,3 +25,9 @@ def __hash__(self): self._hash = hash_ return self._hash + + def __str__(self): + return str(self._d) + + def __repr__(self): + return repr(self._d) From 7b3074fc2f5e64c135496a9cb1d3ad70f1c5de19 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:46:15 +0200 Subject: [PATCH 15/19] Revert immutability of complex types Because complex types can be self-referential, by design, they cannot be immutable. --- decompiler/frontend/binaryninja/frontend.py | 2 + .../frontend/binaryninja/handlers/types.py | 3 +- decompiler/structures/pseudo/complextypes.py | 4 +- decompiler/util/frozen_dict.py | 33 ----- tests/structures/pseudo/test_complextypes.py | 118 ++++++++++++++---- 5 files changed, 101 insertions(+), 59 deletions(-) delete mode 100644 decompiler/util/frozen_dict.py diff --git a/decompiler/frontend/binaryninja/frontend.py b/decompiler/frontend/binaryninja/frontend.py index dd0b9ebe5..90aa0a002 100644 --- a/decompiler/frontend/binaryninja/frontend.py +++ b/decompiler/frontend/binaryninja/frontend.py @@ -73,6 +73,8 @@ def lift(self, task: DecompilerTask): tagging = CompilerIdiomsTagging(self._bv, function.start, task.options) tagging.run() + instructions = list(function.instructions) + task.cfg = parser.parse(function) task.complex_types = parser.complex_types except Exception as e: diff --git a/decompiler/frontend/binaryninja/handlers/types.py b/decompiler/frontend/binaryninja/handlers/types.py index cdcefb089..5b0305b10 100644 --- a/decompiler/frontend/binaryninja/handlers/types.py +++ b/decompiler/frontend/binaryninja/handlers/types.py @@ -25,7 +25,6 @@ from decompiler.structures.pseudo import CustomType, Float, FunctionTypeDef, Integer, Pointer, UnknownType from decompiler.structures.pseudo.complextypes import Class, ComplexTypeMember, ComplexTypeName, Enum, Struct from decompiler.structures.pseudo.complextypes import Union as Union_ -from decompiler.util.frozen_dict import FrozenDict class TypeHandler(Handler): @@ -104,7 +103,7 @@ def lift_struct(self, struct: StructureType, name: str = None, **kwargs) -> Unio """Lift struct or union type.""" match struct.type: case StructureVariant.StructStructureType: - keyword, type, to_members = "struct", Struct, lambda members: FrozenDict({m.offset: m for m in members}) + keyword, type, to_members = "struct", Struct, lambda members: {m.offset: m for m in members} case StructureVariant.UnionStructureType: keyword, type, to_members = "union", Union_, lambda members: members case StructureVariant.ClassStructureType: diff --git a/decompiler/structures/pseudo/complextypes.py b/decompiler/structures/pseudo/complextypes.py index 8b5f6f709..61e863530 100644 --- a/decompiler/structures/pseudo/complextypes.py +++ b/decompiler/structures/pseudo/complextypes.py @@ -4,7 +4,7 @@ from typing import Dict, List, Optional from decompiler.structures.pseudo.typing import Type -from decompiler.util.frozen_dict import FrozenDict +from pydot import frozendict class ComplexTypeSpecifier(Enum): @@ -58,7 +58,7 @@ def declaration(self) -> str: class _BaseStruct(ComplexType): """Class representing a struct type.""" - members: FrozenDict[int, ComplexTypeMember] = field(compare=False) + members: frozendict[int, ComplexTypeMember] = field(compare=False) type_specifier: ComplexTypeSpecifier def get_member_by_offset(self, offset: int) -> Optional[ComplexTypeMember]: diff --git a/decompiler/util/frozen_dict.py b/decompiler/util/frozen_dict.py deleted file mode 100644 index 9c79a35b8..000000000 --- a/decompiler/util/frozen_dict.py +++ /dev/null @@ -1,33 +0,0 @@ -from collections.abc import Mapping - - -class FrozenDict(Mapping): - """Same as python dicts, but immutable""" - - def __init__(self, *args, **kwargs): - self._d = dict(*args, **kwargs) - self._hash = None - - def __iter__(self): - return iter(self._d) - - def __len__(self): - return len(self._d) - - def __getitem__(self, key): - return self._d[key] - - def __hash__(self): - if self._hash is None: - hash_ = 0 - for pair in self.items(): - hash_ ^= hash(pair) - self._hash = hash_ - - return self._hash - - def __str__(self): - return str(self._d) - - def __repr__(self): - return repr(self._d) diff --git a/tests/structures/pseudo/test_complextypes.py b/tests/structures/pseudo/test_complextypes.py index c9eaa8471..3fc0b9702 100644 --- a/tests/structures/pseudo/test_complextypes.py +++ b/tests/structures/pseudo/test_complextypes.py @@ -5,21 +5,45 @@ ComplexTypeMap, ComplexTypeMember, ComplexTypeName, + ComplexTypeSpecifier, Enum, Struct, Union, UniqueNameProvider, ) -from decompiler.util.frozen_dict import FrozenDict +from pydot import frozendict class TestStruct: - def test_declaration(self, book: Struct): + def test_declaration(self, book: Struct, record_id: Union): assert book.declaration() == "struct Book {\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n}" + # nest complex type + book.add_member( + m := ComplexTypeMember(size=64, name="id", offset=12, type=record_id), + ) + result = f"struct Book {{\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n\t{m.declaration()};\n}}" + assert book.declaration() == result def test_str(self, book: Struct): assert str(book) == "Book" + def test_copy(self, book: Struct): + new_book: Struct = book.copy() + assert id(new_book) != id(book) + assert new_book.size == book.size + assert new_book.type_specifier == book.type_specifier == ComplexTypeSpecifier.STRUCT + assert id(new_book.members) != id(book.members) + assert new_book.get_member_by_offset(0) == book.get_member_by_offset(0) + assert id(new_book.get_member_by_offset(0)) != id(book.get_member_by_offset(0)) + assert len(new_book.members) == len(book.members) + + def test_add_members(self, book, title, num_pages, author): + empty_book = Struct(name="Book", members={}, size=96) + empty_book.add_member(title) + empty_book.add_member(author) + empty_book.add_member(num_pages) + assert empty_book == book + def test_get_member_by_offset(self, book, title, num_pages, author): assert book.get_member_by_offset(0) == title assert book.get_member_by_offset(4) == num_pages @@ -37,12 +61,41 @@ def test_get_complex_type_name(self, book): class TestClass: - def test_declaration(self, class_book): - assert class_book.declaration() == "class ClassBook {\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n}" + def test_declaration(self, record_id: Union): + m = ComplexTypeMember(size=64, name="id", offset=12, type=record_id) + class_book = Struct( + name="Book", + members=frozendict({ + 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), + 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), + 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), + 12: m + }), + size=96, + ) + result = f"class ClassBook {{\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n\t{m.declaration()};\n}}" + assert class_book.declaration() == result def test_str(self, class_book: Struct): assert str(class_book) == "ClassBook" + def test_copy(self, class_book: Struct): + new_class_book: Struct = class_book.copy() + assert id(new_class_book) != id(class_book) + assert new_class_book.size == class_book.size + assert new_class_book.type_specifier == class_book.type_specifier == ComplexTypeSpecifier.CLASS + assert id(new_class_book.members) != id(class_book.members) + assert new_class_book.get_member_by_offset(0) == class_book.get_member_by_offset(0) + assert id(new_class_book.get_member_by_offset(0)) != id(class_book.get_member_by_offset(0)) + assert len(new_class_book.members) == len(class_book.members) + + def test_add_members(self, class_book, title, num_pages, author): + empty_class_book = Class(name="ClassBook", members={}, size=96) + empty_class_book.add_member(title) + empty_class_book.add_member(author) + empty_class_book.add_member(num_pages) + assert empty_class_book == class_book + def test_get_member_by_offset(self, class_book, title, num_pages, author): assert class_book.get_member_by_offset(0) == title assert class_book.get_member_by_offset(4) == num_pages @@ -66,13 +119,11 @@ def test_class_not_struct(self, class_book, book): def book() -> Struct: return Struct( name="Book", - members=FrozenDict( - { - 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), - 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), - 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), - } - ), + members={ + 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), + 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), + 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), + }, size=96, ) @@ -81,13 +132,11 @@ def book() -> Struct: def class_book() -> Class: return Class( name="ClassBook", - members=FrozenDict( - { - 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), - 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), - 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), - } - ), + members={ + 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), + 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), + 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), + }, size=96, ) @@ -114,6 +163,20 @@ def test_declaration(self, record_id): def test_str(self, record_id): assert str(record_id) == "RecordID" + def test_copy(self, record_id): + new_record_id: Union = record_id.copy() + assert new_record_id == record_id + assert id(new_record_id) != id(record_id) + assert id(new_record_id.members) != id(record_id.members) + assert new_record_id.get_member_by_type(Float.float()) == record_id.get_member_by_type(Float.float()) + assert id(new_record_id.get_member_by_type(Float.float())) != id(record_id.get_member_by_type(Float.float())) + + def test_add_members(self, empty_record_id, record_id, float_id, int_id, double_id): + empty_record_id.add_member(float_id) + empty_record_id.add_member(int_id) + empty_record_id.add_member(double_id) + assert empty_record_id == record_id + def test_get_member_by_type(self, record_id, float_id, int_id, double_id): assert record_id.get_member_by_type(Float.float()) == float_id assert record_id.get_member_by_type(Integer.int32_t()) == int_id @@ -169,6 +232,17 @@ def test_declaration(self, color): def test_str(self, color): assert str(color) == "Color" + def test_copy(self, color): + new_color = color.copy() + assert new_color == color + assert id(new_color) != color + + def test_add_members(self, empty_color, color, red, green, blue): + empty_color.add_member(red) + empty_color.add_member(green) + empty_color.add_member(blue) + assert empty_color == color + def test_get_complex_type_name(self, color): assert color.complex_type_name == (ComplexTypeName(0, "Color")) @@ -187,22 +261,22 @@ def color(): @pytest.fixture -def empty_color() -> Enum: +def empty_color(): return Enum(0, "Color", {}) @pytest.fixture -def red() -> ComplexTypeMember: +def red(): return ComplexTypeMember(0, "red", value=0, offset=0, type=Integer.int32_t()) @pytest.fixture -def green() -> ComplexTypeMember: +def green(): return ComplexTypeMember(0, "green", value=1, offset=0, type=Integer.int32_t()) @pytest.fixture -def blue() -> ComplexTypeMember: +def blue(): return ComplexTypeMember(0, "blue", value=2, offset=0, type=Integer.int32_t()) From 8ce78d30e29d7f896ba351e4987155e35ada8c7d Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:46:16 +0200 Subject: [PATCH 16/19] Revert "tmp" This reverts commit 64f3c8bbf23ba2dc42af170e4d63f9fe3e7e8882. --- .../frontend/binaryninja/handlers/types.py | 30 ++++++++----------- decompiler/structures/pseudo/complextypes.py | 12 ++++++-- tests/structures/pseudo/test_complextypes.py | 17 ++++------- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/decompiler/frontend/binaryninja/handlers/types.py b/decompiler/frontend/binaryninja/handlers/types.py index 5b0305b10..929162a75 100644 --- a/decompiler/frontend/binaryninja/handlers/types.py +++ b/decompiler/frontend/binaryninja/handlers/types.py @@ -1,6 +1,6 @@ import logging from abc import abstractmethod -from typing import Union +from typing import Optional, Union from binaryninja import BinaryView, StructureVariant from binaryninja.types import ( @@ -22,7 +22,7 @@ ) from decompiler.frontend.lifter import Handler from decompiler.structures.pseudo import ArrayType as PseudoArrayType -from decompiler.structures.pseudo import CustomType, Float, FunctionTypeDef, Integer, Pointer, UnknownType +from decompiler.structures.pseudo import CustomType, Float, FunctionTypeDef, Integer, Pointer, UnknownType, Variable from decompiler.structures.pseudo.complextypes import Class, ComplexTypeMember, ComplexTypeName, Enum, Struct from decompiler.structures.pseudo.complextypes import Union as Union_ @@ -101,25 +101,21 @@ def lift_struct(self, struct: StructureType, name: str = None, **kwargs) -> Unio return cached_type """Lift struct or union type.""" - match struct.type: - case StructureVariant.StructStructureType: - keyword, type, to_members = "struct", Struct, lambda members: {m.offset: m for m in members} - case StructureVariant.UnionStructureType: - keyword, type, to_members = "union", Union_, lambda members: members - case StructureVariant.ClassStructureType: - keyword, type, to_members = "class", Class, lambda members: members - case _: - raise RuntimeError(f"Unknown struct type {struct.type.name}") + if struct.type == StructureVariant.StructStructureType: + keyword, type, members = "struct", Struct, {} + elif struct.type == StructureVariant.UnionStructureType: + keyword, type, members = "union", Union_, [] + elif struct.type == StructureVariant.ClassStructureType: + keyword, type, members = "class", Class, {} + else: + raise RuntimeError(f"Unknown struct type {struct.type.name}") type_name = self._get_data_type_name(struct, keyword=keyword, provided_name=name) - - members: list[ComplexTypeMember] = [] - for member in struct.members: - members.append(self.lift_struct_member(member, type_name)) - - lifted_struct = type(struct.width * self.BYTE_SIZE, type_name, to_members(members)) + lifted_struct = type(struct.width * self.BYTE_SIZE, type_name, members) self._lifter.complex_types.add(lifted_struct, type_id) + for member in struct.members: + lifted_struct.add_member(self.lift_struct_member(member, type_name)) return lifted_struct @abstractmethod diff --git a/decompiler/structures/pseudo/complextypes.py b/decompiler/structures/pseudo/complextypes.py index 61e863530..3c304053e 100644 --- a/decompiler/structures/pseudo/complextypes.py +++ b/decompiler/structures/pseudo/complextypes.py @@ -4,7 +4,6 @@ from typing import Dict, List, Optional from decompiler.structures.pseudo.typing import Type -from pydot import frozendict class ComplexTypeSpecifier(Enum): @@ -58,9 +57,12 @@ def declaration(self) -> str: class _BaseStruct(ComplexType): """Class representing a struct type.""" - members: frozendict[int, ComplexTypeMember] = field(compare=False) + members: Dict[int, ComplexTypeMember] = field(compare=False) type_specifier: ComplexTypeSpecifier + def add_member(self, member: ComplexTypeMember): + self.members[member.offset] = member + def get_member_by_offset(self, offset: int) -> Optional[ComplexTypeMember]: return self.members.get(offset) @@ -93,6 +95,9 @@ class Union(ComplexType): members: List[ComplexTypeMember] = field(compare=False) type_specifier = ComplexTypeSpecifier.UNION + def add_member(self, member: ComplexTypeMember): + self.members.append(member) + def declaration(self) -> str: members = ";\n\t".join(x.declaration() for x in self.members) + ";" return f"{self.type_specifier.value} {self.name} {{\n\t{members}\n}}" @@ -118,6 +123,9 @@ class Enum(ComplexType): members: Dict[int, ComplexTypeMember] = field(compare=False) type_specifier = ComplexTypeSpecifier.ENUM + def add_member(self, member: ComplexTypeMember): + self.members[member.value] = member + def get_name_by_value(self, value: int) -> Optional[str]: member = self.members.get(value) return member.name if member is not None else None diff --git a/tests/structures/pseudo/test_complextypes.py b/tests/structures/pseudo/test_complextypes.py index 3fc0b9702..7b09e82b3 100644 --- a/tests/structures/pseudo/test_complextypes.py +++ b/tests/structures/pseudo/test_complextypes.py @@ -11,7 +11,6 @@ Union, UniqueNameProvider, ) -from pydot import frozendict class TestStruct: @@ -61,17 +60,11 @@ def test_get_complex_type_name(self, book): class TestClass: - def test_declaration(self, record_id: Union): - m = ComplexTypeMember(size=64, name="id", offset=12, type=record_id) - class_book = Struct( - name="Book", - members=frozendict({ - 0: ComplexTypeMember(size=32, name="title", offset=0, type=Pointer(Integer.char())), - 4: ComplexTypeMember(size=32, name="num_pages", offset=4, type=Integer.int32_t()), - 8: ComplexTypeMember(size=32, name="author", offset=8, type=Pointer(Integer.char())), - 12: m - }), - size=96, + def test_declaration(self, class_book: Struct, record_id: Union): + assert class_book.declaration() == "class ClassBook {\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n}" + # nest complex type + class_book.add_member( + m := ComplexTypeMember(size=64, name="id", offset=12, type=record_id), ) result = f"class ClassBook {{\n\tchar * title;\n\tint num_pages;\n\tchar * author;\n\t{m.declaration()};\n}}" assert class_book.declaration() == result From 8676ccc533ed418ea7f7da593daec8d3a4dc9950 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:55:40 +0200 Subject: [PATCH 17/19] Fix hash for complex types --- decompiler/structures/pseudo/complextypes.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/decompiler/structures/pseudo/complextypes.py b/decompiler/structures/pseudo/complextypes.py index 3c304053e..90d2e6dd9 100644 --- a/decompiler/structures/pseudo/complextypes.py +++ b/decompiler/structures/pseudo/complextypes.py @@ -79,6 +79,10 @@ def declaration(self) -> str: members = ";\n\t".join(self.members[k].declaration() for k in sorted(self.members.keys())) + ";" return f"{self.type_specifier.value} {self.name} {{\n\t{members}\n}}" + def __hash__(self) -> int: + # Because dict is not hashable, we need our own hash implementation + return hash(repr(self)) + @dataclass(frozen=True, order=True) class Struct(_BaseStruct): @@ -117,6 +121,10 @@ def get_member_name_by_type(self, _type: Type) -> str: logging.warning(f"Cannot get member name for union {self}") return "unknown_field" + def __hash__(self) -> int: + # Because list is not hashable, we need our own hash implementation + return hash(repr(self)) + @dataclass(frozen=True, order=True) class Enum(ComplexType): @@ -134,6 +142,10 @@ def declaration(self) -> str: members = ",\n\t".join(f"{x.name} = {x.value}" for x in self.members.values()) return f"{self.type_specifier.value} {self.name} {{\n\t{members}\n}}" + def __hash__(self) -> int: + # Because dict is not hashable, we need our own hash implementation + return hash(repr(self)) + @dataclass(frozen=True, order=True) class ComplexTypeName(Type): From 9f953a8a1bd8f149b6ff1283db78c888ec62fd51 Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:02:42 +0200 Subject: [PATCH 18/19] Fix tests --- tests/structures/pseudo/test_complextypes.py | 33 -------------------- 1 file changed, 33 deletions(-) diff --git a/tests/structures/pseudo/test_complextypes.py b/tests/structures/pseudo/test_complextypes.py index 7b09e82b3..ad0f2ab17 100644 --- a/tests/structures/pseudo/test_complextypes.py +++ b/tests/structures/pseudo/test_complextypes.py @@ -26,16 +26,6 @@ def test_declaration(self, book: Struct, record_id: Union): def test_str(self, book: Struct): assert str(book) == "Book" - def test_copy(self, book: Struct): - new_book: Struct = book.copy() - assert id(new_book) != id(book) - assert new_book.size == book.size - assert new_book.type_specifier == book.type_specifier == ComplexTypeSpecifier.STRUCT - assert id(new_book.members) != id(book.members) - assert new_book.get_member_by_offset(0) == book.get_member_by_offset(0) - assert id(new_book.get_member_by_offset(0)) != id(book.get_member_by_offset(0)) - assert len(new_book.members) == len(book.members) - def test_add_members(self, book, title, num_pages, author): empty_book = Struct(name="Book", members={}, size=96) empty_book.add_member(title) @@ -72,16 +62,6 @@ def test_declaration(self, class_book: Struct, record_id: Union): def test_str(self, class_book: Struct): assert str(class_book) == "ClassBook" - def test_copy(self, class_book: Struct): - new_class_book: Struct = class_book.copy() - assert id(new_class_book) != id(class_book) - assert new_class_book.size == class_book.size - assert new_class_book.type_specifier == class_book.type_specifier == ComplexTypeSpecifier.CLASS - assert id(new_class_book.members) != id(class_book.members) - assert new_class_book.get_member_by_offset(0) == class_book.get_member_by_offset(0) - assert id(new_class_book.get_member_by_offset(0)) != id(class_book.get_member_by_offset(0)) - assert len(new_class_book.members) == len(class_book.members) - def test_add_members(self, class_book, title, num_pages, author): empty_class_book = Class(name="ClassBook", members={}, size=96) empty_class_book.add_member(title) @@ -156,14 +136,6 @@ def test_declaration(self, record_id): def test_str(self, record_id): assert str(record_id) == "RecordID" - def test_copy(self, record_id): - new_record_id: Union = record_id.copy() - assert new_record_id == record_id - assert id(new_record_id) != id(record_id) - assert id(new_record_id.members) != id(record_id.members) - assert new_record_id.get_member_by_type(Float.float()) == record_id.get_member_by_type(Float.float()) - assert id(new_record_id.get_member_by_type(Float.float())) != id(record_id.get_member_by_type(Float.float())) - def test_add_members(self, empty_record_id, record_id, float_id, int_id, double_id): empty_record_id.add_member(float_id) empty_record_id.add_member(int_id) @@ -225,11 +197,6 @@ def test_declaration(self, color): def test_str(self, color): assert str(color) == "Color" - def test_copy(self, color): - new_color = color.copy() - assert new_color == color - assert id(new_color) != color - def test_add_members(self, empty_color, color, red, green, blue): empty_color.add_member(red) empty_color.add_member(green) From 0fb6969c9ca30bbefc6f0861d0693b1d334f12ad Mon Sep 17 00:00:00 2001 From: rihi <19492038+rihi@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:05:35 +0200 Subject: [PATCH 19/19] Remove unused code --- decompiler/frontend/binaryninja/frontend.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/decompiler/frontend/binaryninja/frontend.py b/decompiler/frontend/binaryninja/frontend.py index 90aa0a002..dd0b9ebe5 100644 --- a/decompiler/frontend/binaryninja/frontend.py +++ b/decompiler/frontend/binaryninja/frontend.py @@ -73,8 +73,6 @@ def lift(self, task: DecompilerTask): tagging = CompilerIdiomsTagging(self._bv, function.start, task.options) tagging.run() - instructions = list(function.instructions) - task.cfg = parser.parse(function) task.complex_types = parser.complex_types except Exception as e: