Force Utf-8 encoding throughout code (#376)

* Create draft PR for #373
* Add explicit utf-8 encoding to file apis

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: rihi <[email protected]>

decompiler/util/commandline.py

@@ -1,5 +1,5 @@
 """Command line interface for the decompiler."""
-from argparse import SUPPRESS, ArgumentParser, HelpFormatter
+from argparse import SUPPRESS, ArgumentParser
 from enum import Enum
 from os import isatty
 from os.path import isfile
@@ -66,7 +66,7 @@ def main(interface: "Decompiler"):
         output_stream = None
         color = args.color == Colorize.ALWAYS or (args.color != Colorize.NEVER and isatty(stdout.fileno()))
     else:
-        output_stream = open(args.outfile, "w")
+        output_stream = open(args.outfile, "w", encoding="utf-8")
         color = False
     try:
         if args.all or not args.function:

decompiler/util/decoration.py

@@ -77,7 +77,7 @@ def export_ascii(self) -> str:
 
     def export_dot(self, path: str):
         """Export the graph into a dotfile at the given location."""
-        with open(path, "w") as outfile:
+        with open(path, "w", encoding="utf-8") as outfile:
             self._write_dot(outfile)
 
     def export_plot(self, path: str, type="png"):
@@ -352,7 +352,7 @@ def reformat(self):
 
             run(["astyle", "-z2", "-n", file.name], check=True, capture_output=True)
 
-            with open(file.name, "r") as output:
+            with open(file.name, "r", encoding="utf-8") as output:
                 self._text = output.read()
 
     def export_ascii(self) -> str:

decompiler/util/options.py

@@ -80,7 +80,7 @@ def _load_user_config(self):
         """Load additional user settings and override defaults"""
         if isfile(self.USER_CONFIG):
             logging.debug(f"user config found at {self.USER_CONFIG}")
-            with open(self.USER_CONFIG, "r") as f:
+            with open(self.USER_CONFIG, "r", encoding="utf-8") as f:
                 try:
                     self._settings_key_values.update(json.load(f))
                 except json.JSONDecodeError:
@@ -275,7 +275,7 @@ def _iter_argparse_kwargs_for_default_options(cls, options: List[Dict]):
     @staticmethod
     def _read_json_file(filepath: str):
         """Return parsed JSON file"""
-        with open(filepath, "r") as f:
+        with open(filepath, "r", encoding="utf-8") as f:
             return json.load(f)
 
     @staticmethod

tests/conftest.py

@@ -70,7 +70,7 @@ def _discover_system_tests() -> Dict[pathlib.Path, List[str]]:
 
 def _discover_coreutils_tests() -> List[Tuple[pathlib.Path, str]]:
     """Returns list of (binary, func_name) from a text file for the coreutils binaries."""
-    with pathlib.Path("tests/coreutils/functions.txt").open("r") as f:
+    with pathlib.Path("tests/coreutils/functions.txt").open("r", encoding="utf-8") as f:
         funcs_contents = f.readlines()
     files = []
     for line in funcs_contents:
@@ -98,7 +98,7 @@ def _discover_test_functions_in_sample_code(sample: pathlib.Path) -> List[str]:
     """Discover test functions in the given source file.
     Test function to be included have to be named 'testN' where 'N' has to be an integer."""
     test_names = list()
-    with sample.open("r") as f:
+    with sample.open("r", encoding="utf-8") as f:
         for line in f.readlines():
             if match := re.match(r"\w+ (?P<test_name>test\d+)\(.*\)", line):
                 test_names.append(match.group("test_name"))
@@ -109,7 +109,7 @@ def _parse_makefile() -> Dict[str, pathlib.Path]:
     """Parse from Makefile path to systemtests sources and binaries as well as
     path to extended tests sources and binaries"""
     makefile = dict()
-    with pathlib.Path("Makefile").open("r") as f:
+    with pathlib.Path("Makefile").open("r", encoding="utf-8") as f:
         mkfile_contents = f.readlines()
     for line in mkfile_contents:
         if match := re.match(r"^SYSTEM_TESTS_BIN_PATH\s:=\s(.*)$", line):

tests/frontend/test-lifting.py

@@ -1,11 +1,9 @@
-import argparse
 import os
 import sys
 
 current_dir = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(current_dir)
 
-import collections
 import logging
 
 from binaryninja import *
@@ -34,7 +32,7 @@
 
                 pass
 
-    # with open('/tmp/unimplemented.json', 'w') as f:
+    # with open('/tmp/unimplemented.json', 'w', encoding="utf-8") as f:
     #
     #     json.dump(lifting_utils.unimplemented_map, f)
 

tests/util/test_closeable_named_temporary_file.py

@@ -5,10 +5,10 @@
 
 class TestCloseableNamedTemporaryFile:
     def test_usage_after_closing(self):
-        with CloseableNamedTemporaryFile(mode="w") as file:
+        with CloseableNamedTemporaryFile(mode="w", encoding="utf-8") as file:
             file.write("test")
             file.close()
-            with open(file.name, "r") as reopened_file:
+            with open(file.name, "r", encoding="utf-8") as reopened_file:
                 assert reopened_file.read() == "test"
 
     def test_deletion_with_close(self):