fix \r in xlsx (#631)

* fix \r in xlsx

* Update credsweeper/filters/value_discord_bot_check.py

credsweeper/deep_scanner/xlsx_scanner.py

@@ -25,11 +25,10 @@ def data_scan(
         candidates = []
         try:
             book = pd.read_excel(io.BytesIO(data_provider.data), sheet_name=None, header=None)
-            sheet_lines = []
             for sheet_name, sheet_data in book.items():
-                text = sheet_data.fillna('').astype(str)
-                for i in text.values:
-                    sheet_lines.append('\t'.join(i))
+                # replace open xml carriage returns _x000D_ before line feed only
+                df = sheet_data.replace(to_replace="_x000D_\n", value='\n', regex=True).fillna('').astype(str)
+                sheet_lines = ['\t'.join(x) for x in df.values]
                 string_data_provider = StringContentProvider(lines=sheet_lines,
                                                              file_path=data_provider.file_path,
                                                              file_type=data_provider.file_type,

credsweeper/filters/value_discord_bot_check.py

@@ -1,9 +1,11 @@
 import contextlib
 
+from credsweeper.common.constants import Chars
 from credsweeper.config import Config
 from credsweeper.credentials import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.filters import Filter
+from credsweeper.filters.value_entropy_base64_check import ValueEntropyBase64Check
 from credsweeper.utils import Util
 
 
@@ -28,6 +30,10 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
             # . must be in value according regex
             dot_separator_index = line_data.value.index('.')
             id_part = line_data.value[:dot_separator_index]
-            if int(Util.decode_base64(id_part, padding_safe=True, urlsafe_detect=True)):
+            discord_id = int(Util.decode_base64(id_part, padding_safe=True, urlsafe_detect=True))
+            entropy_part = line_data.value[dot_separator_index:]
+            entropy = Util.get_shannon_entropy(entropy_part, Chars.BASE64STD_CHARS.value)
+            min_entropy = ValueEntropyBase64Check.get_min_data_entropy(len(line_data.value))
+            if not 1000 > discord_id and not min_entropy < entropy:
                 return False
         return True

tests/__init__.py

@@ -17,10 +17,10 @@
 SAMPLES_POST_CRED_COUNT = SAMPLES_CRED_COUNT - ML_FILTERED
 
 # with option --doc
-SAMPLES_IN_DOC = 453
+SAMPLES_IN_DOC = 463
 
 # archived credentials that are not found without --depth
-SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 29
+SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 33
 SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 54
 SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 1