From dcf0a751fe30c83ab0b6f7e5db56b3078ae4ce5c Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Mon, 22 Jul 2024 15:28:03 +0300 Subject: [PATCH] Code enchancement (#586) * Code enchancement * DCfix --- credsweeper/credentials/candidate.py | 3 + credsweeper/credentials/line_data.py | 41 +- credsweeper/deep_scanner/bzip2_scanner.py | 9 +- credsweeper/deep_scanner/deep_scanner.py | 3 +- credsweeper/deep_scanner/gzip_scanner.py | 11 +- .../filters/value_entropy_base32_check.py | 24 +- .../filters/value_entropy_base36_check.py | 30 +- .../filters/value_entropy_base64_check.py | 36 +- .../value_not_allowed_pattern_check.py | 4 +- .../filters/value_not_part_encoded_check.py | 54 +- .../filters/value_token_base32_check.py | 22 +- .../filters/value_token_base36_check.py | 32 +- .../filters/value_token_base64_check.py | 32 +- .../variable_not_allowed_pattern_check.py | 6 +- tests/data/depth_3.json | 756 +++++++++--------- tests/test_main.py | 10 +- 16 files changed, 545 insertions(+), 528 deletions(-) diff --git a/credsweeper/credentials/candidate.py b/credsweeper/credentials/candidate.py index 85cc1c19e..8800a283e 100644 --- a/credsweeper/credentials/candidate.py +++ b/credsweeper/credentials/candidate.py @@ -96,6 +96,9 @@ def __str__(self) -> str: f" | api_validation: {self.api_validation.name}" \ f" | ml_validation: {self.ml_validation.name}" + def __repr__(self): + return str(self) + def to_json(self) -> Dict: """Convert credential candidate object to dictionary. diff --git a/credsweeper/credentials/line_data.py b/credsweeper/credentials/line_data.py index 42dcd4310..06a68fb5b 100644 --- a/credsweeper/credentials/line_data.py +++ b/credsweeper/credentials/line_data.py @@ -136,14 +136,8 @@ def sanitize_value(self): self.value_start += start self.value_end = self.value_start + len(self.value) - def clean_url_parameters(self) -> None: - """Clean url address from 'query parameters'. - - If line seem to be a URL - split by & character. - Variable should be right most value after & or ? ([-1]). And value should be left most before & ([0]) - """ - # line length cannot exceed MAX_LINE_LENGTH - assert MAX_LINE_LENGTH >= len(self.line) + def check_url_part(self) -> bool: + """Determines whether value is part of url like line""" line_before_value = self.line[:self.value_start] url_pos = -1 find_pos = 0 @@ -161,17 +155,23 @@ def clean_url_parameters(self) -> None: self.url_part &= not self.url_chars_not_allowed_pattern.search(line_before_value, pos=url_pos + 3) self.url_part |= self.line[self.variable_start - 1] in "?&" if 0 < self.variable_start else False self.url_part |= bool(self.url_value_pattern.match(self.value)) - if not self.url_part: - return + return self.url_part - # all checks have passed - line before the value may be a URL - self.variable = self.variable.rsplit('&')[-1].rsplit('?')[-1].rsplit(';')[-1] - self.value = self.value.split('&', maxsplit=1)[0].split(';', maxsplit=1)[0].split('#', maxsplit=1)[0] - if not self.variable.endswith("://"): - # skip sanitize in case of URL credential rule - value_spl = self.url_param_split.split(self.value) - if len(value_spl) > 1: - self.value = value_spl[0] + def clean_url_parameters(self) -> None: + """Clean url address from 'query parameters'. + + If line seem to be a URL - split by & character. + Variable should be right most value after & or ? ([-1]). And value should be left most before & ([0]) + """ + if self.check_url_part(): + # all checks have passed - line before the value may be a URL + self.variable = self.variable.rsplit('&')[-1].rsplit('?')[-1].rsplit(';')[-1] + self.value = self.value.split('&', maxsplit=1)[0].split(';', maxsplit=1)[0].split('#', maxsplit=1)[0] + if not self.variable.endswith("://"): + # skip sanitize in case of URL credential rule + value_spl = self.url_param_split.split(self.value) + if len(value_spl) > 1: + self.value = value_spl[0] def clean_bash_parameters(self) -> None: """Split variable and value by bash special characters, if line assumed to be CLI command.""" @@ -287,10 +287,13 @@ def is_source_file_with_quotes(self) -> bool: return True return False - def __repr__(self) -> str: + def __str__(self): return f"line: '{self.line}' | line_num: {self.line_num} | path: {self.path}" \ f" | value: '{self.value}' | entropy_validation: {EntropyValidator(self.value)}" + def __repr__(self): + return str(self) + def to_json(self) -> Dict: """Convert line data object to dictionary. diff --git a/credsweeper/deep_scanner/bzip2_scanner.py b/credsweeper/deep_scanner/bzip2_scanner.py index ea862744f..0d33b44b1 100644 --- a/credsweeper/deep_scanner/bzip2_scanner.py +++ b/credsweeper/deep_scanner/bzip2_scanner.py @@ -1,6 +1,7 @@ import bz2 import logging from abc import ABC +from pathlib import Path from typing import List from credsweeper.credentials import Candidate @@ -22,10 +23,12 @@ def data_scan( """Extracts data from bzip2 archive and launches data_scan""" candidates = [] try: - new_path = data_provider.file_path if ".bz2" != Util.get_extension( - data_provider.file_path) else data_provider.file_path[:-4] + file_path = Path(data_provider.file_path) + new_path = file_path.as_posix() + if ".bz2" == file_path.suffix: + new_path = new_path[:-4] bzip2_content_provider = DataContentProvider(data=bz2.decompress(data_provider.data), - file_path=data_provider.file_path, + file_path=new_path, file_type=Util.get_extension(new_path), info=f"{data_provider.info}|BZIP2|{new_path}") new_limit = recursive_limit_size - len(bzip2_content_provider.data) diff --git a/credsweeper/deep_scanner/deep_scanner.py b/credsweeper/deep_scanner/deep_scanner.py index f288cd4de..119f569f4 100644 --- a/credsweeper/deep_scanner/deep_scanner.py +++ b/credsweeper/deep_scanner/deep_scanner.py @@ -1,5 +1,6 @@ import datetime import logging +from pathlib import Path from typing import List, Optional, Any, Tuple, Union from credsweeper.common.constants import RECURSIVE_SCAN_LIMITATION @@ -136,7 +137,7 @@ def scan(self, data_provider = DataContentProvider(data=data, file_path=content_provider.file_path, file_type=content_provider.file_type, - info=content_provider.file_path) + info=Path(content_provider.file_path).as_posix()) # iterate for all possibly scanner methods WITHOUT ByteContentProvider for TextContentProvider scanner_classes = self.get_deep_scanners(data, content_provider.file_type) for scan_class in scanner_classes: diff --git a/credsweeper/deep_scanner/gzip_scanner.py b/credsweeper/deep_scanner/gzip_scanner.py index e8ac2be9d..06e2321ca 100644 --- a/credsweeper/deep_scanner/gzip_scanner.py +++ b/credsweeper/deep_scanner/gzip_scanner.py @@ -2,12 +2,13 @@ import io import logging from abc import ABC +from pathlib import Path from typing import List +from credsweeper.utils import Util from credsweeper.credentials import Candidate from credsweeper.deep_scanner.abstract_scanner import AbstractScanner from credsweeper.file_handler.data_content_provider import DataContentProvider -from credsweeper.utils import Util logger = logging.getLogger(__name__) @@ -24,10 +25,12 @@ def data_scan( candidates = [] try: with gzip.open(io.BytesIO(data_provider.data)) as f: - new_path = data_provider.file_path if ".gz" != Util.get_extension( - data_provider.file_path) else data_provider.file_path[:-3] + file_path = Path(data_provider.file_path) + new_path = file_path.as_posix() + if ".gz" == file_path.suffix: + new_path = new_path[:-3] gzip_content_provider = DataContentProvider(data=f.read(), - file_path=data_provider.file_path, + file_path=new_path, file_type=Util.get_extension(new_path), info=f"{data_provider.info}|GZIP|{new_path}") new_limit = recursive_limit_size - len(gzip_content_provider.data) diff --git a/credsweeper/filters/value_entropy_base32_check.py b/credsweeper/filters/value_entropy_base32_check.py index d6b85580d..294a7e47a 100644 --- a/credsweeper/filters/value_entropy_base32_check.py +++ b/credsweeper/filters/value_entropy_base32_check.py @@ -14,6 +14,18 @@ class ValueEntropyBase32Check(Filter): def __init__(self, config: Config = None) -> None: pass + @staticmethod + def get_min_data_entropy(x: int) -> float: + """Returns average entropy for size of random data. Precalculated data is applied for speedup""" + if 16 == x: + y = 3.46 + elif 10 <= x: + # approximation does not exceed stdev + y = 0.64 * math.log2(x) + 0.9 + else: + y = 0 + return y + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: """Run filter checks on received credential candidate data 'line_data'. @@ -28,15 +40,3 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE32_CHARS.value) min_entropy = ValueEntropyBase32Check.get_min_data_entropy(len(line_data.value)) return min_entropy > entropy or 0 == min_entropy - - @staticmethod - def get_min_data_entropy(x: int) -> float: - """Returns average entropy for size of random data. Precalculated data is applied for speedup""" - if 16 == x: - y = 3.46 - elif 10 <= x: - # approximation does not exceed stdev - y = 0.64 * math.log2(x) + 0.9 - else: - y = 0 - return y diff --git a/credsweeper/filters/value_entropy_base36_check.py b/credsweeper/filters/value_entropy_base36_check.py index 44058d03d..8ad48b85f 100644 --- a/credsweeper/filters/value_entropy_base36_check.py +++ b/credsweeper/filters/value_entropy_base36_check.py @@ -14,21 +14,6 @@ class ValueEntropyBase36Check(Filter): def __init__(self, config: Config = None) -> None: pass - def run(self, line_data: LineData, target: AnalysisTarget) -> bool: - """Run filter checks on received credential candidate data 'line_data'. - - Args: - line_data: credential candidate data - target: multiline target from which line data was obtained - - Return: - True, if need to filter candidate and False if left - - """ - entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE36_CHARS.value) - min_entropy = ValueEntropyBase36Check.get_min_data_entropy(len(line_data.value)) - return min_entropy > entropy or 0 == min_entropy - @staticmethod def get_min_data_entropy(x: int) -> float: """Returns minimal entropy for size of random data. Precalculated data is applied for speedup""" @@ -44,3 +29,18 @@ def get_min_data_entropy(x: int) -> float: else: y = 0 return y + + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: + """Run filter checks on received credential candidate data 'line_data'. + + Args: + line_data: credential candidate data + target: multiline target from which line data was obtained + + Return: + True, if need to filter candidate and False if left + + """ + entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE36_CHARS.value) + min_entropy = ValueEntropyBase36Check.get_min_data_entropy(len(line_data.value)) + return min_entropy > entropy or 0 == min_entropy diff --git a/credsweeper/filters/value_entropy_base64_check.py b/credsweeper/filters/value_entropy_base64_check.py index 3a6b6a2c9..f97741a0d 100644 --- a/credsweeper/filters/value_entropy_base64_check.py +++ b/credsweeper/filters/value_entropy_base64_check.py @@ -14,24 +14,6 @@ class ValueEntropyBase64Check(Filter): def __init__(self, config: Config = None) -> None: pass - def run(self, line_data: LineData, target: AnalysisTarget) -> bool: - """Run filter checks on received credential candidate data 'line_data'. - - Args: - line_data: credential candidate data - target: multiline target from which line data was obtained - - Return: - True, if need to filter candidate and False if left - - """ - if '-' in line_data.value or '_' in line_data.value: - entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE64URL_CHARS.value) - else: - entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE64STD_CHARS.value) - min_entropy = ValueEntropyBase64Check.get_min_data_entropy(len(line_data.value)) - return min_entropy > entropy or 0 == min_entropy - @staticmethod def get_min_data_entropy(x: int) -> float: """Returns minimal average entropy for size of random data. Precalculated round data is applied for speedup""" @@ -54,3 +36,21 @@ def get_min_data_entropy(x: int) -> float: else: y = 0 return y + + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: + """Run filter checks on received credential candidate data 'line_data'. + + Args: + line_data: credential candidate data + target: multiline target from which line data was obtained + + Return: + True, if need to filter candidate and False if left + + """ + if '-' in line_data.value or '_' in line_data.value: + entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE64URL_CHARS.value) + else: + entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE64STD_CHARS.value) + min_entropy = ValueEntropyBase64Check.get_min_data_entropy(len(line_data.value)) + return min_entropy > entropy or 0 == min_entropy diff --git a/credsweeper/filters/value_not_allowed_pattern_check.py b/credsweeper/filters/value_not_allowed_pattern_check.py index 265e4cc13..944c9c34e 100644 --- a/credsweeper/filters/value_not_allowed_pattern_check.py +++ b/credsweeper/filters/value_not_allowed_pattern_check.py @@ -29,8 +29,6 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: True, if need to filter candidate and False if left """ - if line_data.is_well_quoted_value: - return False - if self.NOT_ALLOWED_PATTERN.search(line_data.value): + if not line_data.is_well_quoted_value and self.NOT_ALLOWED_PATTERN.search(line_data.value): return True return False diff --git a/credsweeper/filters/value_not_part_encoded_check.py b/credsweeper/filters/value_not_part_encoded_check.py index 86d47405d..6de327a9f 100644 --- a/credsweeper/filters/value_not_part_encoded_check.py +++ b/credsweeper/filters/value_not_part_encoded_check.py @@ -18,6 +18,29 @@ class ValueNotPartEncodedCheck(Filter): def __init__(self, config: Config = None) -> None: pass + @staticmethod + def check_line_target_fit(line_data: LineData, target: AnalysisTarget) -> bool: + """Verifies whether line data fit to be a part of many lines""" + return line_data.line_num == target.line_num \ + and len(line_data.line) == target.line_len \ + and line_data.line == target.line \ + and 0 < target.line_num <= target.lines_len \ + and line_data.line == target.lines[target.line_num - 1] + + @staticmethod + def check_val(line: str, pattern: re.Pattern) -> Optional[bool]: + """Verifies whether the line looks like a pattern""" + match_obj = pattern.match(line) + if match_obj: + val = match_obj.group("val") + # not a path-like + if not val.startswith('/'): + return True + # padding sign + if '=' == val[-1]: + return True + return None + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: """Run filter checks on received credential candidate data 'line_data'. @@ -30,20 +53,16 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: """ - if line_data.line_num == target.line_num \ - and len(line_data.line) == target.line_len \ - and line_data.line == target.line \ - and 0 < target.line_num <= target.lines_len \ - and line_data.line == target.lines[target.line_num - 1]: + if ValueNotPartEncodedCheck.check_line_target_fit(line_data, target): # suppose, there is plain lines order if 1 < target.line_num: - result = ValueNotPartEncodedCheck._check_val( - target.lines[line_data.line_num - 2], ValueNotPartEncodedCheck.BASE64_ENCODED_DATA_PATTERN_BEFORE) + result = ValueNotPartEncodedCheck.check_val(target.lines[line_data.line_num - 2], + ValueNotPartEncodedCheck.BASE64_ENCODED_DATA_PATTERN_BEFORE) if result is not None: return result if target.lines_len > target.line_num: - result = ValueNotPartEncodedCheck._check_val(target.lines[line_data.line_num], - ValueNotPartEncodedCheck.BASE64_ENCODED_DATA_PATTERN_AFTER) + result = ValueNotPartEncodedCheck.check_val(target.lines[line_data.line_num], + ValueNotPartEncodedCheck.BASE64_ENCODED_DATA_PATTERN_AFTER) if result is not None: return result else: @@ -51,28 +70,15 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: for i in range(target.lines_len): if line_data.line == target.lines[i]: if 0 < i: - result = ValueNotPartEncodedCheck._check_val( + result = ValueNotPartEncodedCheck.check_val( target.lines[i - 1], ValueNotPartEncodedCheck.BASE64_ENCODED_DATA_PATTERN_BEFORE) if result is not None: return result i += 1 if target.lines_len > i: - result = ValueNotPartEncodedCheck._check_val( + result = ValueNotPartEncodedCheck.check_val( target.lines[i], ValueNotPartEncodedCheck.BASE64_ENCODED_DATA_PATTERN_AFTER) if result is not None: return result break return False - - @staticmethod - def _check_val(line: str, pattern: re.Pattern) -> Optional[bool]: - match_obj = pattern.match(line) - if match_obj: - val = match_obj.group("val") - # not a path-like - if not val.startswith('/'): - return True - # padding sign - if '=' == val[-1]: - return True - return None diff --git a/credsweeper/filters/value_token_base32_check.py b/credsweeper/filters/value_token_base32_check.py index c3b862239..e51214206 100644 --- a/credsweeper/filters/value_token_base32_check.py +++ b/credsweeper/filters/value_token_base32_check.py @@ -12,6 +12,17 @@ class ValueTokenBase32Check(Filter): def __init__(self, config: Config = None) -> None: pass + @staticmethod + def get_min_strength(x: int) -> float: + """Returns minimal strength. Precalculated data is applied for speedup""" + if 16 == x: + y = 0.7047 + elif 8 <= x <= 32: + y = ((0.000046 * x - 0.0044) * x + 0.146) * x - 0.7 + else: + y = 1 + return y + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: """Run filter checks on received credential candidate data 'line_data'. @@ -27,14 +38,3 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool: strength = float(PasswordStats(line_data.value).strength()) min_strength = ValueTokenBase32Check.get_min_strength(len(line_data.value)) return min_strength > strength - - @staticmethod - def get_min_strength(x: int) -> float: - """Returns minimal strength. Precalculated data is applied for speedup""" - if 16 == x: - y = 0.7047 - elif 8 <= x <= 32: - y = ((0.000046 * x - 0.0044) * x + 0.146) * x - 0.7 - else: - y = 1 - return y diff --git a/credsweeper/filters/value_token_base36_check.py b/credsweeper/filters/value_token_base36_check.py index 4a5e001e1..b0952d20e 100644 --- a/credsweeper/filters/value_token_base36_check.py +++ b/credsweeper/filters/value_token_base36_check.py @@ -12,22 +12,6 @@ class ValueTokenBase36Check(Filter): def __init__(self, config: Config = None) -> None: pass - def run(self, line_data: LineData, target: AnalysisTarget) -> bool: - """Run filter checks on received credential candidate data 'line_data'. - - Args: - line_data: credential candidate data - target: multiline target from which line data was obtained - - Return: - True, if need to filter candidate and False if left - - """ - - strength = float(PasswordStats(line_data.value).strength()) - min_strength = ValueTokenBase36Check.get_min_strength(len(line_data.value)) - return min_strength > strength - @staticmethod def get_min_strength(x: int) -> float: """Returns minimal strength. Precalculated data is applied for speedup""" @@ -43,3 +27,19 @@ def get_min_strength(x: int) -> float: else: y = 1 return y + + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: + """Run filter checks on received credential candidate data 'line_data'. + + Args: + line_data: credential candidate data + target: multiline target from which line data was obtained + + Return: + True, if need to filter candidate and False if left + + """ + + strength = float(PasswordStats(line_data.value).strength()) + min_strength = ValueTokenBase36Check.get_min_strength(len(line_data.value)) + return min_strength > strength diff --git a/credsweeper/filters/value_token_base64_check.py b/credsweeper/filters/value_token_base64_check.py index ced4bb730..d8072796e 100644 --- a/credsweeper/filters/value_token_base64_check.py +++ b/credsweeper/filters/value_token_base64_check.py @@ -12,22 +12,6 @@ class ValueTokenBase64Check(Filter): def __init__(self, config: Config = None) -> None: pass - def run(self, line_data: LineData, target: AnalysisTarget) -> bool: - """Run filter checks on received credential candidate data 'line_data'. - - Args: - line_data: credential candidate data - target: multiline target from which line data was obtained - - Return: - True, if need to filter candidate and False if left - - """ - - strength = float(PasswordStats(line_data.value).strength()) - min_strength = ValueTokenBase64Check.get_min_strength(len(line_data.value)) - return min_strength > strength - @staticmethod def get_min_strength(x: int) -> float: """Returns minimal strength. Precalculated rounded data is applied for speedup""" @@ -44,3 +28,19 @@ def get_min_strength(x: int) -> float: else: y = 1 return y + + def run(self, line_data: LineData, target: AnalysisTarget) -> bool: + """Run filter checks on received credential candidate data 'line_data'. + + Args: + line_data: credential candidate data + target: multiline target from which line data was obtained + + Return: + True, if need to filter candidate and False if left + + """ + + strength = float(PasswordStats(line_data.value).strength()) + min_strength = ValueTokenBase64Check.get_min_strength(len(line_data.value)) + return min_strength > strength diff --git a/credsweeper/filters/variable_not_allowed_pattern_check.py b/credsweeper/filters/variable_not_allowed_pattern_check.py index aaa332077..92b54c5f8 100644 --- a/credsweeper/filters/variable_not_allowed_pattern_check.py +++ b/credsweeper/filters/variable_not_allowed_pattern_check.py @@ -4,17 +4,13 @@ from credsweeper.credentials import LineData from credsweeper.file_handler.analysis_target import AnalysisTarget from credsweeper.filters import Filter -from credsweeper.utils import Util class VariableNotAllowedPatternCheck(Filter): """Check if candidate variable is a regex placeholder or ends with match character (like + or >).""" - NOT_ALLOWED = [ - r"^([<]|\{\{).*", r"(@.*)", r"[!><+*/^|)](\s)?$", r".*public", r".*pubkey", r".*_id$", r".*name$", r".*type$" - ] NOT_ALLOWED_PATTERN = re.compile( # - Util.get_regex_combine_or(NOT_ALLOWED), # + r"(^(<|\{\{).*)|(@.*)|([!><+*/^|)](\s)?$)|(.*(public|pubkey|_id$|name$|type$))", # flags=re.IGNORECASE) def __init__(self, config: Config = None) -> None: diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index b0f34278b..448776d10 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -9871,13 +9871,13 @@ "confidence": "strong", "line_data_list": [ { - "line": "str value = \"-----BEGIN RSA PRIVATE KEY-----\\n\" +", - "line_num": 2, + "line": "-----BEGIN RSA PRIVATE KEY-----", + "line_num": 1, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", "value": "-----BEGIN RSA PRIVATE KEY-----", - "value_start": 13, - "value_end": 44, + "value_start": 0, + "value_end": 31, "variable": null, "variable_start": -2, "variable_end": -2, @@ -9888,243 +9888,206 @@ } }, { - "line": "\"MIICXQIBAAKBgQDwcEN7vZygGg6DvPpsw17hRD6S5N8+huaqs1JGXQfPhbvLTUsHdzGLVNQ/Z0wQVGdPiaJDflqhcT1IH8BLD4SHn+ \\n\"", - "line_num": 3, + "line": "MIICXAIBAAKBgQCqGKukO1De7zhZj6+H0qtjTkVxwTCpvKe4eCZ0FPqri0cb2JZfXJ/DgYSF6vUp", + "line_num": 2, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "MIICXQIBAAKBgQDwcEN7vZygGg6DvPpsw17hRD6S5N8+huaqs1JGXQfPhbvLTUsHdzGLVNQ/Z0wQVGdPiaJDflqhcT1IH8BLD4SHn+", - "value_start": 1, - "value_end": 103, + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "MIICXAIBAAKBgQCqGKukO1De7zhZj6+H0qtjTkVxwTCpvKe4eCZ0FPqri0cb2JZfXJ/DgYSF6vUp", + "value_start": 0, + "value_end": 76, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.429600283437103, + "entropy": 5.388924717191689, "valid": true } }, { - "line": "\t+ \"WuRIzX77P7oVKM2CoTA6VzT6s/bvr7HxFLl4NhohfyDsV0YCDc4I6EHGWMCUHZb0IWxzEGRWD3jbG8KAZUsQIDAQABAoGAOD7a2o\\r \\n\"", - "line_num": 4, + "line": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5", + "line_num": 3, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "WuRIzX77P7oVKM2CoTA6VzT6s/bvr7HxFLl4NhohfyDsV0YCDc4I6EHGWMCUHZb0IWxzEGRWD3jbG8KAZUsQIDAQABAoGAOD7a2o", - "value_start": 4, - "value_end": 104, + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5", + "value_start": 0, + "value_end": 76, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.307627100346911, + "entropy": 5.213332901823973, "valid": true } }, { - "line": "\" CRujY+PP0hS/4sHOBdDvnxa2wdW8NVNNagdCBhvP5Y1edBNMnkWOyuM4e7HzUgO0+8ndWis1OSJTz9EFTyHJm6GOn+/JR62NWNr \"", - "line_num": 5, + "line": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh", + "line_num": 4, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "CRujY+PP0hS/4sHOBdDvnxa2wdW8NVNNagdCBhvP5Y1edBNMnkWOyuM4e7HzUgO0+8ndWis1OSJTz9EFTyHJm6GOn+/JR62NWNr", - "value_start": 3, - "value_end": 102, + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh", + "value_start": 0, + "value_end": 76, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.391947595190542, + "entropy": 5.325046388609145, "valid": true } }, { - "line": "\t\t\"9lfCb8cWq0eoOk3UUO9P+1nZNHjE/iDhuTi3x/5naW4SzdkZfxHo/NMI6i5w1ZnQ60CQQD9d0G9gBy6lPhC \\\\", - "line_num": 6, + "line": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2", + "line_num": 5, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "9lfCb8cWq0eoOk3UUO9P+1nZNHjE/iDhuTi3x/5naW4SzdkZfxHo/NMI6i5w1ZnQ60CQQD9d0G9gBy6lPhC", - "value_start": 3, - "value_end": 86, + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2", + "value_start": 0, + "value_end": 76, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.410712925114303, + "entropy": 5.267278500114894, "valid": true } }, { - "line": "\"\"\" mbHzYovVjSnyfzUtyWPNSrXNiUCR5vu2f6eCgSVzFZ0oHAv8nLaYnXrhyT25lwzNK5OhR/oPAkEA8tep3NmfxV \"\"\" +", - "line_num": 7, + "line": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX", + "line_num": 6, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "mbHzYovVjSnyfzUtyWPNSrXNiUCR5vu2f6eCgSVzFZ0oHAv8nLaYnXrhyT25lwzNK5OhR/oPAkEA8tep3NmfxV", - "value_start": 4, - "value_end": 90, + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX", + "value_start": 0, + "value_end": 76, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.417032341684991, + "entropy": 5.293594289588578, "valid": true } }, { - "line": "QSBfKGfotblIG709xxfd6vHfDS0eZuTFUxkZDcayZDhMDjTMZxP8rokSbMaOSqUseUeYhx5TqFPwJAAhC0smyWz1ZjZ3eFIayN4yGRU+6B\\r\\n", - "line_num": 8, + "line": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il", + "line_num": 7, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "QSBfKGfotblIG709xxfd6vHfDS0eZuTFUxkZDcayZDhMDjTMZxP8rokSbMaOSqUseUeYhx5TqFPwJAAhC0smyWz1ZjZ3eFIayN4yGRU+6B", + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il", "value_start": 0, - "value_end": 106, + "value_end": 76, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.497697548960288, + "entropy": 5.405307776373748, "valid": true } }, { - "line": "\"amwXqhaPwKr7obS2HFiR7thKi9ODQk5oMpi8TCYMWEahgB+g9RMD0u6ZNQJBAMla\\\\\\\\nUJmySGuRnbAYu7PJURH90AOG0QOQ1Jp6yBMKgYIB\\\\", - "line_num": 9, + "line": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF", + "line_num": 8, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "amwXqhaPwKr7obS2HFiR7thKi9ODQk5oMpi8TCYMWEahgB+g9RMD0u6ZNQJBAMla\\\\\\\\nUJmySGuRnbAYu7PJURH90AOG0QOQ1Jp6yBMKgYIB", - "value_start": 1, - "value_end": 110, + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF", + "value_start": 0, + "value_end": 76, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.30959588168204, + "entropy": 5.372541658009631, "valid": true } }, { - "line": "Uaxk9J5Th8BXPyC1mclPMS7J\\\\\\\\ngMuobiFdIpryq51pvHkCQQDInvDaKI24Ho+cj6SCKnEO4kbjh/yx1XGwNmY0ld1i\\\\r5micHFiMI7/IcVZ4\\n", - "line_num": 10, + "line": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k", + "line_num": 9, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "Uaxk9J5Th8BXPyC1mclPMS7J", + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k", "value_start": 0, - "value_end": 24, + "value_end": 76, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.418295834054489, - "valid": false + "entropy": 5.308663329427085, + "valid": true } }, { - "line": "2cl1OwdGjRdmO1LT6P1cl8UYIj/S\\n-----END RSA PRIVATE KEY-----\"\"\";", - "line_num": 11, + "line": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl", + "line_num": 10, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "2cl1OwdGjRdmO1LT6P1cl8UYIj/S\\n-----END RSA PRIVATE KEY-----", + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl", "value_start": 0, - "value_end": 59, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.8721063510981755, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "PEM Private Key", - "severity": "high", - "confidence": "strong", - "line_data_list": [ - { - "line": "char pk[] = \"\\\\n-----BEGIN EC PRIVATE KEY-----\\nMHcCAQEEID4VC4s0b2n3yvypHb2GO9prFUctYt\\r\\r\\n\\nHmGCMvpxkpexbHoAoGCCqGSM49\\\\\\\\\\nAwEHoUQDQgAE2GwUUuO9/dKl51bOryWzHF8wTSezSqdRIucGhDRsmDITLcNEZw3V\\\\\\\\rIaQP59Ufkz8NIkLeIAa1HZGZxCVMQ907FA==\\\\n-----END EC PRIVATE KEY-----\\n\";", - "line_num": 14, - "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "-----BEGIN EC PRIVATE KEY-----\\nMHcCAQEEID4VC4s0b2n3yvypHb2GO9prFUctYt\\r\\r\\n\\nHmGCMvpxkpexbHoAoGCCqGSM49\\\\\\\\\\nAwEHoUQDQgAE2GwUUuO9/dKl51bOryWzHF8wTSezSqdRIucGhDRsmDITLcNEZw3V\\\\\\\\rIaQP59Ufkz8NIkLeIAa1HZGZxCVMQ907FA==\\\\n-----END EC PRIVATE KEY-----", - "value_start": 16, - "value_end": 262, + "value_end": 76, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.884128060579619, + "entropy": 5.425173236956998, "valid": true } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "PEM Private Key", - "severity": "high", - "confidence": "strong", - "line_data_list": [ + }, { - "line": "-----BEGIN OPENSSH LOW ENTROPY PRIVATE KEY-----", - "line_num": 27, + "line": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ", + "line_num": 11, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "-----BEGIN OPENSSH LOW ENTROPY PRIVATE KEY-----", + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ", "value_start": 0, - "value_end": 47, + "value_end": 76, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.0185646866544076, - "valid": false + "entropy": 5.107555270017291, + "valid": true } }, { - "line": "12345678901231278634987284736283548102438723941563428762374129402103402394932746672734543664375t7323341253845186253784==", - "line_num": 28, + "line": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=", + "line_num": 12, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "12345678901231278634987284736283548102438723941563428762374129402103402394932746672734543664375t7323341253845186253784==", + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=", "value_start": 0, - "value_end": 120, + "value_end": 52, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2083460372948154, + "iterator": "BASE64_CHARS", + "entropy": 5.0705380354071785, "valid": true } }, { - "line": "-----END LOW ENTROPY PRIVATE KEY-----", - "line_num": 29, + "line": "-----END RSA PRIVATE KEY-----", + "line_num": 13, "path": "tests/samples/pem_key", - "info": "tests/samples/pem_key|RAW", - "value": "-----END LOW ENTROPY PRIVATE KEY-----", + "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "value": "-----END RSA PRIVATE KEY-----", "value_start": 0, - "value_end": 37, + "value_end": 29, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.6977683083639423, + "entropy": 2.3783727041337137, "valid": false } } @@ -10141,8 +10104,8 @@ { "line": "-----BEGIN RSA PRIVATE KEY-----", "line_num": 1, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "-----BEGIN RSA PRIVATE KEY-----", "value_start": 0, "value_end": 31, @@ -10158,8 +10121,8 @@ { "line": "MIICXAIBAAKBgQCqGKukO1De7zhZj6+H0qtjTkVxwTCpvKe4eCZ0FPqri0cb2JZfXJ/DgYSF6vUp", "line_num": 2, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "MIICXAIBAAKBgQCqGKukO1De7zhZj6+H0qtjTkVxwTCpvKe4eCZ0FPqri0cb2JZfXJ/DgYSF6vUp", "value_start": 0, "value_end": 76, @@ -10175,8 +10138,8 @@ { "line": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5", "line_num": 3, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5", "value_start": 0, "value_end": 76, @@ -10192,8 +10155,8 @@ { "line": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh", "line_num": 4, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh", "value_start": 0, "value_end": 76, @@ -10209,8 +10172,8 @@ { "line": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2", "line_num": 5, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2", "value_start": 0, "value_end": 76, @@ -10226,8 +10189,8 @@ { "line": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX", "line_num": 6, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX", "value_start": 0, "value_end": 76, @@ -10243,8 +10206,8 @@ { "line": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il", "line_num": 7, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il", "value_start": 0, "value_end": 76, @@ -10260,8 +10223,8 @@ { "line": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF", "line_num": 8, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF", "value_start": 0, "value_end": 76, @@ -10277,8 +10240,8 @@ { "line": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k", "line_num": 9, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k", "value_start": 0, "value_end": 76, @@ -10294,8 +10257,8 @@ { "line": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl", "line_num": 10, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl", "value_start": 0, "value_end": 76, @@ -10311,8 +10274,8 @@ { "line": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ", "line_num": 11, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ", "value_start": 0, "value_end": 76, @@ -10328,8 +10291,8 @@ { "line": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=", "line_num": 12, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=", "value_start": 0, "value_end": 52, @@ -10345,8 +10308,8 @@ { "line": "-----END RSA PRIVATE KEY-----", "line_num": 13, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", "value": "-----END RSA PRIVATE KEY-----", "value_start": 0, "value_end": 29, @@ -10361,60 +10324,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "PayPal Braintree Access Token", - "severity": "high", - "confidence": "strong", - "line_data_list": [ - { - "line": "access_token$production$gireogi121451781$abcaeaabadef01134517891121451781", - "line_num": 1, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|.git/paypal_key|RAW", - "value": "access_token$production$gireogi121451781$abcaeaabadef01134517891121451781", - "value_start": 0, - "value_end": 73, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 4.125814952938166, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "Jfrog Token", - "severity": "high", - "confidence": "strong", - "line_data_list": [ - { - "line": "cmVmdGtuOjAxOjAxMjM0NTY3ODk6QWJjZGVmR2hpamtsbW5vUHFyc3R1dnd4eXow", - "line_num": 1, - "path": "tests/samples/pem_key.apk", - "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", - "value": "cmVmdGtuOjAxOjAxMjM0NTY3ODk6QWJjZGVmR2hpamtsbW5vUHFyc3R1dnd4eXow", - "value_start": 0, - "value_end": 64, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.288909765557392, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -10424,13 +10333,13 @@ "confidence": "strong", "line_data_list": [ { - "line": "-----BEGIN RSA PRIVATE KEY-----", - "line_num": 1, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", + "line": "str value = \"-----BEGIN RSA PRIVATE KEY-----\\n\" +", + "line_num": 2, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", "value": "-----BEGIN RSA PRIVATE KEY-----", - "value_start": 0, - "value_end": 31, + "value_start": 13, + "value_end": 44, "variable": null, "variable_start": -2, "variable_end": -2, @@ -10441,206 +10350,243 @@ } }, { - "line": "MIICXAIBAAKBgQCqGKukO1De7zhZj6+H0qtjTkVxwTCpvKe4eCZ0FPqri0cb2JZfXJ/DgYSF6vUp", - "line_num": 2, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "MIICXAIBAAKBgQCqGKukO1De7zhZj6+H0qtjTkVxwTCpvKe4eCZ0FPqri0cb2JZfXJ/DgYSF6vUp", - "value_start": 0, - "value_end": 76, + "line": "\"MIICXQIBAAKBgQDwcEN7vZygGg6DvPpsw17hRD6S5N8+huaqs1JGXQfPhbvLTUsHdzGLVNQ/Z0wQVGdPiaJDflqhcT1IH8BLD4SHn+ \\n\"", + "line_num": 3, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "MIICXQIBAAKBgQDwcEN7vZygGg6DvPpsw17hRD6S5N8+huaqs1JGXQfPhbvLTUsHdzGLVNQ/Z0wQVGdPiaJDflqhcT1IH8BLD4SHn+", + "value_start": 1, + "value_end": 103, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.388924717191689, + "entropy": 5.429600283437103, "valid": true } }, { - "line": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5", - "line_num": 3, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5", - "value_start": 0, - "value_end": 76, + "line": "\t+ \"WuRIzX77P7oVKM2CoTA6VzT6s/bvr7HxFLl4NhohfyDsV0YCDc4I6EHGWMCUHZb0IWxzEGRWD3jbG8KAZUsQIDAQABAoGAOD7a2o\\r \\n\"", + "line_num": 4, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "WuRIzX77P7oVKM2CoTA6VzT6s/bvr7HxFLl4NhohfyDsV0YCDc4I6EHGWMCUHZb0IWxzEGRWD3jbG8KAZUsQIDAQABAoGAOD7a2o", + "value_start": 4, + "value_end": 104, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.213332901823973, + "entropy": 5.307627100346911, "valid": true } }, { - "line": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh", - "line_num": 4, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh", - "value_start": 0, - "value_end": 76, + "line": "\" CRujY+PP0hS/4sHOBdDvnxa2wdW8NVNNagdCBhvP5Y1edBNMnkWOyuM4e7HzUgO0+8ndWis1OSJTz9EFTyHJm6GOn+/JR62NWNr \"", + "line_num": 5, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "CRujY+PP0hS/4sHOBdDvnxa2wdW8NVNNagdCBhvP5Y1edBNMnkWOyuM4e7HzUgO0+8ndWis1OSJTz9EFTyHJm6GOn+/JR62NWNr", + "value_start": 3, + "value_end": 102, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.325046388609145, + "entropy": 5.391947595190542, "valid": true } }, { - "line": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2", - "line_num": 5, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2", - "value_start": 0, - "value_end": 76, + "line": "\t\t\"9lfCb8cWq0eoOk3UUO9P+1nZNHjE/iDhuTi3x/5naW4SzdkZfxHo/NMI6i5w1ZnQ60CQQD9d0G9gBy6lPhC \\\\", + "line_num": 6, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "9lfCb8cWq0eoOk3UUO9P+1nZNHjE/iDhuTi3x/5naW4SzdkZfxHo/NMI6i5w1ZnQ60CQQD9d0G9gBy6lPhC", + "value_start": 3, + "value_end": 86, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.267278500114894, + "entropy": 5.410712925114303, "valid": true } }, { - "line": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX", - "line_num": 6, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX", - "value_start": 0, - "value_end": 76, + "line": "\"\"\" mbHzYovVjSnyfzUtyWPNSrXNiUCR5vu2f6eCgSVzFZ0oHAv8nLaYnXrhyT25lwzNK5OhR/oPAkEA8tep3NmfxV \"\"\" +", + "line_num": 7, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "mbHzYovVjSnyfzUtyWPNSrXNiUCR5vu2f6eCgSVzFZ0oHAv8nLaYnXrhyT25lwzNK5OhR/oPAkEA8tep3NmfxV", + "value_start": 4, + "value_end": 90, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.293594289588578, + "entropy": 5.417032341684991, "valid": true } }, { - "line": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il", - "line_num": 7, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il", + "line": "QSBfKGfotblIG709xxfd6vHfDS0eZuTFUxkZDcayZDhMDjTMZxP8rokSbMaOSqUseUeYhx5TqFPwJAAhC0smyWz1ZjZ3eFIayN4yGRU+6B\\r\\n", + "line_num": 8, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "QSBfKGfotblIG709xxfd6vHfDS0eZuTFUxkZDcayZDhMDjTMZxP8rokSbMaOSqUseUeYhx5TqFPwJAAhC0smyWz1ZjZ3eFIayN4yGRU+6B", "value_start": 0, - "value_end": 76, + "value_end": 106, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.405307776373748, + "entropy": 5.497697548960288, "valid": true } }, { - "line": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF", - "line_num": 8, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF", - "value_start": 0, - "value_end": 76, + "line": "\"amwXqhaPwKr7obS2HFiR7thKi9ODQk5oMpi8TCYMWEahgB+g9RMD0u6ZNQJBAMla\\\\\\\\nUJmySGuRnbAYu7PJURH90AOG0QOQ1Jp6yBMKgYIB\\\\", + "line_num": 9, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "amwXqhaPwKr7obS2HFiR7thKi9ODQk5oMpi8TCYMWEahgB+g9RMD0u6ZNQJBAMla\\\\\\\\nUJmySGuRnbAYu7PJURH90AOG0QOQ1Jp6yBMKgYIB", + "value_start": 1, + "value_end": 110, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.372541658009631, + "entropy": 5.30959588168204, "valid": true } }, { - "line": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k", - "line_num": 9, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k", + "line": "Uaxk9J5Th8BXPyC1mclPMS7J\\\\\\\\ngMuobiFdIpryq51pvHkCQQDInvDaKI24Ho+cj6SCKnEO4kbjh/yx1XGwNmY0ld1i\\\\r5micHFiMI7/IcVZ4\\n", + "line_num": 10, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "Uaxk9J5Th8BXPyC1mclPMS7J", "value_start": 0, - "value_end": 76, + "value_end": 24, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.308663329427085, - "valid": true + "entropy": 4.418295834054489, + "valid": false } }, { - "line": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl", - "line_num": 10, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl", + "line": "2cl1OwdGjRdmO1LT6P1cl8UYIj/S\\n-----END RSA PRIVATE KEY-----\"\"\";", + "line_num": 11, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "2cl1OwdGjRdmO1LT6P1cl8UYIj/S\\n-----END RSA PRIVATE KEY-----", "value_start": 0, - "value_end": 76, + "value_end": 59, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.8721063510981755, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "PEM Private Key", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "char pk[] = \"\\\\n-----BEGIN EC PRIVATE KEY-----\\nMHcCAQEEID4VC4s0b2n3yvypHb2GO9prFUctYt\\r\\r\\n\\nHmGCMvpxkpexbHoAoGCCqGSM49\\\\\\\\\\nAwEHoUQDQgAE2GwUUuO9/dKl51bOryWzHF8wTSezSqdRIucGhDRsmDITLcNEZw3V\\\\\\\\rIaQP59Ufkz8NIkLeIAa1HZGZxCVMQ907FA==\\\\n-----END EC PRIVATE KEY-----\\n\";", + "line_num": 14, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "-----BEGIN EC PRIVATE KEY-----\\nMHcCAQEEID4VC4s0b2n3yvypHb2GO9prFUctYt\\r\\r\\n\\nHmGCMvpxkpexbHoAoGCCqGSM49\\\\\\\\\\nAwEHoUQDQgAE2GwUUuO9/dKl51bOryWzHF8wTSezSqdRIucGhDRsmDITLcNEZw3V\\\\\\\\rIaQP59Ufkz8NIkLeIAa1HZGZxCVMQ907FA==\\\\n-----END EC PRIVATE KEY-----", + "value_start": 16, + "value_end": 262, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.425173236956998, + "entropy": 4.884128060579619, "valid": true } - }, + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "PEM Private Key", + "severity": "high", + "confidence": "strong", + "line_data_list": [ { - "line": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ", - "line_num": 11, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ", + "line": "-----BEGIN OPENSSH LOW ENTROPY PRIVATE KEY-----", + "line_num": 27, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "-----BEGIN OPENSSH LOW ENTROPY PRIVATE KEY-----", "value_start": 0, - "value_end": 76, + "value_end": 47, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 5.107555270017291, - "valid": true + "entropy": 3.0185646866544076, + "valid": false } }, { - "line": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=", - "line_num": 12, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=", + "line": "12345678901231278634987284736283548102438723941563428762374129402103402394932746672734543664375t7323341253845186253784==", + "line_num": 28, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "12345678901231278634987284736283548102438723941563428762374129402103402394932746672734543664375t7323341253845186253784==", "value_start": 0, - "value_end": 52, + "value_end": 120, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.0705380354071785, + "iterator": "BASE36_CHARS", + "entropy": 3.2083460372948154, "valid": true } }, { - "line": "-----END RSA PRIVATE KEY-----", - "line_num": 13, - "path": "tests/samples/pem_key.bz2", - "info": "tests/samples/pem_key.bz2|BZIP2|tests/samples/pem_key|RAW", - "value": "-----END RSA PRIVATE KEY-----", + "line": "-----END LOW ENTROPY PRIVATE KEY-----", + "line_num": 29, + "path": "tests/samples/pem_key", + "info": "tests/samples/pem_key|RAW", + "value": "-----END LOW ENTROPY PRIVATE KEY-----", "value_start": 0, - "value_end": 29, + "value_end": 37, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.3783727041337137, + "entropy": 2.6977683083639423, "valid": false } } @@ -10657,8 +10603,8 @@ { "line": "-----BEGIN RSA PRIVATE KEY-----", "line_num": 1, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "-----BEGIN RSA PRIVATE KEY-----", "value_start": 0, "value_end": 31, @@ -10674,8 +10620,8 @@ { "line": "MIICXAIBAAKBgQCqGKukO1De7zhZj6+H0qtjTkVxwTCpvKe4eCZ0FPqri0cb2JZfXJ/DgYSF6vUp", "line_num": 2, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "MIICXAIBAAKBgQCqGKukO1De7zhZj6+H0qtjTkVxwTCpvKe4eCZ0FPqri0cb2JZfXJ/DgYSF6vUp", "value_start": 0, "value_end": 76, @@ -10691,8 +10637,8 @@ { "line": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5", "line_num": 3, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5", "value_start": 0, "value_end": 76, @@ -10708,8 +10654,8 @@ { "line": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh", "line_num": 4, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh", "value_start": 0, "value_end": 76, @@ -10725,8 +10671,8 @@ { "line": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2", "line_num": 5, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2", "value_start": 0, "value_end": 76, @@ -10742,8 +10688,8 @@ { "line": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX", "line_num": 6, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX", "value_start": 0, "value_end": 76, @@ -10759,8 +10705,8 @@ { "line": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il", "line_num": 7, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il", "value_start": 0, "value_end": 76, @@ -10776,8 +10722,8 @@ { "line": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF", "line_num": 8, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF", "value_start": 0, "value_end": 76, @@ -10793,8 +10739,8 @@ { "line": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k", "line_num": 9, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k", "value_start": 0, "value_end": 76, @@ -10810,8 +10756,8 @@ { "line": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl", "line_num": 10, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl", "value_start": 0, "value_end": 76, @@ -10827,8 +10773,8 @@ { "line": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ", "line_num": 11, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ", "value_start": 0, "value_end": 76, @@ -10844,8 +10790,8 @@ { "line": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=", "line_num": 12, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=", "value_start": 0, "value_end": 52, @@ -10861,8 +10807,8 @@ { "line": "-----END RSA PRIVATE KEY-----", "line_num": 13, - "path": "tests/samples/pem_key.gz", - "info": "tests/samples/pem_key.gz|GZIP|tests/samples/pem_key|RAW", + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|cred/pem_key.zip|ZIP|key.pem|RAW", "value": "-----END RSA PRIVATE KEY-----", "value_start": 0, "value_end": 29, @@ -10877,6 +10823,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "PayPal Braintree Access Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "access_token$production$gireogi121451781$abcaeaabadef01134517891121451781", + "line_num": 1, + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|.git/paypal_key|RAW", + "value": "access_token$production$gireogi121451781$abcaeaabadef01134517891121451781", + "value_start": 0, + "value_end": 73, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 4.125814952938166, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Jfrog Token", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "cmVmdGtuOjAxOjAxMjM0NTY3ODk6QWJjZGVmR2hpamtsbW5vUHFyc3R1dnd4eXow", + "line_num": 1, + "path": "tests/samples/pem_key.apk", + "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW", + "value": "cmVmdGtuOjAxOjAxMjM0NTY3ODk6QWJjZGVmR2hpamtsbW5vUHFyc3R1dnd4eXow", + "value_start": 0, + "value_end": 64, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.288909765557392, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -11243,6 +11243,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.852, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t : Password = WeR15tr0n6", + "line_num": 1, + "path": "tests/samples/sample.docx", + "info": "tests/samples/sample.docx.gz|GZIP|tests/samples/sample.docx|ZIP|word/document.xml|XML", + "value": "WeR15tr0n6", + "value_start": 77, + "value_end": 87, + "variable": "Password", + "variable_start": 66, + "variable_end": 74, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.321928094887362, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -11279,10 +11306,10 @@ "confidence": "weak", "line_data_list": [ { - "line": "18LibreOffice/7.6.2.1$Linux_X86_64 LibreOffice_project/9d0b4c0791fc17bc4181a67fd90c5aaed576d1c015.00001460622", + "line": "18LibreOffice/7.6.2.1$Linux_X86_64 LibreOffice_project/9d0b4c0791fc17bc4181a67fd90c5aaed576d1c015.00001459612", "line_num": 2, "path": "tests/samples/sample.docx", - "info": "tests/samples/sample.docx|ZIP|docProps/app.xml|RAW", + "info": "tests/samples/sample.docx.gz|GZIP|tests/samples/sample.docx|ZIP|docProps/app.xml|RAW", "value": "7.6.2.1", "value_start": 245, "value_end": 252, @@ -11301,51 +11328,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "Github Classic Token", - "severity": "high", - "confidence": "strong", + "rule": "IPv4", + "severity": "info", + "confidence": "weak", "line_data_list": [ { - "line": "Password = WeR15tr0n6ghs_00000000000000000000000000000004WZ4EQ", + "line": "18LibreOffice/7.6.2.1$Linux_X86_64 LibreOffice_project/9d0b4c0791fc17bc4181a67fd90c5aaed576d1c015.00001460622", "line_num": 2, "path": "tests/samples/sample.docx", - "info": "tests/samples/sample.docx|ZIP|word/document.xml|RAW", - "value": "ghs_00000000000000000000000000000004WZ4EQ", - "value_start": 1628, - "value_end": 1669, + "info": "tests/samples/sample.docx|ZIP|docProps/app.xml|RAW", + "value": "7.6.2.1", + "value_start": 245, + "value_end": 252, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.4322437698226884, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.852, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t : Password = WeR15tr0n6", - "line_num": 1, - "path": "tests/samples/sample.docx.gz", - "info": "tests/samples/sample.docx.gz|GZIP|tests/samples/sample.docx|ZIP|word/document.xml|XML", - "value": "WeR15tr0n6", - "value_start": 77, - "value_end": 87, - "variable": "Password", - "variable_start": 66, - "variable_end": 74, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.321928094887362, + "entropy": 1.6042028126043455, "valid": false } } @@ -11355,24 +11355,24 @@ "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "rule": "Github Classic Token", + "severity": "high", + "confidence": "strong", "line_data_list": [ { - "line": "18LibreOffice/7.6.2.1$Linux_X86_64 LibreOffice_project/9d0b4c0791fc17bc4181a67fd90c5aaed576d1c015.00001459612", + "line": "Password = WeR15tr0n6ghs_00000000000000000000000000000004WZ4EQ", "line_num": 2, - "path": "tests/samples/sample.docx.gz", - "info": "tests/samples/sample.docx.gz|GZIP|tests/samples/sample.docx|ZIP|docProps/app.xml|RAW", - "value": "7.6.2.1", - "value_start": 245, - "value_end": 252, + "path": "tests/samples/sample.docx", + "info": "tests/samples/sample.docx|ZIP|word/document.xml|RAW", + "value": "ghs_00000000000000000000000000000004WZ4EQ", + "value_start": 1628, + "value_end": 1669, "variable": null, "variable_start": -2, "variable_end": -2, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.6042028126043455, + "entropy": 1.4322437698226884, "valid": false } } diff --git a/tests/test_main.py b/tests/test_main.py index 96265b323..d7066debf 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -33,6 +33,12 @@ class TestMain(unittest.TestCase): + def setUp(self): + self.maxDiff = None + + def tearDown(self): + pass + def test_ml_validation_p(self) -> None: cred_sweeper = CredSweeper() self.assertEqual(ThresholdPreset.medium, cred_sweeper.ml_threshold) @@ -412,7 +418,7 @@ def test_tar_n(self) -> None: with patch('logging.Logger.error') as mocked_logger: cred_sweeper.run(content_provider=content_provider) self.assertEqual(0, len(cred_sweeper.credential_manager.get_credentials())) - mocked_logger.assert_called_with(f"{file_path}:unexpected end of data") + mocked_logger.assert_called_with(f"{file_path.as_posix()[:-4]}:unexpected end of data") # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -732,8 +738,6 @@ def prepare(report: List[Dict[str, Any]]): k["ml_probability"], )) - # do not use parametrised tests with unittests - self.maxDiff = 65536 # instead the config file is used with tempfile.TemporaryDirectory() as tmp_dir: for cfg in DATA_TEST_CFG: