diff --git a/cicd/benchmark.txt b/cicd/benchmark.txt index 3bd78ff61..394e5d888 100644 --- a/cicd/benchmark.txt +++ b/cicd/benchmark.txt @@ -10,16 +10,16 @@ Predefined Pattern 326 2 40 Private Key 1001 1 3 Seed, Salt, Nonce 40 4 4 TOTAL: 5307 63688 5644 -Detected Credentials: 5993 -credsweeper result_cnt : 5337, lost_cnt : 0, true_cnt : 4439, false_cnt : 898 +Detected Credentials: 5997 +credsweeper result_cnt : 5339, lost_cnt : 0, true_cnt : 4441, false_cnt : 898 Category TP FP TN FN FPR FNR ACC PRC RCL F1 -------------------------- ---- ---- -------- ---- --------- --------- -------- -------- -------- -------- Authentication Key & Token 54 4 28 16 0.125 0.228571 0.803922 0.931034 0.771429 0.84375 Generic Secret 973 3 215 83 0.0137615 0.0785985 0.932496 0.996926 0.921402 0.957677 -Generic Token 287 7 596 46 0.0116086 0.138138 0.943376 0.97619 0.861862 0.91547 +Generic Token 289 7 596 44 0.0116086 0.132132 0.945513 0.976351 0.867868 0.918919 Other 818 750 63395 258 0.0116923 0.239777 0.984545 0.521684 0.760223 0.618759 Password 995 130 4150 410 0.0303738 0.291815 0.905013 0.884444 0.708185 0.786561 Predefined Pattern 309 2 40 17 0.0476191 0.0521472 0.94837 0.993569 0.947853 0.970173 Private Key 967 0 4 34 0.033966 0.966169 1 0.966034 0.982724 Seed, Salt, Nonce 36 2 6 4 0.25 0.1 0.875 0.947368 0.9 0.923077 - 4439 898 19428253 868 4.622e-05 0.163558 0.999909 0.831741 0.836442 0.834085 + 4441 898 19428253 866 4.622e-05 0.163181 0.999909 0.831804 0.836819 0.834304 diff --git a/credsweeper/filters/value_allowlist_check.py b/credsweeper/filters/value_allowlist_check.py index 0db5085a5..be30dedca 100644 --- a/credsweeper/filters/value_allowlist_check.py +++ b/credsweeper/filters/value_allowlist_check.py @@ -11,7 +11,7 @@ class ValueAllowlistCheck(Filter): """Check that patterns from the list is not present in the candidate value.""" ALLOWED = [ - r"ENC\(.*\)", r"ENC\[.*\]", r"\$\{.*\}", r"#\{.*\}", r"\{\{.+\}\}", r"([.a-z0-9]|->)+\(.*\)", r"\*\*\*\*\*" + r"ENC\(.*\)", r"ENC\[.*\]", r"\$\{.*\}", r"#\{.*\}", r"\{\{.+\}\}", r"([.a-z0-9]|->)+\(.*\)", r"\S{0,5}\*{5,}" ] ALLOWED_PATTERN = re.compile( # Util.get_regex_combine_or(ALLOWED), # diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index 71b7c5b1c..0aabb7fa1 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -5,6 +5,7 @@ - (?P[`'\"]?(?i:token|secret|key|키|암호|암호화|토큰)[`'\"]?)((\s)*[=:](\s)*)(?P[`'\"(])?(?P\S{4,})(?(quote)[)`'\"]) filter_type: - ValueAllowlistCheck + - ValuePatternCheck min_line_len: 10 required_substrings: - token @@ -26,6 +27,7 @@ - (?P[`'\"]?(?i:(?[`'\"(])?(?P\S{4,})(?(quote)[)`'\"]) filter_type: - ValueAllowlistCheck + - ValuePatternCheck min_line_len: 10 required_substrings: - pass @@ -43,9 +45,10 @@ severity: medium type: pattern values: - - (^|(?P(?i:\bip[\s/]+id[\s/]+pw[\s/:]*))|(?P://)|\s)(?P[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})((?P\s*\()?\s*|(?(variable)[\s,/]+|\s*(?(url)[,]|[,/])\s*))[\w.-]{3,}[\s,/]+(?P(?(lpar)[^)\s/]{4,}|(?(url)[^\s/]{4,}|[^\s]{4,}))) + - (^|(?P(?i:\bip[\s/]+id[\s/]+pw[\s/:]*))|(?P://)|\s)(?P[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2}\.[0-2]?[0-9]{1,2})((?P\s*(\w+\s+)?\()?\s*|(?(variable)[\s,/]+|\s*(?(url)[,]|[,/]))\s*)[\w.-]{3,}[\s,/]+(?P(?(lpar)[^)\s]{4,}|[^\s/]{4,}))(?:\s|[^/]|$) filter_type: - ValueAllowlistCheck + - ValuePatternCheck min_line_len: 10 required_substrings: - "." @@ -59,6 +62,7 @@ - (?P--)?(?P(?i:user\s*)?(?i:id|login|account|root|admin|user|name|wifi|role|host|default|계정|아이디))\s*?(?(ddash)[ =]|[ :=])\s*?(?P\S+) filter_type: - ValueAllowlistCheck + - ValuePatternCheck min_line_len: 10 required_substrings: - pass @@ -79,6 +83,7 @@ filter_type: - ValueAllowlistCheck - ValueDictionaryKeywordCheck + - ValuePatternCheck min_line_len: 10 required_substrings: - pw @@ -194,7 +199,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?P(ABIA|ACCA|AGPA|AIDA|AIPA|AKIA|ANPA|ANVA|AROA|APKA|ASCA|ASIA)[0-9A-Z]{16,17})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: true required_substrings: - A min_line_len: 20 @@ -207,7 +211,6 @@ - (^|[^.0-9A-Za-z_/+-])(?P(AKIA|ASIA)[0-9A-Z]{16,17})([^=0-9A-Za-z_/+-]|$) - (?P[0-9a-zA-Z/+]{40}) filter_type: GeneralPattern - use_ml: true required_substrings: - AKIA - ASIA @@ -219,7 +222,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?Pamzn\.mws\.[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: true required_substrings: - amzn min_line_len: 30 @@ -242,7 +244,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?Pdt0[a-zA-Z]{1}[0-9]{2}\.[A-Z0-9]{24}\.[A-Z0-9]{64})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: true required_substrings: - dt0 min_line_len: 90 @@ -253,7 +254,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?PEAAC[0-9A-Za-z]{27,}) filter_type: GeneralPattern - use_ml: true required_substrings: - EAAC min_line_len: 31 @@ -277,7 +277,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?PAIza[0-9A-Za-z_-]{35})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: false validations: - GoogleApiKeyValidation required_substrings: @@ -291,7 +290,6 @@ - (?P[0-9]+\-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com) - (?[0-9a-zA-Z_-]{24})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: false validations: - GoogleMultiValidation required_substrings: @@ -304,7 +302,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?Pya29\.[0-9A-Za-z_-]{22,}) filter_type: GeneralPattern - use_ml: true required_substrings: - ya29. min_line_len: 27 @@ -315,7 +312,6 @@ values: - (?i)(?Pheroku(.{0,20})?[0-9a-f]{8}(-[0-9a-f]{4})+-[0-9a-f]{12})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: true required_substrings: - heroku min_line_len: 24 @@ -326,7 +322,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?PIGQVJ[\w]{100,}) filter_type: GeneralPattern - use_ml: true required_substrings: - IGQVJ min_line_len: 105 @@ -348,7 +343,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?P[0-9a-zA-Z]{32}-us[0-9]{1,2})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: false validations: - MailChimpKeyValidation required_substrings: @@ -361,7 +355,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?Pkey-[0-9a-zA-Z]{32})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: true required_substrings: - key- min_line_len: 36 @@ -385,7 +378,6 @@ values: - (?Paccess_token\$production\$[0-9a-z]{16}\$[0-9a-z]{32})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: false required_substrings: - access_token$production$ min_line_len: 72 @@ -405,7 +397,6 @@ values: - (?Psk_live_[0-9a-z]{32})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: false required_substrings: - sk_live_ min_line_len: 40 @@ -428,7 +419,6 @@ values: - (?PSG\.[\w_]{16,32}\.[\w_]{16,64}) filter_type: GeneralPattern - use_ml: false required_substrings: - SG. min_line_len: 34 @@ -449,7 +439,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?Pxox[a|b|p|r|o|s]\-[-a-zA-Z0-9]{10,250}) filter_type: GeneralPattern - use_ml: true validations: - SlackTokenValidation required_substrings: @@ -462,7 +451,6 @@ values: - (?Phooks\.slack\.com/services/T\w{8}/B\w{8}/\w{24}) filter_type: GeneralPattern - use_ml: true required_substrings: - hooks.slack.com/services/T min_line_len: 61 @@ -473,7 +461,6 @@ values: - (?Psk_live_[0-9a-zA-Z]{24})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: true validations: - StripeApiKeyValidation required_substrings: @@ -486,7 +473,6 @@ values: - (?Prk_live_[0-9a-zA-Z]{24})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: true required_substrings: - rk_live_ min_line_len: 32 @@ -497,7 +483,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?PEAAA[0-9A-Za-z_-]{60})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: true validations: - SquareAccessTokenValidation required_substrings: @@ -510,7 +495,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?Psq0[a-z]{3}-[0-9A-Za-z_-]{22})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: true validations: - SquareClientIdValidation required_substrings: @@ -523,7 +507,6 @@ values: - (?Psq0csp-[0-9A-Za-z_-]{43})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: false required_substrings: - sq0csp min_line_len: 50 @@ -546,7 +529,6 @@ values: - (^|[^.0-9A-Za-z_/+-])(?PSK[0-9a-fA-F]{32})([^=0-9A-Za-z_/+-]|$) filter_type: GeneralPattern - use_ml: true required_substrings: - SK min_line_len: 34 diff --git a/tests/__init__.py b/tests/__init__.py index ffc9dd122..7bdebc357 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,10 +8,10 @@ SAMPLES_CRED_LINE_COUNT: int = 402 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 293 +SAMPLES_POST_CRED_COUNT: int = 296 # with option --doc -SAMPLES_IN_DOC = 431 +SAMPLES_IN_DOC = 427 # archived credentials that are not found without --depth SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 21 diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index 44d55fd7b..880f7a216 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -97,8 +97,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -121,8 +121,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -145,8 +145,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -184,8 +184,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -208,8 +208,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -247,8 +247,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91871, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS MWS Key", "severity": "high", "line_data_list": [ @@ -271,8 +271,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91871, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Key", "severity": "medium", "line_data_list": [ @@ -2341,13 +2341,13 @@ "severity": "medium", "line_data_list": [ { - "line": "username:xxx pw:ihqSb1Gg", + "line": "username:master pw:ihqSb1Gg", "line_num": 90, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "tests/samples/doc_id_pair_passwd_pair|RAW", "value": "ihqSb1Gg", - "value_start": 16, - "value_end": 24, + "value_start": 19, + "value_end": 27, "variable": "pw", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -4831,8 +4831,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99108, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Dynatrace API Token", "severity": "high", "line_data_list": [ @@ -4903,8 +4903,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83427, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Facebook Access Token", "severity": "high", "line_data_list": [ @@ -4927,8 +4927,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83427, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Token", "severity": "medium", "line_data_list": [ @@ -5326,8 +5326,80 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99757, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Auth", + "severity": "medium", + "line_data_list": [ + { + "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line_num": 1, + "path": "tests/samples/google_oauth_key", + "info": "tests/samples/google_oauth_key|RAW", + "value": "ya29.gi_reo_gi_crackle_ln22", + "value_start": 20, + "value_end": 47, + "variable": "google_oauth_key", + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.1797273164975133, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Google OAuth Access Token", + "severity": "high", + "line_data_list": [ + { + "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line_num": 1, + "path": "tests/samples/google_oauth_key", + "info": "tests/samples/google_oauth_key|RAW", + "value": "ya29.gi_reo_gi_crackle_ln22", + "value_start": 20, + "value_end": 47, + "variable": null, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.1797273164975133, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Key", + "severity": "medium", + "line_data_list": [ + { + "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line_num": 1, + "path": "tests/samples/google_oauth_key", + "info": "tests/samples/google_oauth_key|RAW", + "value": "ya29.gi_reo_gi_crackle_ln22", + "value_start": 20, + "value_end": 47, + "variable": "google_oauth_key", + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.1797273164975133, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Google OAuth Access Token", "severity": "high", "line_data_list": [ @@ -5422,8 +5494,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95517, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Heroku API Key", "severity": "high", "line_data_list": [ @@ -5470,8 +5542,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71488, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Instagram Access Token", "severity": "high", "line_data_list": [ @@ -5902,8 +5974,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99189, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "MailGun API Key", "severity": "high", "line_data_list": [ @@ -5926,8 +5998,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -5950,8 +6022,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -6013,8 +6085,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -6037,8 +6109,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -6148,8 +6220,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99994, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -6172,8 +6244,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99994, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Password", "severity": "medium", "line_data_list": [ @@ -8218,8 +8290,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99994, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -8386,8 +8458,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89421, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Slack Token", "severity": "high", "line_data_list": [ @@ -8410,8 +8482,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6364, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Slack Webhook", "severity": "high", "line_data_list": [ @@ -8458,8 +8530,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.7944, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Square Access Token", "severity": "high", "line_data_list": [ @@ -8482,8 +8554,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.75821, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Square Client ID", "severity": "medium", "line_data_list": [ @@ -8890,8 +8962,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85074, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Google OAuth Access Token", "severity": "high", "line_data_list": [ @@ -8938,8 +9010,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85074, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Google OAuth Access Token", "severity": "high", "line_data_list": [ @@ -8986,8 +9058,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76194, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Facebook Access Token", "severity": "high", "line_data_list": [ @@ -9010,8 +9082,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76194, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Facebook Access Token", "severity": "high", "line_data_list": [ @@ -9058,8 +9130,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6423, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Twilio API Key", "severity": "high", "line_data_list": [ diff --git a/tests/data/doc.json b/tests/data/doc.json index f19ee0566..e88d39e40 100644 --- a/tests/data/doc.json +++ b/tests/data/doc.json @@ -184,8 +184,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -208,8 +208,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -5155,13 +5155,13 @@ "severity": "medium", "line_data_list": [ { - "line": "username:xxx pw:ihqSb1Gg", + "line": "username:master pw:ihqSb1Gg", "line_num": 90, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "tests/samples/doc_id_pair_passwd_pair|RAW", "value": "ihqSb1Gg", - "value_start": 16, - "value_end": 24, + "value_start": 19, + "value_end": 27, "variable": "pw", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -5170,17 +5170,17 @@ } }, { - "line": "username:xxx pw:ihqSb1Gg", + "line": "username:master pw:ihqSb1Gg", "line_num": 90, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "xxx", + "value": "master", "value_start": 9, - "value_end": 12, + "value_end": 15, "variable": "username", "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 0.0, + "iterator": "BASE64_CHARS", + "entropy": 2.584962500721156, "valid": false } } @@ -5194,13 +5194,13 @@ "severity": "medium", "line_data_list": [ { - "line": "username:xxx pw:ihqSb1Gg", + "line": "username:master pw:ihqSb1Gg", "line_num": 90, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "tests/samples/doc_id_pair_passwd_pair|RAW", "value": "ihqSb1Gg", - "value_start": 16, - "value_end": 24, + "value_start": 19, + "value_end": 27, "variable": "pw", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -8186,13 +8186,13 @@ "line_num": 14, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "(master/IhqSb1Gg)", - "value_start": 17, - "value_end": 34, + "value": "IhqSb1Gg", + "value_start": 25, + "value_end": 33, "variable": null, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.606584859926771, + "entropy": 3.0, "valid": false } } @@ -8750,13 +8750,13 @@ "line_num": 48, "path": "tests/samples/doc_various", "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg,master", + "value": "IhqSb1Gg,master/IhqSb1Gg", "value_start": 19, - "value_end": 34, + "value_end": 43, "variable": null, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.64643122256795, + "entropy": 3.727255729857775, "valid": false } } @@ -8834,45 +8834,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "ID_PAIR_PASSWD_PAIR", - "severity": "medium", - "line_data_list": [ - { - "line": "ANY_user:xxxx ANY_pwd:IhqSb1Gg", - "line_num": 61, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 22, - "value_end": 30, - "variable": "ANY_pwd", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - }, - { - "line": "ANY_user:xxxx ANY_pwd:IhqSb1Gg", - "line_num": 61, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "xxxx", - "value_start": 9, - "value_end": 13, - "variable": "user", - "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 0.0, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -8897,45 +8858,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "ID_PAIR_PASSWD_PAIR", - "severity": "medium", - "line_data_list": [ - { - "line": "Acount name:xxxx Initial Password:IhqSb1Gg", - "line_num": 62, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 34, - "value_end": 42, - "variable": "Password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - }, - { - "line": "Acount name:xxxx Initial Password:IhqSb1Gg", - "line_num": 62, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "xxxx", - "value_start": 12, - "value_end": 16, - "variable": "name", - "entropy_validation": { - "iterator": "HEX_CHARS", - "entropy": 0.0, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -9581,45 +9503,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "ID_PAIR_PASSWD_PAIR", - "severity": "medium", - "line_data_list": [ - { - "line": "ID:gildong.hong@xxxx.net mailto:{1} pw:IhqSb1Gg", - "line_num": 106, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "IhqSb1Gg", - "value_start": 39, - "value_end": 47, - "variable": "pw", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, - "valid": false - } - }, - { - "line": "ID:gildong.hong@xxxx.net mailto:{1} pw:IhqSb1Gg", - "line_num": 106, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "gildong.hong@xxxx.net", - "value_start": 3, - "value_end": 24, - "variable": "ID", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.8358066002709883, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -9742,8 +9625,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99108, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Dynatrace API Token", "severity": "high", "line_data_list": [ @@ -10237,8 +10120,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99757, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Google OAuth Access Token", "severity": "high", "line_data_list": [ @@ -10333,8 +10216,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95517, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Heroku API Key", "severity": "high", "line_data_list": [ @@ -10357,8 +10240,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71488, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Instagram Access Token", "severity": "high", "line_data_list": [ @@ -10525,8 +10408,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99189, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "MailGun API Key", "severity": "high", "line_data_list": [ @@ -10549,8 +10432,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -10573,8 +10456,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -10636,8 +10519,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -10660,8 +10543,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -10793,30 +10676,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "PASSWD_PAIR", - "severity": "medium", - "line_data_list": [ - { - "line": "mypw: KrAcMe12345,", - "line_num": 2, - "path": "tests/samples/passwd.groovy", - "info": "tests/samples/passwd.groovy|RAW", - "value": "KrAcMe12345,", - "value_start": 6, - "value_end": 18, - "variable": "pw", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.2862156256610597, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -11533,8 +11392,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89421, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Slack Token", "severity": "high", "line_data_list": [ @@ -11557,8 +11416,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6364, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Slack Webhook", "severity": "high", "line_data_list": [ @@ -11605,8 +11464,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.7944, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Square Access Token", "severity": "high", "line_data_list": [ @@ -11629,8 +11488,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.75821, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Square Client ID", "severity": "medium", "line_data_list": [ @@ -11893,8 +11752,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85074, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Google OAuth Access Token", "severity": "high", "line_data_list": [ @@ -11941,8 +11800,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76194, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Facebook Access Token", "severity": "high", "line_data_list": [ @@ -11989,8 +11848,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6423, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Twilio API Key", "severity": "high", "line_data_list": [ diff --git a/tests/data/ml_threshold_0.json b/tests/data/ml_threshold_0.json index 9a28ec61e..9d04198a4 100644 --- a/tests/data/ml_threshold_0.json +++ b/tests/data/ml_threshold_0.json @@ -2437,13 +2437,13 @@ "severity": "medium", "line_data_list": [ { - "line": "username:xxx pw:ihqSb1Gg", + "line": "username:master pw:ihqSb1Gg", "line_num": 90, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", "value": "ihqSb1Gg", - "value_start": 16, - "value_end": 24, + "value_start": 19, + "value_end": 27, "variable": "pw", "entropy_validation": { "iterator": "BASE64_CHARS", diff --git a/tests/data/output.json b/tests/data/output.json index d2e2f7d19..565999a64 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -97,8 +97,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -121,8 +121,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -145,8 +145,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -184,8 +184,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -208,8 +208,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -247,8 +247,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91871, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS MWS Key", "severity": "high", "line_data_list": [ @@ -271,8 +271,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91871, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Key", "severity": "medium", "line_data_list": [ @@ -2317,13 +2317,13 @@ "severity": "medium", "line_data_list": [ { - "line": "username:xxx pw:ihqSb1Gg", + "line": "username:master pw:ihqSb1Gg", "line_num": 90, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", "value": "ihqSb1Gg", - "value_start": 16, - "value_end": 24, + "value_start": 19, + "value_end": 27, "variable": "pw", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -4807,8 +4807,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99108, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Dynatrace API Token", "severity": "high", "line_data_list": [ @@ -4831,8 +4831,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83427, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Facebook Access Token", "severity": "high", "line_data_list": [ @@ -4855,8 +4855,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83427, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Token", "severity": "medium", "line_data_list": [ @@ -5254,8 +5254,80 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99757, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Auth", + "severity": "medium", + "line_data_list": [ + { + "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line_num": 1, + "path": "tests/samples/google_oauth_key", + "info": "", + "value": "ya29.gi_reo_gi_crackle_ln22", + "value_start": 20, + "value_end": 47, + "variable": "google_oauth_key", + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.1797273164975133, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Google OAuth Access Token", + "severity": "high", + "line_data_list": [ + { + "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line_num": 1, + "path": "tests/samples/google_oauth_key", + "info": "", + "value": "ya29.gi_reo_gi_crackle_ln22", + "value_start": 20, + "value_end": 47, + "variable": null, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.1797273164975133, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Key", + "severity": "medium", + "line_data_list": [ + { + "line": "google_oauth_key = \"ya29.gi_reo_gi_crackle_ln22\"", + "line_num": 1, + "path": "tests/samples/google_oauth_key", + "info": "", + "value": "ya29.gi_reo_gi_crackle_ln22", + "value_start": 20, + "value_end": 47, + "variable": "google_oauth_key", + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.1797273164975133, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Google OAuth Access Token", "severity": "high", "line_data_list": [ @@ -5350,8 +5422,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95517, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Heroku API Key", "severity": "high", "line_data_list": [ @@ -5398,8 +5470,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71488, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Instagram Access Token", "severity": "high", "line_data_list": [ @@ -5782,8 +5854,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99189, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "MailGun API Key", "severity": "high", "line_data_list": [ @@ -5806,8 +5878,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -5830,8 +5902,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -5893,8 +5965,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Client ID", "severity": "high", "line_data_list": [ @@ -5917,8 +5989,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99998, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "AWS Multi", "severity": "high", "line_data_list": [ @@ -6910,8 +6982,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89421, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Slack Token", "severity": "high", "line_data_list": [ @@ -6934,8 +7006,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6364, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Slack Webhook", "severity": "high", "line_data_list": [ @@ -6958,8 +7030,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.7944, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Square Access Token", "severity": "high", "line_data_list": [ @@ -6982,8 +7054,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.75821, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Square Client ID", "severity": "medium", "line_data_list": [ @@ -7198,8 +7270,8 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6423, + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, "rule": "Twilio API Key", "severity": "high", "line_data_list": [ diff --git a/tests/filters/test_value_allowlist_check.py b/tests/filters/test_value_allowlist_check.py index 9c1374c27..9e50eecf2 100644 --- a/tests/filters/test_value_allowlist_check.py +++ b/tests/filters/test_value_allowlist_check.py @@ -11,9 +11,7 @@ def test_value_allowlist_check_p(self, file_path: pytest.fixture, success_line: line_data = get_line_data(file_path, line=success_line, pattern=LINE_VALUE_PATTERN) assert ValueAllowlistCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is False - @pytest.mark.parametrize("line", [ - "ENC(Crackle123)", - ]) + @pytest.mark.parametrize("line", ["test*****", "ENC(Crackle123)"]) def test_value_allowlist_check_n(self, file_path: pytest.fixture, line: str) -> None: line_data = get_line_data(file_path, line=line, pattern=LINE_VALUE_PATTERN) assert ValueAllowlistCheck().run(line_data, DUMMY_ANALYSIS_TARGET) is True diff --git a/tests/samples/doc_id_pair_passwd_pair b/tests/samples/doc_id_pair_passwd_pair index 10989cef6..2a2b441ad 100644 --- a/tests/samples/doc_id_pair_passwd_pair +++ b/tests/samples/doc_id_pair_passwd_pair @@ -87,7 +87,7 @@ id/pw id:master pw:ihqSb1Gg user:master,pwd:ihqSb1Gg ID master PWD ihqSb1Gg username:master/pw:master -username:xxx pw:ihqSb1Gg +username:master pw:ihqSb1Gg PW:ihqSb1Gg ID:master 아이디:master 비밀번호:ihqSb1Gg ID:master,PW ihqSb1Gg diff --git a/tests/samples/doc_ip_id_password_triple b/tests/samples/doc_ip_id_password_triple index 30ed935d7..08cffcf2f 100644 --- a/tests/samples/doc_ip_id_password_triple +++ b/tests/samples/doc_ip_id_password_triple @@ -4,3 +4,8 @@ IP ID PW 192.168.0.1 master IhqSb1Gg IP/ID/PW 192.168.0.1/master/IhqSb1Gg + +# FALSE +

sirius 192.168.0.1/mnt/user/USBx/\\[Root_Test\\]UseCase/12345678/76125-733_FFB_2038_judgement_day6_238811_6524.apk

+http://192.168.0.1/master/IhqSb1Gg/api +http://192.168.0.1/master/branch diff --git a/tests/test_app.py b/tests/test_app.py index b5dfdc044..03e58231a 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -129,7 +129,7 @@ def test_it_works_with_multiline_in_patch_p(self) -> None: / value: 'AKIAQWADE5R42RDZ4JEM' / entropy_validation: BASE64_CHARS 3.684184 False] / api_validation: NOT_AVAILABLE - / ml_validation: VALIDATED_KEY + / ml_validation: NOT_AVAILABLE rule: AWS Multi / severity: high / line_data_list: @@ -144,7 +144,7 @@ def test_it_works_with_multiline_in_patch_p(self) -> None: / value: 'V84C7sDU001tFFodKU95USNy97TkqXymnvsFmYhQ' / entropy_validation: BASE64_CHARS 4.784184 True] / api_validation: NOT_AVAILABLE - / ml_validation: VALIDATED_KEY + / ml_validation: NOT_AVAILABLE rule: Token / severity: medium / line_data_list: diff --git a/tests/test_main.py b/tests/test_main.py index 6ac4b999a..22412c89a 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -824,7 +824,9 @@ def test_param_p(self) -> None: # internal parametrized tests to keep items = [(" STP_PASSWORD=qbgomdtpqch \\", "qbgomdtpqch")] for i in items: - content_provider: FilesProvider = TextProvider(["test.template", io.BytesIO(i[0].encode())]) + content_provider: FilesProvider = TextProvider([ + ("test.template", io.BytesIO(i[0].encode())), + ]) cred_sweeper = CredSweeper(ml_threshold=0) cred_sweeper.run(content_provider=content_provider) creds = cred_sweeper.credential_manager.get_credentials()