diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6c343e7b2..0cfa1cf9b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -58,7 +58,7 @@ jobs: - name: Check ml_model.onnx integrity if: ${{ always() && steps.code_checkout.conclusion == 'success' }} run: | - md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 70a864232576f9b88a08296a5e628208 + md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 62d92ab2f91a18e861d846a7b8a0c3a7 # # # Python setup diff --git a/credsweeper/ml_model/ml_model.onnx b/credsweeper/ml_model/ml_model.onnx index ca90ae43a..76804356c 100644 Binary files a/credsweeper/ml_model/ml_model.onnx and b/credsweeper/ml_model/ml_model.onnx differ diff --git a/credsweeper/ml_model/model_config.json b/credsweeper/ml_model/model_config.json index 5de603ec5..d9e214688 100644 --- a/credsweeper/ml_model/model_config.json +++ b/credsweeper/ml_model/model_config.json @@ -361,7 +361,6 @@ ".adoc", ".asciidoc", ".axaml", - ".backup", ".bash", ".bat", ".bats", @@ -374,7 +373,6 @@ ".cjs", ".cljc", ".cmd", - ".cmm", ".cnf", ".coffee", ".conf", @@ -396,7 +394,6 @@ ".example", ".exs", ".ext", - ".fsproj", ".g4", ".gml", ".gni", @@ -431,7 +428,7 @@ ".log", ".lua", ".m", - ".manifest", + ".map", ".markerb", ".md", ".mdx", @@ -440,7 +437,6 @@ ".ml", ".mlir", ".moo", - ".msg", ".ndjson", ".nix", ".nolint", @@ -478,7 +474,6 @@ ".scala", ".secrets", ".sh", - ".sln", ".snap", ".sql", ".storyboard", @@ -489,7 +484,6 @@ ".tdf", ".template", ".test", - ".testsettings", ".tf", ".tfstate", ".tfvars", @@ -500,9 +494,7 @@ ".travis", ".ts", ".tsx", - ".ttar", ".txt", - ".vsmdi", ".vue", ".xaml", ".xib", @@ -530,8 +522,7 @@ "Salt", "Secret", "Token", - "URL Credentials", - "UUID" + "URL Credentials" ] } } diff --git a/tests/__init__.py b/tests/__init__.py index 792fa3d01..b9485a52d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,18 +7,18 @@ NEGLIGIBLE_ML_THRESHOLD = 0.0001 # credentials count after scan -SAMPLES_CRED_COUNT: int = 360 -SAMPLES_CRED_LINE_COUNT: int = 377 +SAMPLES_CRED_COUNT: int = 363 +SAMPLES_CRED_LINE_COUNT: int = 380 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 332 +SAMPLES_POST_CRED_COUNT: int = 321 # with option --doc -SAMPLES_IN_DOC = 416 +SAMPLES_IN_DOC = 415 # archived credentials that are not found without --depth SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 24 -SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 16 +SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 17 SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 1 # well known string with all latin letters diff --git a/tests/data/__init__.py b/tests/data/__init__.py index 150772923..56f2ddcd2 100644 --- a/tests/data/__init__.py +++ b/tests/data/__init__.py @@ -11,14 +11,14 @@ "sort_output": True, "json_filename": "ml_threshold.json", "ml_threshold": NEGLIGIBLE_ML_THRESHOLD -}, { - "__cred_count": SAMPLES_IN_DEEP_3, - "sort_output": True, - "json_filename": "depth_3.json", - "depth": 3 }, { "__cred_count": SAMPLES_IN_DOC, "sort_output": True, "json_filename": "doc.json", "doc": True +}, { + "__cred_count": SAMPLES_IN_DEEP_3, + "sort_output": True, + "json_filename": "depth_3.json", + "depth": 3 }] diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index 4d1705590..d4d44afa9 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -124,7 +124,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.999, "rule": "API", "severity": "medium", "confidence": "moderate", @@ -286,7 +286,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.962, + "ml_probability": 0.87, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -313,7 +313,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -340,7 +340,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.991, + "ml_probability": 0.99, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -367,7 +367,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -394,7 +394,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.963, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -543,6 +543,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.924, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "\"AwsAccessKey\": \"AKIAGIREOGIAWSKEY123\", \"AwsAccessKey2\": \"AKIAGIREOGIAWSKEY555\",", + "line_num": 2, + "path": "tests/samples/aws_multi.groovy", + "info": "tests/samples/aws_multi.groovy|RAW", + "value": "AKIAGIREOGIAWSKEY123", + "value_start": 17, + "value_end": 37, + "variable": "AwsAccessKey", + "variable_start": 1, + "variable_end": 13, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.5464393446710156, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -614,6 +641,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.922, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "\"AwsAccessKey\": \"AKIAGIREOGIAWSKEY123\", \"AwsAccessKey2\": \"AKIAGIREOGIAWSKEY555\",", + "line_num": 2, + "path": "tests/samples/aws_multi.groovy", + "info": "tests/samples/aws_multi.groovy|RAW", + "value": "AKIAGIREOGIAWSKEY555", + "value_start": 58, + "value_end": 78, + "variable": "AwsAccessKey2", + "variable_start": 41, + "variable_end": 54, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.3086949695628425, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -644,7 +698,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.994, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -995,7 +1049,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.966, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -1022,7 +1076,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Certificate", "severity": "medium", "confidence": "moderate", @@ -1130,7 +1184,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.996, "rule": "Credential", "severity": "medium", "confidence": "moderate", @@ -1238,7 +1292,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.775, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1265,61 +1319,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.918, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ID:master PW:dipPr11Gg!", - "line_num": 2, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr11Gg!", - "value_start": 13, - "value_end": 23, - "variable": "PW", - "variable_start": 10, - "variable_end": 12, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.7897352853986264, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ANYID:master PW:dipPr12Gg!", - "line_num": 3, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr12Gg!", - "value_start": 16, - "value_end": 26, - "variable": "PW", - "variable_start": 13, - "variable_end": 15, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.989735285398626, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.926, + "ml_probability": 0.881, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1346,7 +1346,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.977, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1373,7 +1373,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.786, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1400,7 +1400,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1427,7 +1427,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1454,7 +1454,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1481,7 +1481,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.957, + "ml_probability": 0.812, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1508,7 +1508,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.886, + "ml_probability": 0.941, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1535,34 +1535,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.896, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ID:master PWD:dipPr112Gg!", - "line_num": 13, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr112Gg!", - "value_start": 14, - "value_end": 25, - "variable": "PWD", - "variable_start": 10, - "variable_end": 13, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.9631196533066344, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.984, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1589,7 +1562,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.985, + "ml_probability": 0.975, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1616,7 +1589,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.969, + "ml_probability": 0.963, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1643,7 +1616,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, + "ml_probability": 0.97, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1670,7 +1643,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.976, + "ml_probability": 0.972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1697,7 +1670,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.946, + "ml_probability": 0.965, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1724,7 +1697,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.959, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1751,7 +1724,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.974, + "ml_probability": 0.926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1778,7 +1751,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.954, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1805,7 +1778,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.968, + "ml_probability": 0.735, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1832,7 +1805,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.909, + "ml_probability": 0.928, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1859,7 +1832,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.974, + "ml_probability": 0.893, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1886,7 +1859,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.791, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1913,7 +1886,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.915, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1940,7 +1913,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.941, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1967,7 +1940,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.921, + "ml_probability": 0.639, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1994,7 +1967,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.945, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2021,7 +1994,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.945, + "ml_probability": 0.85, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2048,7 +2021,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.893, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2075,7 +2048,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.811, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2102,7 +2075,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, + "ml_probability": 0.973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2129,7 +2102,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.991, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2156,7 +2129,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2183,7 +2156,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.949, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2210,7 +2183,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2237,7 +2210,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.99, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2264,7 +2237,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.991, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2291,7 +2264,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.752, + "ml_probability": 0.849, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2318,7 +2291,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2345,7 +2318,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.984, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2372,7 +2345,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.991, + "ml_probability": 0.977, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2399,7 +2372,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.928, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2426,34 +2399,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "(98.76.54.32)ID:master PW:dipPr149Gg!", - "line_num": 50, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr149Gg!", - "value_start": 26, - "value_end": 37, - "variable": "PW", - "variable_start": 23, - "variable_end": 25, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.1449378351248165, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.968, + "ml_probability": 0.736, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2480,7 +2426,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.937, + "ml_probability": 0.629, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2507,7 +2453,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.984, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2534,7 +2480,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2561,7 +2507,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2588,7 +2534,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2615,7 +2561,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.986, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2642,7 +2588,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2669,7 +2615,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.987, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2696,7 +2642,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.953, + "ml_probability": 0.976, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2723,7 +2669,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.865, + "ml_probability": 0.652, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2750,34 +2696,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.944, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "dipPr163Gg! ID:master dipPr163Gg! PWD:dipPr163Gg!", - "line_num": 64, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr163Gg!", - "value_start": 38, - "value_end": 49, - "variable": "PWD", - "variable_start": 34, - "variable_end": 37, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.1449378351248165, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2804,7 +2723,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2831,7 +2750,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.932, + "ml_probability": 0.855, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2858,7 +2777,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.991, + "ml_probability": 0.925, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2885,7 +2804,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.921, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2912,7 +2831,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.944, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2939,7 +2858,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2966,7 +2885,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2993,7 +2912,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.993, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3020,7 +2939,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3047,7 +2966,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3074,7 +2993,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.993, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3101,7 +3020,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.975, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3128,7 +3047,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.885, + "ml_probability": 0.759, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3155,34 +3074,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.943, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "id:master,default pw:dipPr182Gg!", - "line_num": 83, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr182Gg!", - "value_start": 21, - "value_end": 32, - "variable": "pw", - "variable_start": 18, - "variable_end": 20, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.1449378351248165, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.989, + "ml_probability": 0.766, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3209,7 +3101,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.971, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3236,7 +3128,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.839, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3263,7 +3155,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.871, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3290,34 +3182,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "PW:dipPr190Gg! ID:master", - "line_num": 91, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr190Gg!", - "value_start": 3, - "value_end": 14, - "variable": "PW", - "variable_start": 0, - "variable_end": 2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.1449378351248165, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.904, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3344,7 +3209,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3371,7 +3236,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.921, + "ml_probability": 0.968, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3398,7 +3263,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.866, + "ml_probability": 0.921, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3425,34 +3290,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.954, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "98.76.54.32(ID:master/PW:iPp10@GRq) # todo: move into other sample ?", - "line_num": 11, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "iPp10@GRq)", - "value_start": 25, - "value_end": 35, - "variable": "PW", - "variable_start": 22, - "variable_end": 24, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.65754247590989, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.937, + "ml_probability": 0.745, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3479,7 +3317,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.842, + "ml_probability": 0.874, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3506,7 +3344,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.959, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3533,7 +3371,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.951, + "ml_probability": 0.847, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3560,7 +3398,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.964, + "ml_probability": 0.939, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3587,7 +3425,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.968, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3614,7 +3452,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3641,7 +3479,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98, + "ml_probability": 0.967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3668,7 +3506,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.984, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3695,7 +3533,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.952, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3722,7 +3560,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.931, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3749,7 +3587,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.951, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3776,7 +3614,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3803,7 +3641,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.978, + "ml_probability": 0.883, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3830,34 +3668,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.759, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "PW:Prl23Db#@,password:Prl23Db#@", - "line_num": 24, - "path": "tests/samples/doc_passwd_pair", - "info": "tests/samples/doc_passwd_pair|RAW", - "value": "Prl23Db#@", - "value_start": 3, - "value_end": 12, - "variable": "PW", - "variable_start": 0, - "variable_end": 2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.4654972233440207, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, + "ml_probability": 0.946, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3884,7 +3695,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3911,7 +3722,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3938,7 +3749,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.939, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3965,7 +3776,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3992,7 +3803,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.959, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4019,7 +3830,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, + "ml_probability": 0.886, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4046,7 +3857,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.965, + "ml_probability": 0.67, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4073,7 +3884,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4100,7 +3911,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.951, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4127,7 +3938,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.989, + "ml_probability": 0.822, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4154,7 +3965,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.767, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4181,7 +3992,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.97, + "ml_probability": 0.916, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4208,7 +4019,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.992, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4235,7 +4046,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.992, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4262,52 +4073,25 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password=>Prl23Db#@", - "line_num": 48, - "path": "tests/samples/doc_passwd_pair", - "info": "tests/samples/doc_passwd_pair|RAW", - "value": "Prl23Db#@", - "value_start": 10, - "value_end": 19, - "variable": "password", - "variable_start": 0, - "variable_end": 8, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.4654972233440207, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.82, + "ml_probability": 0.991, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "# password: keep empty", - "line_num": 56, + "line": "password=>Prl23Db#@", + "line_num": 48, "path": "tests/samples/doc_passwd_pair", "info": "tests/samples/doc_passwd_pair|RAW", - "value": "keep", - "value_start": 12, - "value_end": 16, + "value": "Prl23Db#@", + "value_start": 10, + "value_end": 19, "variable": "password", - "variable_start": 2, - "variable_end": 10, + "variable_start": 0, + "variable_end": 8, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 1.5, + "entropy": 2.4654972233440207, "valid": false } } @@ -4343,7 +4127,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4586,7 +4370,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4667,7 +4451,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4748,7 +4532,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.994, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4775,7 +4559,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4802,7 +4586,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4883,7 +4667,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.995, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -4910,7 +4694,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.995, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4937,7 +4721,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.958, + "ml_probability": 0.956, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4964,7 +4748,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.896, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4991,7 +4775,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.964, + "ml_probability": 0.943, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5018,7 +4802,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.967, + "ml_probability": 0.972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5099,7 +4883,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5126,7 +4910,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.94, + "ml_probability": 0.933, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5153,7 +4937,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.979, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5180,7 +4964,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5207,7 +4991,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5234,7 +5018,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.966, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5261,7 +5045,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5288,7 +5072,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.822, + "ml_probability": 0.899, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5315,7 +5099,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.981, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5369,7 +5153,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5396,7 +5180,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5423,7 +5207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.758, + "ml_probability": 0.894, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -5450,7 +5234,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.758, + "ml_probability": 0.894, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -5477,7 +5261,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5504,7 +5288,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.967, + "ml_probability": 0.938, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5531,7 +5315,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.936, + "ml_probability": 0.992, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5558,7 +5342,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5585,7 +5369,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.984, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5612,7 +5396,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.94, + "ml_probability": 0.896, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5639,7 +5423,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.929, + "ml_probability": 0.949, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5666,7 +5450,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5693,7 +5477,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5720,7 +5504,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.987, + "ml_probability": 0.953, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5747,7 +5531,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5801,34 +5585,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.865, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "(ID&PWD):master/IhqSb1Gg", - "line_num": 71, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "master/IhqSb1Gg", - "value_start": 9, - "value_end": 24, - "variable": "PWD)", - "variable_start": 4, - "variable_end": 8, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.906890595608518, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.958, + "ml_probability": 0.956, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5855,7 +5612,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.99, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5882,7 +5639,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.976, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5909,7 +5666,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5936,7 +5693,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.952, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5963,7 +5720,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.966, + "ml_probability": 0.943, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5990,7 +5747,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6017,7 +5774,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.995, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6044,7 +5801,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6071,7 +5828,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6098,7 +5855,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.987, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6125,7 +5882,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.995, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6152,7 +5909,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.881, + "ml_probability": 0.979, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6179,7 +5936,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6287,7 +6044,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.934, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6314,7 +6071,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.97, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6341,7 +6098,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.975, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6476,7 +6233,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.943, + "ml_probability": 0.756, "rule": "Github Old Token", "severity": "high", "confidence": "moderate", @@ -6503,7 +6260,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.943, + "ml_probability": 0.756, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -6557,7 +6314,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -7266,7 +7023,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7428,34 +7185,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.853, - "rule": "Key", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "prKeyValid=LS0tLS1CRUdJTiBQUklWQVRFIENDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ0ViVnpmUGWxhQW9KQWwrLzZYdDJPNG1PQjYxMXNPaFJBTkNBQVNnRlRLandKQUFVOTVnKysvdnpLV0hrekFWbU5NSQp0QjV2VGpaT09Jd25FYjcwTXNXWkZJeVVGRDFQOUd3c3R6NCtha0hYN3ZJOEJINmhIbUJtZmVRbAotLS0tLUVORCBQUklWJNR0J5cUdTTTQ5QW5aUHhmQXl4cUUKWlYwNdFR0QVRFIEtFWS0tLS0tCgtFWS0tLS0tCk1JR0hBZ0VBTU==", - "line_num": 1, - "path": "tests/samples/key.hs", - "info": "tests/samples/key.hs|RAW", - "value": "LS0tLS1CRUdJTiBQUklWQVRFIENDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ0ViVnpmUGWxhQW9KQWwrLzZYdDJPNG1PQjYxMXNPaFJBTkNBQVNnRlRLandKQUFVOTVnKysvdnpLV0hrekFWbU5NSQp0QjV2VGpaT09Jd25FYjcwTXNXWkZJeVVGRDFQOUd3c3R6NCtha0hYN3ZJOEJINmhIbUJtZmVRbAotLS0tLUVORCBQUklWJNR0J5cUdTTTQ5QW5aUHhmQXl4cUUKWlYwNdFR0QVRFIEtFWS0tLS0tCgtFWS0tLS0tCk1JR0hBZ0VBTU==", - "value_start": 11, - "value_end": 335, - "variable": "prKeyValid", - "variable_start": 0, - "variable_end": 10, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.428808109703668, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7482,7 +7212,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7509,7 +7239,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7536,7 +7266,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7590,7 +7320,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "API", "severity": "medium", "confidence": "moderate", @@ -7617,7 +7347,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -7769,7 +7499,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -7867,7 +7597,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -7894,7 +7624,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Nonce", "severity": "medium", "confidence": "moderate", @@ -8002,7 +7732,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.98, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8029,7 +7759,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8056,7 +7786,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.904, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8110,7 +7840,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8137,7 +7867,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8164,7 +7894,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.904, + "ml_probability": 0.818, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8191,7 +7921,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8218,7 +7948,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8245,7 +7975,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8272,7 +8002,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.963, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9626,7 +9356,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.963, + "ml_probability": 0.911, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9653,7 +9383,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.989, + "ml_probability": 0.887, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9680,7 +9410,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.992, + "ml_probability": 0.907, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9707,7 +9437,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9, + "ml_probability": 0.816, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9734,7 +9464,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.992, + "ml_probability": 0.665, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9761,7 +9491,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.906, + "ml_probability": 0.852, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9788,7 +9518,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.906, + "ml_probability": 0.852, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9866,6 +9596,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.719, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "AwsAccessKey:AKIAGIREOGIAWSKEY123", + "line_num": 1, + "path": "tests/samples/sample.pdf", + "info": "tests/samples/sample.pdf|PDF:1|ENCODED|RAW", + "value": "AKIAGIREOGIAWSKEY123", + "value_start": 13, + "value_end": 33, + "variable": "AwsAccessKey", + "variable_start": 0, + "variable_end": 12, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.5464393446710156, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -9923,7 +9680,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.995, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -10382,7 +10139,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -10409,7 +10166,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.966, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -10436,7 +10193,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.994, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -10517,7 +10274,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.991, + "ml_probability": 0.931, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -10544,7 +10301,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -10787,7 +10544,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 1.0, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -10868,7 +10625,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88, + "ml_probability": 0.804, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -10895,7 +10652,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.716, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -10922,7 +10679,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -10976,7 +10733,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.994, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -11030,7 +10787,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.989, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -11084,7 +10841,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.993, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -11111,7 +10868,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.971, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -11162,33 +10919,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "UUID", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", - "line_num": 1, - "path": "tests/samples/uuid", - "info": "tests/samples/uuid|RAW", - "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", - "value_start": 0, - "value_end": 36, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2373263071270246, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -11219,7 +10949,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.911, "rule": "Password", "severity": "medium", "confidence": "moderate", diff --git a/tests/data/doc.json b/tests/data/doc.json index 6c32cd1f7..46c8cdb7d 100644 --- a/tests/data/doc.json +++ b/tests/data/doc.json @@ -13064,33 +13064,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "UUID", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", - "line_num": 1, - "path": "tests/samples/uuid", - "info": "tests/samples/uuid|RAW", - "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", - "value_start": 0, - "value_end": 36, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2373263071270246, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json index fa80d0d45..66298f39c 100644 --- a/tests/data/ml_threshold.json +++ b/tests/data/ml_threshold.json @@ -29,7 +29,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.999, "rule": "API", "severity": "medium", "confidence": "moderate", @@ -191,7 +191,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.962, + "ml_probability": 0.87, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -218,7 +218,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -245,7 +245,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.991, + "ml_probability": 0.99, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -272,7 +272,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -299,7 +299,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.963, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -451,7 +451,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.13, + "ml_probability": 0.924, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -549,7 +549,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.4, + "ml_probability": 0.922, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -576,7 +576,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.034, + "ml_probability": 0.453, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -603,7 +603,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.034, + "ml_probability": 0.453, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -657,7 +657,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.994, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -981,7 +981,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.966, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -1008,7 +1008,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Certificate", "severity": "medium", "confidence": "moderate", @@ -1035,7 +1035,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.996, "rule": "Credential", "severity": "medium", "confidence": "moderate", @@ -1143,7 +1143,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.775, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1170,7 +1170,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.918, + "ml_probability": 0.402, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1197,7 +1197,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9, + "ml_probability": 0.327, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1224,7 +1224,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.926, + "ml_probability": 0.881, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1251,7 +1251,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.977, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1278,7 +1278,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.786, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1305,7 +1305,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1332,7 +1332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1359,7 +1359,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1386,7 +1386,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.957, + "ml_probability": 0.812, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1413,7 +1413,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.886, + "ml_probability": 0.941, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1440,7 +1440,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.896, + "ml_probability": 0.573, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1467,7 +1467,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.984, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1494,7 +1494,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.985, + "ml_probability": 0.975, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1521,7 +1521,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.969, + "ml_probability": 0.963, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1548,7 +1548,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, + "ml_probability": 0.97, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1575,7 +1575,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.976, + "ml_probability": 0.972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1602,7 +1602,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.946, + "ml_probability": 0.965, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1629,7 +1629,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.959, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1656,7 +1656,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.974, + "ml_probability": 0.926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1683,7 +1683,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.954, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1710,7 +1710,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.494, + "ml_probability": 0.558, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1737,7 +1737,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.968, + "ml_probability": 0.735, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1764,7 +1764,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.909, + "ml_probability": 0.928, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1791,7 +1791,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.974, + "ml_probability": 0.893, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1818,7 +1818,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.791, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1845,7 +1845,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.915, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1872,7 +1872,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.941, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1899,7 +1899,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.921, + "ml_probability": 0.639, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1926,7 +1926,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.945, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1953,7 +1953,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.945, + "ml_probability": 0.85, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1980,7 +1980,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.893, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2007,7 +2007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.811, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2034,7 +2034,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, + "ml_probability": 0.973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2061,7 +2061,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.991, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2088,7 +2088,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2115,7 +2115,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.949, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2142,7 +2142,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2169,7 +2169,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.99, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2196,7 +2196,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.991, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2223,7 +2223,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.752, + "ml_probability": 0.849, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2250,7 +2250,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2277,7 +2277,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.984, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2304,7 +2304,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.991, + "ml_probability": 0.977, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2331,7 +2331,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.928, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2358,7 +2358,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98, + "ml_probability": 0.472, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2385,7 +2385,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.968, + "ml_probability": 0.736, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2412,7 +2412,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.937, + "ml_probability": 0.629, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2439,7 +2439,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.984, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2466,7 +2466,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2493,7 +2493,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2520,7 +2520,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2547,7 +2547,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.986, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2574,7 +2574,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2601,7 +2601,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.987, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2628,7 +2628,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.953, + "ml_probability": 0.976, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2655,7 +2655,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.865, + "ml_probability": 0.652, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2682,7 +2682,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.944, + "ml_probability": 0.438, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2709,7 +2709,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2736,7 +2736,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2763,7 +2763,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.932, + "ml_probability": 0.855, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2790,7 +2790,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.991, + "ml_probability": 0.925, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2817,7 +2817,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.921, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2844,7 +2844,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.944, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2871,7 +2871,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2898,7 +2898,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2925,7 +2925,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.993, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2952,7 +2952,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2979,7 +2979,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3006,7 +3006,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.993, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3033,7 +3033,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.975, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3060,7 +3060,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.885, + "ml_probability": 0.759, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3087,7 +3087,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.943, + "ml_probability": 0.538, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3114,7 +3114,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.989, + "ml_probability": 0.766, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3141,7 +3141,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.971, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3168,7 +3168,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.839, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3195,7 +3195,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.871, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3222,7 +3222,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.51, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3249,7 +3249,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.904, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3276,7 +3276,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3303,7 +3303,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.086, + "ml_probability": 0.2, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3330,7 +3330,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.001, + "ml_probability": 0.076, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3357,7 +3357,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.617, + "ml_probability": 0.178, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3384,7 +3384,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.301, + "ml_probability": 0.577, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3411,7 +3411,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.202, + "ml_probability": 0.269, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3438,7 +3438,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.609, + "ml_probability": 0.466, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3465,7 +3465,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.427, + "ml_probability": 0.117, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3492,7 +3492,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.921, + "ml_probability": 0.968, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3519,7 +3519,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.866, + "ml_probability": 0.921, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3546,7 +3546,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.954, + "ml_probability": 0.435, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3573,7 +3573,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.04, + "ml_probability": 0.035, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3600,7 +3600,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.329, + "ml_probability": 0.135, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3627,7 +3627,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.937, + "ml_probability": 0.745, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3654,7 +3654,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.245, + "ml_probability": 0.236, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3681,7 +3681,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.035, + "ml_probability": 0.222, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3708,7 +3708,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.19, + "ml_probability": 0.567, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3735,7 +3735,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.842, + "ml_probability": 0.874, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3762,7 +3762,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.517, + "ml_probability": 0.195, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3789,7 +3789,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.959, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3816,7 +3816,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.951, + "ml_probability": 0.847, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3843,7 +3843,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.964, + "ml_probability": 0.939, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3870,7 +3870,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.968, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3897,7 +3897,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3924,7 +3924,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98, + "ml_probability": 0.967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3951,7 +3951,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.984, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3978,7 +3978,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.952, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4005,7 +4005,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.931, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4032,7 +4032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.951, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4059,7 +4059,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4086,7 +4086,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.978, + "ml_probability": 0.883, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4113,7 +4113,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.759, + "ml_probability": 0.229, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4140,7 +4140,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, + "ml_probability": 0.946, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4167,7 +4167,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4194,7 +4194,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4221,7 +4221,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.939, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4248,7 +4248,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4275,7 +4275,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.959, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4302,7 +4302,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, + "ml_probability": 0.886, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4329,7 +4329,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.965, + "ml_probability": 0.67, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4356,7 +4356,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4383,7 +4383,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.951, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4410,7 +4410,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.989, + "ml_probability": 0.822, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4437,7 +4437,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.767, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4464,7 +4464,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.97, + "ml_probability": 0.916, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4491,7 +4491,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.992, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4518,7 +4518,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.992, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4545,7 +4545,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.991, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4572,7 +4572,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.82, + "ml_probability": 0.488, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4626,7 +4626,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4869,7 +4869,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4950,7 +4950,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -5031,7 +5031,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.994, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -5058,7 +5058,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -5085,7 +5085,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -5166,7 +5166,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.995, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -5193,7 +5193,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.995, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -5220,7 +5220,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.0, + "ml_probability": 0.031, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -5274,7 +5274,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.958, + "ml_probability": 0.956, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5301,7 +5301,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.896, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5328,7 +5328,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.964, + "ml_probability": 0.943, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5355,7 +5355,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.967, + "ml_probability": 0.972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5436,7 +5436,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5463,7 +5463,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.94, + "ml_probability": 0.933, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5490,7 +5490,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.979, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5517,7 +5517,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5544,7 +5544,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5571,7 +5571,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.966, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5598,7 +5598,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5625,7 +5625,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.822, + "ml_probability": 0.899, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5652,7 +5652,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.981, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5706,7 +5706,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5733,7 +5733,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5760,7 +5760,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.758, + "ml_probability": 0.894, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -5787,7 +5787,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.758, + "ml_probability": 0.894, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -5814,7 +5814,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5841,7 +5841,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.967, + "ml_probability": 0.938, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5868,7 +5868,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.936, + "ml_probability": 0.992, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5895,7 +5895,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5922,7 +5922,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.984, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5949,7 +5949,34 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.94, + "ml_probability": 0.1, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "gildong.hong@98.76.54.32,pw:IhqSb1Gg", + "line_num": 47, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 28, + "value_end": 36, + "variable": "gildong.hong@98.76.54.32,pw", + "variable_start": 0, + "variable_end": 27, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.896, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5976,7 +6003,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.03, + "ml_probability": 0.01, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6003,7 +6030,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.929, + "ml_probability": 0.949, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6030,7 +6057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6057,7 +6084,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6084,7 +6111,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.987, + "ml_probability": 0.953, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6111,7 +6138,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6165,7 +6192,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.579, + "ml_probability": 0.131, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6192,7 +6219,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.865, + "ml_probability": 0.603, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6219,7 +6246,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.958, + "ml_probability": 0.956, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6246,7 +6273,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.003, + "ml_probability": 0.029, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6273,7 +6300,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.99, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6300,7 +6327,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.976, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6327,7 +6354,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6354,7 +6381,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.952, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6381,7 +6408,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.966, + "ml_probability": 0.943, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6408,7 +6435,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6435,7 +6462,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.995, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6462,7 +6489,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6489,7 +6516,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6516,7 +6543,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.987, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6543,7 +6570,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.995, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6570,7 +6597,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.881, + "ml_probability": 0.979, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6597,7 +6624,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6705,7 +6732,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.934, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6732,7 +6759,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.97, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6759,7 +6786,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.975, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6921,7 +6948,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -7630,7 +7657,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7792,7 +7819,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.853, + "ml_probability": 0.458, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -7819,7 +7846,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7846,7 +7873,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7873,7 +7900,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7900,7 +7927,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -8079,7 +8106,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -8177,7 +8204,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -8204,7 +8231,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Nonce", "severity": "medium", "confidence": "moderate", @@ -8312,7 +8339,34 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.0, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "MYSQLPASS: Ce7shE0ENPiBlE_EdEose0cBAA", + "line_num": 1, + "path": "tests/samples/pass_valid", + "info": "", + "value": "Ce7shE0ENPiBlE_EdEose0cBAA", + "value_start": 11, + "value_end": 37, + "variable": "MYSQLPASS", + "variable_start": 0, + "variable_end": 9, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.688513556888096, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.98, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8339,7 +8393,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8366,7 +8420,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.013, + "ml_probability": 0.038, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8393,7 +8447,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.163, + "ml_probability": 0.087, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8420,7 +8474,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.904, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8474,7 +8528,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8501,7 +8555,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8528,7 +8582,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.904, + "ml_probability": 0.818, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8555,7 +8609,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8582,7 +8636,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8609,7 +8663,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8636,7 +8690,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.963, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8985,7 +9039,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.378, + "ml_probability": 0.308, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9039,7 +9093,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.989, + "ml_probability": 0.887, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9066,7 +9120,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.322, + "ml_probability": 0.125, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9093,7 +9147,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.992, + "ml_probability": 0.907, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9120,7 +9174,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9, + "ml_probability": 0.816, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9147,7 +9201,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.992, + "ml_probability": 0.665, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9174,7 +9228,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.264, + "ml_probability": 0.192, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -9633,7 +9687,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88, + "ml_probability": 0.804, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -9660,7 +9714,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.716, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -9687,7 +9741,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -9741,7 +9795,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.994, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -9795,7 +9849,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.989, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -9849,7 +9903,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.993, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -9876,7 +9930,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.971, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -9930,7 +9984,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.076, "rule": "UUID", "severity": "info", "confidence": "strong", @@ -9984,7 +10038,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.911, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -10007,5 +10061,32 @@ } } ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.0, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "password : peace_for_ukraine", + "line_num": 9, + "path": "tests/samples/xml_password.xml", + "info": "", + "value": "peace_for_ukraine", + "value_start": 11, + "value_end": 28, + "variable": "password", + "variable_start": 0, + "variable_end": 8, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.091591477446567, + "valid": true + } + } + ] } ] \ No newline at end of file diff --git a/tests/data/output.json b/tests/data/output.json index a52688415..24dc955c2 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -29,7 +29,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.999, "rule": "API", "severity": "medium", "confidence": "moderate", @@ -191,7 +191,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.962, + "ml_probability": 0.87, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -218,7 +218,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -245,7 +245,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.991, + "ml_probability": 0.99, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -272,7 +272,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -299,7 +299,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.963, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -448,6 +448,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.924, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "\"AwsAccessKey\": \"AKIAGIREOGIAWSKEY123\", \"AwsAccessKey2\": \"AKIAGIREOGIAWSKEY555\",", + "line_num": 2, + "path": "tests/samples/aws_multi.groovy", + "info": "", + "value": "AKIAGIREOGIAWSKEY123", + "value_start": 17, + "value_end": 37, + "variable": "AwsAccessKey", + "variable_start": 1, + "variable_end": 13, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.5464393446710156, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -519,6 +546,33 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.922, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "\"AwsAccessKey\": \"AKIAGIREOGIAWSKEY123\", \"AwsAccessKey2\": \"AKIAGIREOGIAWSKEY555\",", + "line_num": 2, + "path": "tests/samples/aws_multi.groovy", + "info": "", + "value": "AKIAGIREOGIAWSKEY555", + "value_start": 58, + "value_end": 78, + "variable": "AwsAccessKey2", + "variable_start": 41, + "variable_end": 54, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.3086949695628425, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -549,7 +603,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.994, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -873,7 +927,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.966, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -900,7 +954,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Certificate", "severity": "medium", "confidence": "moderate", @@ -927,7 +981,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.996, "rule": "Credential", "severity": "medium", "confidence": "moderate", @@ -1035,7 +1089,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.775, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1062,61 +1116,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.918, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ID:master PW:dipPr11Gg!", - "line_num": 2, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "dipPr11Gg!", - "value_start": 13, - "value_end": 23, - "variable": "PW", - "variable_start": 10, - "variable_end": 12, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.7897352853986264, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ANYID:master PW:dipPr12Gg!", - "line_num": 3, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "dipPr12Gg!", - "value_start": 16, - "value_end": 26, - "variable": "PW", - "variable_start": 13, - "variable_end": 15, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.989735285398626, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.926, + "ml_probability": 0.881, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1143,7 +1143,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.977, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1170,7 +1170,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.786, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1197,7 +1197,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1224,7 +1224,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1251,7 +1251,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1278,7 +1278,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.957, + "ml_probability": 0.812, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1305,7 +1305,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.886, + "ml_probability": 0.941, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1332,34 +1332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.896, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "ID:master PWD:dipPr112Gg!", - "line_num": 13, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "dipPr112Gg!", - "value_start": 14, - "value_end": 25, - "variable": "PWD", - "variable_start": 10, - "variable_end": 13, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.9631196533066344, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.984, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1386,7 +1359,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.985, + "ml_probability": 0.975, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1413,7 +1386,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.969, + "ml_probability": 0.963, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1440,7 +1413,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.941, + "ml_probability": 0.97, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1467,7 +1440,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.976, + "ml_probability": 0.972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1494,7 +1467,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.946, + "ml_probability": 0.965, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1521,7 +1494,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.959, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1548,7 +1521,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.974, + "ml_probability": 0.926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1575,7 +1548,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.954, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1602,7 +1575,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.968, + "ml_probability": 0.735, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1629,7 +1602,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.909, + "ml_probability": 0.928, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1656,7 +1629,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.974, + "ml_probability": 0.893, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1683,7 +1656,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.791, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1710,7 +1683,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.915, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1737,7 +1710,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.941, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1764,7 +1737,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.921, + "ml_probability": 0.639, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1791,7 +1764,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.945, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1818,7 +1791,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.945, + "ml_probability": 0.85, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1845,7 +1818,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.893, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1872,7 +1845,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.811, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1899,7 +1872,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, + "ml_probability": 0.973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1926,7 +1899,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.991, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1953,7 +1926,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1980,7 +1953,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.949, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2007,7 +1980,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2034,7 +2007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.99, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2061,7 +2034,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.991, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2088,7 +2061,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.752, + "ml_probability": 0.849, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2115,7 +2088,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2142,7 +2115,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.984, + "ml_probability": 0.969, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2169,7 +2142,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.991, + "ml_probability": 0.977, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2196,7 +2169,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.928, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2223,34 +2196,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "(98.76.54.32)ID:master PW:dipPr149Gg!", - "line_num": 50, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "dipPr149Gg!", - "value_start": 26, - "value_end": 37, - "variable": "PW", - "variable_start": 23, - "variable_end": 25, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.1449378351248165, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.968, + "ml_probability": 0.736, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2277,7 +2223,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.937, + "ml_probability": 0.629, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2304,7 +2250,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.984, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2331,7 +2277,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2358,7 +2304,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2385,7 +2331,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2412,7 +2358,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.986, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2439,7 +2385,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2466,7 +2412,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.987, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2493,7 +2439,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.953, + "ml_probability": 0.976, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2520,7 +2466,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.865, + "ml_probability": 0.652, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2547,34 +2493,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.944, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "dipPr163Gg! ID:master dipPr163Gg! PWD:dipPr163Gg!", - "line_num": 64, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "dipPr163Gg!", - "value_start": 38, - "value_end": 49, - "variable": "PWD", - "variable_start": 34, - "variable_end": 37, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.1449378351248165, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2601,7 +2520,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2628,7 +2547,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.932, + "ml_probability": 0.855, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2655,7 +2574,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.991, + "ml_probability": 0.925, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2682,7 +2601,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.921, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2709,7 +2628,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.944, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2736,7 +2655,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2763,7 +2682,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2790,7 +2709,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.993, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2817,7 +2736,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2844,7 +2763,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2871,7 +2790,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.993, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2898,7 +2817,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.975, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2925,7 +2844,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.885, + "ml_probability": 0.759, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2952,34 +2871,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.943, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "id:master,default pw:dipPr182Gg!", - "line_num": 83, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "dipPr182Gg!", - "value_start": 21, - "value_end": 32, - "variable": "pw", - "variable_start": 18, - "variable_end": 20, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.1449378351248165, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.989, + "ml_probability": 0.766, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3006,7 +2898,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.971, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3033,7 +2925,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.839, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3060,7 +2952,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.871, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3087,34 +2979,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "PW:dipPr190Gg! ID:master", - "line_num": 91, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "dipPr190Gg!", - "value_start": 3, - "value_end": 14, - "variable": "PW", - "variable_start": 0, - "variable_end": 2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.1449378351248165, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.904, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3141,7 +3006,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3168,7 +3033,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.921, + "ml_probability": 0.968, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3195,7 +3060,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.866, + "ml_probability": 0.921, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3222,34 +3087,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.954, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "98.76.54.32(ID:master/PW:iPp10@GRq) # todo: move into other sample ?", - "line_num": 11, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "iPp10@GRq)", - "value_start": 25, - "value_end": 35, - "variable": "PW", - "variable_start": 22, - "variable_end": 24, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.65754247590989, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.937, + "ml_probability": 0.745, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3276,7 +3114,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.842, + "ml_probability": 0.874, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3303,7 +3141,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.959, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3330,7 +3168,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.951, + "ml_probability": 0.847, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3357,7 +3195,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.964, + "ml_probability": 0.939, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3384,7 +3222,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.986, + "ml_probability": 0.968, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3411,7 +3249,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3438,7 +3276,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98, + "ml_probability": 0.967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3465,7 +3303,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.984, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3492,7 +3330,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.952, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3519,7 +3357,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.931, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3546,7 +3384,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.951, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3573,7 +3411,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3600,49 +3438,22 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.978, + "ml_probability": 0.883, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { "line": "Passwd:Prl23Db#@ Prl23Db#@", - "line_num": 23, - "path": "tests/samples/doc_passwd_pair", - "info": "", - "value": "Prl23Db#@", - "value_start": 7, - "value_end": 16, - "variable": "Passwd", - "variable_start": 0, - "variable_end": 6, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.4654972233440207, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.759, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "PW:Prl23Db#@,password:Prl23Db#@", - "line_num": 24, + "line_num": 23, "path": "tests/samples/doc_passwd_pair", "info": "", "value": "Prl23Db#@", - "value_start": 3, - "value_end": 12, - "variable": "PW", + "value_start": 7, + "value_end": 16, + "variable": "Passwd", "variable_start": 0, - "variable_end": 2, + "variable_end": 6, "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 2.4654972233440207, @@ -3654,7 +3465,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, + "ml_probability": 0.946, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3681,7 +3492,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3708,7 +3519,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3735,7 +3546,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.939, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3762,7 +3573,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3789,7 +3600,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.959, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3816,7 +3627,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.982, + "ml_probability": 0.886, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3843,7 +3654,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.965, + "ml_probability": 0.67, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3870,7 +3681,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3897,7 +3708,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.951, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3924,7 +3735,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.989, + "ml_probability": 0.822, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3951,7 +3762,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.767, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3978,7 +3789,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.97, + "ml_probability": 0.916, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4005,7 +3816,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.992, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4032,7 +3843,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.992, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4059,7 +3870,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.991, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4083,33 +3894,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.82, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "# password: keep empty", - "line_num": 56, - "path": "tests/samples/doc_passwd_pair", - "info": "", - "value": "keep", - "value_start": 12, - "value_end": 16, - "variable": "password", - "variable_start": 2, - "variable_end": 10, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 1.5, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -4140,7 +3924,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4383,7 +4167,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4464,7 +4248,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4545,7 +4329,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.994, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4572,7 +4356,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4599,7 +4383,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4680,7 +4464,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.995, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -4707,7 +4491,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.995, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4734,7 +4518,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.958, + "ml_probability": 0.956, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4761,7 +4545,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.896, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4788,7 +4572,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.964, + "ml_probability": 0.943, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4815,7 +4599,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.967, + "ml_probability": 0.972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4896,7 +4680,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4923,7 +4707,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.94, + "ml_probability": 0.933, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4950,7 +4734,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.979, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4977,7 +4761,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5004,7 +4788,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5031,7 +4815,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.966, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5058,7 +4842,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5085,7 +4869,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.822, + "ml_probability": 0.899, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5112,7 +4896,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.981, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5166,7 +4950,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5193,7 +4977,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5220,7 +5004,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.758, + "ml_probability": 0.894, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -5247,7 +5031,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.758, + "ml_probability": 0.894, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -5274,7 +5058,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5301,7 +5085,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.967, + "ml_probability": 0.938, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5328,7 +5112,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.936, + "ml_probability": 0.992, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5355,7 +5139,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5382,7 +5166,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.984, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5409,7 +5193,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.94, + "ml_probability": 0.896, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5436,7 +5220,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.929, + "ml_probability": 0.949, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5463,7 +5247,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5490,7 +5274,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5517,7 +5301,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.987, + "ml_probability": 0.953, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5544,7 +5328,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5598,34 +5382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.865, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "(ID&PWD):master/IhqSb1Gg", - "line_num": 71, - "path": "tests/samples/doc_various", - "info": "", - "value": "master/IhqSb1Gg", - "value_start": 9, - "value_end": 24, - "variable": "PWD)", - "variable_start": 4, - "variable_end": 8, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.906890595608518, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.958, + "ml_probability": 0.956, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5652,7 +5409,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.99, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5679,7 +5436,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.976, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5706,7 +5463,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5733,7 +5490,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.952, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5760,7 +5517,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.966, + "ml_probability": 0.943, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5787,7 +5544,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5814,7 +5571,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.995, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5841,7 +5598,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5868,7 +5625,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5895,7 +5652,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.987, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5922,7 +5679,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.993, + "ml_probability": 0.995, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5949,7 +5706,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.881, + "ml_probability": 0.979, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5976,7 +5733,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6084,7 +5841,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.999, + "ml_probability": 0.934, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6111,7 +5868,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.996, + "ml_probability": 0.97, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6138,7 +5895,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.975, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6300,7 +6057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -7009,7 +6766,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7171,34 +6928,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.853, - "rule": "Key", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "prKeyValid=LS0tLS1CRUdJTiBQUklWQVRFIENDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ0ViVnpmUGWxhQW9KQWwrLzZYdDJPNG1PQjYxMXNPaFJBTkNBQVNnRlRLandKQUFVOTVnKysvdnpLV0hrekFWbU5NSQp0QjV2VGpaT09Jd25FYjcwTXNXWkZJeVVGRDFQOUd3c3R6NCtha0hYN3ZJOEJINmhIbUJtZmVRbAotLS0tLUVORCBQUklWJNR0J5cUdTTTQ5QW5aUHhmQXl4cUUKWlYwNdFR0QVRFIEtFWS0tLS0tCgtFWS0tLS0tCk1JR0hBZ0VBTU==", - "line_num": 1, - "path": "tests/samples/key.hs", - "info": "", - "value": "LS0tLS1CRUdJTiBQUklWQVRFIENDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ0ViVnpmUGWxhQW9KQWwrLzZYdDJPNG1PQjYxMXNPaFJBTkNBQVNnRlRLandKQUFVOTVnKysvdnpLV0hrekFWbU5NSQp0QjV2VGpaT09Jd25FYjcwTXNXWkZJeVVGRDFQOUd3c3R6NCtha0hYN3ZJOEJINmhIbUJtZmVRbAotLS0tLUVORCBQUklWJNR0J5cUdTTTQ5QW5aUHhmQXl4cUUKWlYwNdFR0QVRFIEtFWS0tLS0tCgtFWS0tLS0tCk1JR0hBZ0VBTU==", - "value_start": 11, - "value_end": 335, - "variable": "prKeyValid", - "variable_start": 0, - "variable_end": 10, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.428808109703668, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7225,7 +6955,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7252,7 +6982,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7279,7 +7009,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -7458,7 +7188,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -7556,7 +7286,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -7583,7 +7313,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Nonce", "severity": "medium", "confidence": "moderate", @@ -7691,7 +7421,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, + "ml_probability": 0.98, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7718,7 +7448,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7745,7 +7475,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, + "ml_probability": 0.904, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7799,7 +7529,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7826,7 +7556,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.996, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7853,7 +7583,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.904, + "ml_probability": 0.818, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7880,7 +7610,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7907,7 +7637,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7934,7 +7664,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, + "ml_probability": 0.978, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7961,7 +7691,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.963, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8337,7 +8067,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.989, + "ml_probability": 0.887, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -8364,7 +8094,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.992, + "ml_probability": 0.907, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -8391,7 +8121,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9, + "ml_probability": 0.816, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -8418,7 +8148,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.992, + "ml_probability": 0.665, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -8877,7 +8607,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88, + "ml_probability": 0.804, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -8904,7 +8634,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.998, + "ml_probability": 0.716, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -8931,7 +8661,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -8985,7 +8715,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.994, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -9039,7 +8769,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, + "ml_probability": 0.989, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -9093,7 +8823,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, + "ml_probability": 0.993, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -9120,7 +8850,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.983, + "ml_probability": 0.971, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -9171,33 +8901,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 1.0, - "rule": "UUID", - "severity": "info", - "confidence": "strong", - "line_data_list": [ - { - "line": "bace4d19-fa7e-beef-cafe-9129474bcd81 # tp", - "line_num": 1, - "path": "tests/samples/uuid", - "info": "", - "value": "bace4d19-fa7e-beef-cafe-9129474bcd81", - "value_start": 0, - "value_end": 36, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2373263071270246, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -9228,7 +8931,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, + "ml_probability": 0.911, "rule": "Password", "severity": "medium", "confidence": "moderate", diff --git a/tests/ml_model/test_ml_validator.py b/tests/ml_model/test_ml_validator.py index d2d73403a..0c2d33c64 100644 --- a/tests/ml_model/test_ml_validator.py +++ b/tests/ml_model/test_ml_validator.py @@ -53,17 +53,17 @@ def validate(_candidate: Candidate) -> Tuple[bool, float]: candidate.line_data_list[0].path = "sample.yaml" candidate.line_data_list[0].file_type = ".yaml" decision, probability = validate(candidate) - self.assertAlmostEqual(0.9958975315093994, probability, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.9994515776634216, probability, delta=NEGLIGIBLE_ML_THRESHOLD) candidate.line_data_list[0].path = "test.cc" candidate.line_data_list[0].file_type = ".cc" decision, probability = validate(candidate) - self.assertAlmostEqual(0.9883015155792236, probability, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.9989229440689087, probability, delta=NEGLIGIBLE_ML_THRESHOLD) candidate.line_data_list[0].path = "other.unknown" candidate.line_data_list[0].file_type = ".unknown" decision, probability = validate(candidate) - self.assertAlmostEqual(0.9985307455062866, probability, delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.999495267868042, probability, delta=NEGLIGIBLE_ML_THRESHOLD) def test_ml_validator_auxiliary_p(self): candidate = Candidate.get_dummy_candidate(self.config, "mycred", "", "") @@ -77,22 +77,24 @@ def test_ml_validator_auxiliary_p(self): aux_candidate = copy.deepcopy(candidate) aux_candidate.line_data_list[0].variable = None + # todo: the scores are low for current ML model - will be changed after train + candidate_key = CandidateKey(candidate.line_data_list[0]) sample_as_batch = [(candidate_key, [candidate])] is_cred_batch, probability_batch = self.ml_validator.validate_groups(sample_as_batch, 2) - self.assertAlmostEqual(0.9583272337913513, probability_batch[0], delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.16333681344985962, probability_batch[0], delta=NEGLIGIBLE_ML_THRESHOLD) # auxiliary rule which was not trained - keeps the same ML probability aux_candidate.rule_name = "PASSWD_PAIR" sample_as_batch = [(candidate_key, [candidate, aux_candidate])] is_cred_batch, probability_batch = self.ml_validator.validate_groups(sample_as_batch, 2) - self.assertAlmostEqual(0.9583272337913513, probability_batch[0], delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.16333681344985962, probability_batch[0], delta=NEGLIGIBLE_ML_THRESHOLD) # auxiliary rule in train increases ML probability aux_candidate.rule_name = "UUID" sample_as_batch = [(candidate_key, [candidate, aux_candidate])] is_cred_batch, probability_batch = self.ml_validator.validate_groups(sample_as_batch, 2) - self.assertAlmostEqual(0.9972454309463501, probability_batch[0], delta=NEGLIGIBLE_ML_THRESHOLD) + self.assertAlmostEqual(0.16333681344985962, probability_batch[0], delta=NEGLIGIBLE_ML_THRESHOLD) def test_extract_features_p(self): candidate1 = Candidate.get_dummy_candidate(self.config, "main.py", ".py", "info") diff --git a/tests/test_app.py b/tests/test_app.py index 888eeaf78..cbe4c40fd 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -602,6 +602,7 @@ def test_rules_ml_p(self) -> None: "IP_ID_PASSWORD_TRIPLE", "ID_PAIR_PASSWD_PAIR", "PASSWD_PAIR", + "UUID", # todo: remove this after ML retrain with the rule } self.assertSetEqual(rules_set.difference(missed), report_set, f"\n{_stdout}") self.assertEqual(SAMPLES_POST_CRED_COUNT, len(report)) diff --git a/tests/test_main.py b/tests/test_main.py index 14c8a619b..7e664004b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -487,7 +487,7 @@ def test_pdf_p(self) -> None: cred_sweeper = CredSweeper(depth=7) cred_sweeper.run(content_provider=content_provider) found_credentials = cred_sweeper.credential_manager.get_credentials() - self.assertSetEqual({"AWS Client ID", "Password", "Github Classic Token"}, + self.assertSetEqual({"AWS Client ID", "Password", "Github Classic Token", "Key"}, set(i.rule_name for i in found_credentials)) self.assertSetEqual({"Xdj@jcN834b", "AKIAGIREOGIAWSKEY123", "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"}, set(i.line_data_list[0].value for i in found_credentials))