From 4c8a10bb0b1bc0096b3e66e1ab524a3344857bb1 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Wed, 7 Aug 2024 15:54:56 +0300 Subject: [PATCH] fix --- tests/__init__.py | 12 +- tests/data/depth_3.json | 296 +---------------------------------- tests/data/ml_threshold.json | 296 +---------------------------------- tests/data/output.json | 282 +-------------------------------- 4 files changed, 10 insertions(+), 876 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index a501eae08..792fa3d01 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,24 +1,24 @@ from pathlib import Path # total number of files in test samples -SAMPLES_FILES_COUNT: int = 129 +SAMPLES_FILES_COUNT: int = 130 # the lowest value of ML threshold is used to display possible lowest values NEGLIGIBLE_ML_THRESHOLD = 0.0001 # credentials count after scan -SAMPLES_CRED_COUNT: int = 362 -SAMPLES_CRED_LINE_COUNT: int = 379 +SAMPLES_CRED_COUNT: int = 360 +SAMPLES_CRED_LINE_COUNT: int = 377 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 321 +SAMPLES_POST_CRED_COUNT: int = 332 # with option --doc -SAMPLES_IN_DOC = 415 +SAMPLES_IN_DOC = 416 # archived credentials that are not found without --depth SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 24 -SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 17 +SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 16 SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 1 # well known string with all latin letters diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index 14db68ac0..4d1705590 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -394,11 +394,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD - "ml_probability": 0.945, -======= - "ml_probability": 0.963, ->>>>>>> auxiliary + "ml_probability": 0.998, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -424,36 +420,6 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.945, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "curl -H \"Authorization: Bearer eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj\" http://localhost:8080/.", - "line_num": 9, - "path": "tests/samples/auth_n.template", - "info": "tests/samples/auth_n.template|RAW", - "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj", - "value_start": 31, - "value_end": 65, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2479906920322064, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "NOT_AVAILABLE", "ml_probability": null, "rule": "AWS Client ID", @@ -758,13 +724,8 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, -======= "ml_validation": "NOT_AVAILABLE", "ml_probability": null, ->>>>>>> auxiliary "rule": "JSON Web Token", "severity": "medium", "confidence": "strong", @@ -3328,7 +3289,6 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD "ml_validation": "VALIDATED_KEY", "ml_probability": 0.994, "rule": "Password", @@ -3356,35 +3316,6 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ANYid:master pw:dipPr194Gg! ip:98.76.54.32", - "line_num": 95, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 31, - "value_end": 42, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "VALIDATED_KEY", "ml_probability": 0.998, "rule": "Password", @@ -3491,34 +3422,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID:master/PW:iPp10@GRq) # todo: move into other sample ?", - "line_num": 11, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3548,35 +3451,6 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 id/pw:master/iPp19@GRq", - "line_num": 20, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "VALIDATED_KEY", "ml_probability": 0.937, "rule": "Password", @@ -5924,34 +5798,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID/PW:master/IhqSb1Gg)", - "line_num": 70, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5981,35 +5827,6 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 73, - "path": "tests/samples/doc_various", - "info": "tests/samples/doc_various|RAW", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "VALIDATED_KEY", "ml_probability": 0.958, "rule": "Password", @@ -7340,36 +7157,6 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", - "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", - "info": "tests/samples/grafana_provisioned_api_key|RAW", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", - "value_start": 11, - "value_end": 107, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "NOT_AVAILABLE", "ml_probability": null, "rule": "Grafana Service Account Token", @@ -7613,13 +7400,8 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, -======= "ml_validation": "NOT_AVAILABLE", "ml_probability": null, ->>>>>>> auxiliary "rule": "JSON Web Token", "severity": "medium", "confidence": "strong", @@ -7754,11 +7536,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD "ml_probability": 1.0, -======= - "ml_probability": 0.999, ->>>>>>> auxiliary "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -9848,11 +9626,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD "ml_probability": 0.963, -======= - "ml_probability": 0.911, ->>>>>>> auxiliary "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9879,11 +9653,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD "ml_probability": 0.989, -======= - "ml_probability": 0.887, ->>>>>>> auxiliary "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -11017,38 +10787,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD - "ml_probability": 0.979, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in zip: eyJrIjoiMDAwMDAwNDAwMDAwODAwMDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 1, - "path": "tests/samples/test2.eml", - "info": "tests/samples/test2.eml|EML-DATA|ZIP|token.txt|RAW", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwMDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 14, - "value_end": 108, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.006147345318248, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.979, -======= - "ml_probability": 1.0, ->>>>>>> auxiliary + "ml_probability": 0.999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -11102,38 +10841,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD - "ml_probability": 0.975, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in text: eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 8, - "path": "tests/samples/test2.eml", - "info": "tests/samples/test2.eml|EML-TEXT", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 15, - "value_end": 109, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.0296677144829305, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.975, -======= "ml_probability": 0.999, ->>>>>>> auxiliary "rule": "Token", "severity": "medium", "confidence": "moderate", diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json index e8cca3742..fa80d0d45 100644 --- a/tests/data/ml_threshold.json +++ b/tests/data/ml_threshold.json @@ -299,11 +299,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD - "ml_probability": 0.945, -======= - "ml_probability": 0.963, ->>>>>>> auxiliary + "ml_probability": 0.998, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -329,36 +325,6 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.945, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "curl -H \"Authorization: Bearer eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj\" http://localhost:8080/.", - "line_num": 9, - "path": "tests/samples/auth_n.template", - "info": "", - "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj", - "value_start": 31, - "value_end": 65, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2479906920322064, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "NOT_AVAILABLE", "ml_probability": null, "rule": "AWS Client ID", @@ -771,13 +737,8 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, -======= "ml_validation": "NOT_AVAILABLE", "ml_probability": null, ->>>>>>> auxiliary "rule": "JSON Web Token", "severity": "medium", "confidence": "strong", @@ -5985,161 +5946,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", -======= - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.1, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", ->>>>>>> auxiliary - "line_data_list": [ - { - "line": "gildong.hong@98.76.54.32,pw:IhqSb1Gg", - "line_num": 47, - "path": "tests/samples/doc_various", - "info": "", -<<<<<<< HEAD - "value": "98.76.54.32", - "value_start": 13, - "value_end": 24, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(master/IhqSb1Gg,master/IhqSb1Gg)", - "line_num": 48, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(master/IhqSb1Gg master/IhqSb1Gg)", - "line_num": 49, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "xxxx:98.76.54.32(master/IhqSb1Gg)", - "line_num": 50, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 5, - "value_end": 16, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", - "line_num": 51, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, -======= - "value": "IhqSb1Gg", - "value_start": 28, - "value_end": 36, - "variable": "gildong.hong@98.76.54.32,pw", - "variable_start": 0, - "variable_end": 27, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.0, ->>>>>>> auxiliary - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -7715,36 +7521,6 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", - "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", - "info": "", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", - "value_start": 11, - "value_end": 107, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "NOT_AVAILABLE", "ml_probability": null, "rule": "Grafana Service Account Token", @@ -7988,13 +7764,8 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, -======= "ml_validation": "NOT_AVAILABLE", "ml_probability": null, ->>>>>>> auxiliary "rule": "JSON Web Token", "severity": "medium", "confidence": "strong", @@ -8129,11 +7900,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD "ml_probability": 1.0, -======= - "ml_probability": 0.999, ->>>>>>> auxiliary "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -9811,36 +9578,6 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "eyJUaGVyZSBpcyBub3QgdGhlIEpTT04geW91IGFyZSBsb29raW5nIGZvciJ9CjwvYm9keT4KPC9o", - "line_num": 17, - "path": "tests/samples/test.eml", - "info": "", - "value": "eyJUaGVyZSBpcyBub3QgdGhlIEpTT04geW91IGFyZSBsb29raW5nIGZvciJ9CjwvYm9keT4KPC9o", - "value_start": 0, - "value_end": 76, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.282347539953402, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "NOT_AVAILABLE", "ml_probability": null, "rule": "Grafana Provisioned API Key", @@ -9869,38 +9606,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD - "ml_probability": 0.975, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in text: eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 18, - "path": "tests/samples/test2.eml", - "info": "", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 15, - "value_end": 109, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.0296677144829305, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.975, -======= "ml_probability": 0.999, ->>>>>>> auxiliary "rule": "Token", "severity": "medium", "confidence": "moderate", diff --git a/tests/data/output.json b/tests/data/output.json index bdde5d407..a52688415 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -299,11 +299,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD - "ml_probability": 0.945, -======= - "ml_probability": 0.963, ->>>>>>> auxiliary + "ml_probability": 0.998, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -329,36 +325,6 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.945, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "curl -H \"Authorization: Bearer eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj\" http://localhost:8080/.", - "line_num": 9, - "path": "tests/samples/auth_n.template", - "info": "", - "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj", - "value_start": 31, - "value_end": 65, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2479906920322064, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "NOT_AVAILABLE", "ml_probability": null, "rule": "AWS Client ID", @@ -663,13 +629,8 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.988, -======= "ml_validation": "NOT_AVAILABLE", "ml_probability": null, ->>>>>>> auxiliary "rule": "JSON Web Token", "severity": "medium", "confidence": "strong", @@ -3125,7 +3086,6 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD "ml_validation": "VALIDATED_KEY", "ml_probability": 0.994, "rule": "Password", @@ -3153,35 +3113,6 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "ANYid:master pw:dipPr194Gg! ip:98.76.54.32", - "line_num": 95, - "path": "tests/samples/doc_id_pair_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 31, - "value_end": 42, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "VALIDATED_KEY", "ml_probability": 0.998, "rule": "Password", @@ -3288,34 +3219,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID:master/PW:iPp10@GRq) # todo: move into other sample ?", - "line_num": 11, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -3345,35 +3248,6 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32 id/pw:master/iPp19@GRq", - "line_num": 20, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "VALIDATED_KEY", "ml_probability": 0.937, "rule": "Password", @@ -5721,34 +5595,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "98.76.54.32(ID/PW:master/IhqSb1Gg)", - "line_num": 70, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", @@ -5778,35 +5624,6 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", - "line_data_list": [ - { - "line": "master@98.76.54.32(pw:IhqSb1Gg)", - "line_num": 73, - "path": "tests/samples/doc_various", - "info": "", - "value": "98.76.54.32", - "value_start": 7, - "value_end": 18, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "VALIDATED_KEY", "ml_probability": 0.958, "rule": "Password", @@ -7083,36 +6900,6 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "grafana = 'eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0='", - "line_num": 1, - "path": "tests/samples/grafana_provisioned_api_key", - "info": "", - "value": "eyJrIjoiMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCIsIm4iOiJ4eHh4IiwiaWQiOjIwNDM2MH0=", - "value_start": 11, - "value_end": 107, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.8153130511409934, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", -======= ->>>>>>> auxiliary "ml_validation": "NOT_AVAILABLE", "ml_probability": null, "rule": "Grafana Service Account Token", @@ -7356,13 +7143,8 @@ }, { "api_validation": "NOT_AVAILABLE", -<<<<<<< HEAD - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.997, -======= "ml_validation": "NOT_AVAILABLE", "ml_probability": null, ->>>>>>> auxiliary "rule": "JSON Web Token", "severity": "medium", "confidence": "strong", @@ -7497,11 +7279,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD "ml_probability": 1.0, -======= - "ml_probability": 0.999, ->>>>>>> auxiliary "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -9042,33 +8820,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.994, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "eyJUaGVyZSBpcyBub3QgdGhlIEpTT04geW91IGFyZSBsb29raW5nIGZvciJ9CjwvYm9keT4KPC9o", - "line_num": 17, - "path": "tests/samples/test.eml", - "info": "", - "value": "eyJUaGVyZSBpcyBub3QgdGhlIEpTT04geW91IGFyZSBsb29raW5nIGZvciJ9CjwvYm9keT4KPC9o", - "value_start": 0, - "value_end": 76, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 5.282347539953402, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -9099,38 +8850,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", -<<<<<<< HEAD - "ml_probability": 0.975, - "rule": "JSON Web Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "token in text: eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "line_num": 18, - "path": "tests/samples/test2.eml", - "info": "", - "value": "eyJrIjoiMDAwMDAwNDAwMDAwODAwNDAwMDAwMDAwNDAwMDAwMDAwMDAwMDAyMSIsIm4iOiJ4eHh4IiwiaWQiOjQzMDh9Cg", - "value_start": 15, - "value_end": 109, - "variable": null, - "variable_start": -2, - "variable_end": -2, - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.0296677144829305, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.975, -======= "ml_probability": 0.999, ->>>>>>> auxiliary "rule": "Token", "severity": "medium", "confidence": "moderate",