Command line password patterns

.github/workflows/benchmark.yml

@@ -22,7 +22,8 @@ jobs:
       - name: Checkout CredData
         uses: actions/checkout@v4
         with:
-          repository: Samsung/CredData
+          repository: babenek/CredData
+          ref: auxiliary
 
       - name: Markup hashing
         run: |
@@ -72,7 +73,8 @@ jobs:
       - name: Checkout CredData
         uses: actions/checkout@v4
         with:
-          repository: Samsung/CredData
+          repository: babenek/CredData
+          ref: auxiliary
 
       - name: Markup hashing
         run: |
@@ -169,7 +171,8 @@ jobs:
       - name: Checkout CredData
         uses: actions/checkout@v4
         with:
-          repository: Samsung/CredData
+          repository: babenek/CredData
+          ref: auxiliary
 
       - name: Markup hashing
         run: |
@@ -351,7 +354,8 @@ jobs:
       - name: Checkout CredData
         uses: actions/checkout@v4
         with:
-          repository: Samsung/CredData
+          repository: babenek/CredData
+          ref: auxiliary
 
       - name: Markup hashing
         run: |

.github/workflows/check.yml

@@ -32,9 +32,8 @@ jobs:
     - name: Check ml_model.onnx integrity
       if: ${{ always() && steps.code_checkout.conclusion == 'success' }}
       run: |
-        md5sum --binary credsweeper/ml_model/ml_config.json | grep 2b29c5e1aa199d14b788652bd542c7c0
-        md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 88f37978fc0599ac8d1bf732ad40c077
-
+        md5sum --binary credsweeper/ml_model/ml_config.json | grep caa591316d13b6fde3d27f7929c44651
+        md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 201150931cfea4261d35626237585afe
 
     # # # line ending
 

credsweeper/common/constants.py

@@ -13,13 +13,15 @@ class KeywordPattern:
                 r"[^:='\"`<>{?!&]*)[`'\"]*)"  # <variable>
     separator = r"\s*\]?\s*" \
                 r"(?P<separator>:( [a-z]{3,9}[?]? )?=" \
-                r"|:|=>|!=|===|==|=)" \
-                r"\s*(?P<wrap>((new\s*)?\w|\.|->|\(|\[)*[\[\(\{](\w{1,32}=)?\s*)?"
+                r"|:|=(>|&gt;|\\u0026gt;)|!=|===|==|=)" \
+                r"(\s|\\[tnr])*"\
+                r"(ConvertTo-SecureString(\s*-String)?\s*)?" \
+                r"(?P<wrap>((new\s*)?\w|\.|-(>|&gt;|\\u0026gt;)|\(|\[)*[\[\(\{](\w{1,32}=)?\s*)?"
     # Authentication scheme ( oauth | basic | bearer | apikey ) precedes to credential
     value = r"(?P<value_leftquote>((b|r|br|rb|u|f|rf|fr|\\{0,8})?[`'\"]){1,4})?" \
             r"( ?(oauth|bot|basic|bearer|apikey|accesskey) )?" \
             r"(?P<value>" \
-            r"(?(value_leftquote)(?:\\[tnrux0-7][0-9a-f]*|[^`'\"\\])|(?:\\n|\\r|\\?[^\s`'\"\\,;])){1,8000}" \
+            r"(?(value_leftquote)(?:\\[tnr]|\\[ux0-7][0-9a-f]*|[^`'\"\\])|(?:\\t|\\n|\\r|\\?[^\s`'\"\\,;])){1,8000}" \
             r"|(?:\{[^}]{3,8000}\})|(?:<[^>]{3,8000}>)" \
             r")" \
             r"(?(value_leftquote)(?P<value_rightquote>(\\{0,8}[`'\"]){1,4})?|(?(wrap)[\]\)\},;]))"

credsweeper/credentials/line_data.py

@@ -191,9 +191,17 @@ def clean_bash_parameters(self) -> None:
                 self.value = value_whsp[0]
 
     def clean_toml_parameters(self) -> None:
-        """Curly brackets may be caught in TOML format"""
-        while self.value.endswith('}') and '{' in self.line[:self.value_start]:
-            self.value = self.value[:-1]
+        """Parenthesis, curly and squared brackets may be caught in TOML format and bash. Simple clearing"""
+        dirty = self.value and self.value[-1] in ['}', ']', ')']
+        line_before_value = self.line[:self.value_start]
+        while dirty:
+            dirty = False
+            for left, right in [('{', '}'), ('[', ']'), ('(', ')')]:
+                if self.value.endswith(right) and left not in self.value \
+                      and line_before_value.count(left) > line_before_value.count(right):
+                    # full match does not reasonable to implement due open character may be in other line
+                    self.value = self.value[:-1]
+                    dirty = True
 
     def sanitize_variable(self) -> None:
         """Remove trailing spaces, dashes and quotations around the variable. Correct position."""

credsweeper/filters/value_allowlist_check.py

@@ -11,8 +11,15 @@ class ValueAllowlistCheck(Filter):
     """Check that patterns from the list is not present in the candidate value."""
 
     ALLOWED = [
-        r"ENC\(.*\)", r"ENC\[.*\]", r"\$\{.*\}", r"#\{.*\}", r"\{\{.+\}\}", r"([.a-z0-9]|->)+\(.*\)", r"\S{0,5}\*{5,}",
-        r".*@@@hl@@@(암호|비번|PW|PASS)@@@endhl@@@.*"
+        r"ENC\(.*\)",  #
+        r"ENC\[.*\]",  #
+        r"\$\{[a-z_][0-9a-z_]*\}",  #
+        r"\$[a-z_][0-9a-z_]*",  #
+        r"#\{.*\}",  #
+        r"\{\{.+\}\}",  #
+        r"([.a-z0-9]|->)+\(.*\)(\W|$)",  #
+        r"\S{0,5}\*{5,}",  #
+        r".*@@@hl@@@(암호|비번|PW|PASS)@@@endhl@@@.*",  #
     ]
     ALLOWED_PATTERN = re.compile(  #
         Util.get_regex_combine_or(ALLOWED),  #

credsweeper/filters/value_pattern_check.py

@@ -32,7 +32,7 @@ def __init__(self, config: Config):
             self.pattern_len = config.pem_pattern_len
         else:
             self.pattern_len = config.pattern_len
-        self.pattern = re.compile(fr"(.)\1{{{str(self.pattern_len - 1)},}}")
+        self.pattern = re.compile(fr"([^ ])\1{{{str(self.pattern_len - 1)},}}")
 
     def equal_pattern_check(self, line_data_value: str) -> bool:
         """Check if candidate value contain 4 and more same chars or numbers sequences.