Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Huggingface and Groq patterns #609

Merged
merged 4 commits into from
Sep 30, 2024
Merged

Huggingface and Groq patterns #609

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2981ba7
Groq API Key
babenek Sep 27, 2024
e3cbe71
Hugging Face User Access Token
babenek Sep 27, 2024
f0cf748
testfix
babenek Sep 27, 2024
c37503e
reorganize patterns ranges in common order 0-9A-Za-z
babenek Sep 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 87 additions & 55 deletions credsweeper/rules/config.yaml
View file
Open in desktop

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions tests/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
from pathlib import Path

# total number of files in test samples
SAMPLES_FILES_COUNT: int = 134
SAMPLES_FILES_COUNT: int = 136

# the lowest value of ML threshold is used to display possible lowest values
NEGLIGIBLE_ML_THRESHOLD = 0.0001

# credentials count after scan
SAMPLES_CRED_COUNT: int = 391
SAMPLES_CRED_LINE_COUNT: int = 409
SAMPLES_CRED_COUNT: int = 396
SAMPLES_CRED_LINE_COUNT: int = 414

# credentials count after post-processing
SAMPLES_POST_CRED_COUNT: int = 344
SAMPLES_POST_CRED_COUNT: int = 349

# with option --doc
SAMPLES_IN_DOC = 425
SAMPLES_IN_DOC = 430

# archived credentials that are not found without --depth
SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 23
Expand Down
135 changes: 135 additions & 0 deletions tests/data/depth_3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7226,6 +7226,60 @@
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "Groq API Key",
"severity": "high",
"confidence": "strong",
"line_data_list": [
{
"line": "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",
"line_num": 1,
"path": "./tests/samples/groq_api_key",
"info": "./tests/samples/groq_api_key|RAW",
"value": "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",
"value_start": 0,
"value_end": 56,
"variable": null,
"variable_start": -2,
"variable_end": -2,
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 5.096509298449433,
"valid": true
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "Groq API Key",
"severity": "high",
"confidence": "strong",
"line_data_list": [
{
"line": "url/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?part",
"line_num": 2,
"path": "./tests/samples/groq_api_key",
"info": "./tests/samples/groq_api_key|RAW",
"value": "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",
"value_start": 4,
"value_end": 60,
"variable": null,
"variable_start": -2,
"variable_end": -2,
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 5.096509298449433,
"valid": true
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
Expand Down Expand Up @@ -7280,6 +7334,87 @@
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "Hugging Face User Access Token",
"severity": "high",
"confidence": "moderate",
"line_data_list": [
{
"line": "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",
"line_num": 1,
"path": "./tests/samples/huggingface",
"info": "./tests/samples/huggingface|RAW",
"value": "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",
"value_start": 0,
"value_end": 37,
"variable": null,
"variable_start": -2,
"variable_end": -2,
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 4.5617684773102365,
"valid": true
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "Hugging Face User Access Token",
"severity": "high",
"confidence": "moderate",
"line_data_list": [
{
"line": "url/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/part",
"line_num": 2,
"path": "./tests/samples/huggingface",
"info": "./tests/samples/huggingface|RAW",
"value": "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",
"value_start": 4,
"value_end": 41,
"variable": null,
"variable_start": -2,
"variable_end": -2,
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 4.636224896287629,
"valid": true
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "Hugging Face User Access Token",
"severity": "high",
"confidence": "moderate",
"line_data_list": [
{
"line": "hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample",
"line_num": 6,
"path": "./tests/samples/huggingface",
"info": "./tests/samples/huggingface|RAW",
"value": "hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample",
"value_start": 0,
"value_end": 37,
"variable": null,
"variable_start": -2,
"variable_end": -2,
"entropy_validation": {
"iterator": "BASE36_CHARS",
"entropy": 3.034378196565712,
"valid": true
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
Expand Down
135 changes: 135 additions & 0 deletions tests/data/doc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11571,6 +11571,60 @@
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "Groq API Key",
"severity": "high",
"confidence": "strong",
"line_data_list": [
{
"line": "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",
"line_num": 1,
"path": "./tests/samples/groq_api_key",
"info": "./tests/samples/groq_api_key|RAW",
"value": "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",
"value_start": 0,
"value_end": 56,
"variable": null,
"variable_start": -2,
"variable_end": -2,
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 5.096509298449433,
"valid": true
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "Groq API Key",
"severity": "high",
"confidence": "strong",
"line_data_list": [
{
"line": "url/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?part",
"line_num": 2,
"path": "./tests/samples/groq_api_key",
"info": "./tests/samples/groq_api_key|RAW",
"value": "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",
"value_start": 4,
"value_end": 60,
"variable": null,
"variable_start": -2,
"variable_end": -2,
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 5.096509298449433,
"valid": true
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
Expand Down Expand Up @@ -11625,6 +11679,87 @@
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "Hugging Face User Access Token",
"severity": "high",
"confidence": "moderate",
"line_data_list": [
{
"line": "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",
"line_num": 1,
"path": "./tests/samples/huggingface",
"info": "./tests/samples/huggingface|RAW",
"value": "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",
"value_start": 0,
"value_end": 37,
"variable": null,
"variable_start": -2,
"variable_end": -2,
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 4.5617684773102365,
"valid": true
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "Hugging Face User Access Token",
"severity": "high",
"confidence": "moderate",
"line_data_list": [
{
"line": "url/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/part",
"line_num": 2,
"path": "./tests/samples/huggingface",
"info": "./tests/samples/huggingface|RAW",
"value": "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",
"value_start": 4,
"value_end": 41,
"variable": null,
"variable_start": -2,
"variable_end": -2,
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 4.636224896287629,
"valid": true
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
"ml_probability": null,
"rule": "Hugging Face User Access Token",
"severity": "high",
"confidence": "moderate",
"line_data_list": [
{
"line": "hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample",
"line_num": 6,
"path": "./tests/samples/huggingface",
"info": "./tests/samples/huggingface|RAW",
"value": "hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample",
"value_start": 0,
"value_end": 37,
"variable": null,
"variable_start": -2,
"variable_end": -2,
"entropy_validation": {
"iterator": "BASE36_CHARS",
"entropy": 3.034378196565712,
"valid": true
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "NOT_AVAILABLE",
Expand Down
Loading
Loading