Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue/72 new api allow for full fledged processing of protection profiles #466

Draft
wants to merge 8 commits into
base: main
Choose a base branch
from
+2,149 −1,274
Draft

Issue/72 new api allow for full fledged processing of protection profiles #466

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f3a8f96
update deps
Jan 16, 2025
2165a66
refactor auxiliary dataset handling, heuristics computation
Jan 22, 2025
6b0a27c
merge main
Jan 22, 2025
83017b2
fix notebooks
Jan 22, 2025
e6cab19
merge main again
Jan 23, 2025
0610c07
fix broken fips vulnerability notebook
Jan 23, 2025
29ab25f
implement PP processing
Jan 25, 2025
34d89ee
don't delete CCSchemeDatasetHandler when skipping schemes processing
Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions notebooks/fips/vulnerabilities.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
{
"cell_type": "code",
"execution_count": null,
"execution_count": null,
"id": "41674b9c",
"metadata": {
"ExecuteTime": {
Expand Down Expand Up @@ -204,6 +205,7 @@
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.6"
"version": "3.12.6"
},
"vscode": {
"interpreter": {
Expand Down
17 changes: 17 additions & 0 deletions requirements/dev_requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ accessible-pygments==0.0.4
# via pydata-sphinx-theme
aiohappyeyeballs==2.4.0
# via aiohttp
aiohttp==3.10.11
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This merge commit messed this up a bit. Same for the vulnerabilities notebook, but that is now fixed.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, sorry for that. I'll bump the reqs once more before merging, treating also some open PRs.

aiohttp==3.10.11
# via
# datasets
Expand All @@ -12,6 +13,7 @@ alabaster==0.7.13
# via sphinx
annotated-types==0.6.0
# via pydantic
appnope==0.1.4
appnope==0.1.4
# via
# ipykernel
Expand Down Expand Up @@ -84,6 +86,8 @@ datasets==2.15.0
# via sec-certs (./../pyproject.toml)
dateparser==1.2.0
# via sec-certs (./../pyproject.toml)
dateparser==1.2.0
# via sec-certs (./../pyproject.toml)
debugpy==1.8.0
# via ipykernel
decorator==5.1.1
Expand Down Expand Up @@ -223,6 +227,7 @@ murmurhash==1.0.10
# preshed
# spacy
# thinc
mypy==1.13.0
mypy==1.13.0
# via sec-certs (./../pyproject.toml)
mypy-extensions==1.0.0
Expand Down Expand Up @@ -319,6 +324,8 @@ prompt-toolkit==3.0.41
# via ipython
propcache==0.2.0
# via yarl
propcache==0.2.0
# via yarl
psutil==5.9.6
# via
# ipykernel
Expand Down Expand Up @@ -382,6 +389,7 @@ pytest-profiling==1.7.0
python-dateutil==2.8.2
# via
# dateparser
# dateparser
# jupyter-client
# matplotlib
# pandas
Expand All @@ -392,6 +400,9 @@ pytz==2023.3.post1
# via
# dateparser
# pandas
# via
# dateparser
# pandas
pyyaml==6.0.1
# via
# datasets
Expand All @@ -413,6 +424,8 @@ referencing==0.31.0
# jsonschema-specifications
regex==2024.9.11
# via dateparser
regex==2024.9.11
# via dateparser
requests==2.32.0
# via
# datasets
Expand All @@ -427,6 +440,7 @@ rpds-py==0.13.1
# via
# jsonschema
# referencing
ruff==0.7.4
ruff==0.7.4
# via sec-certs (./../pyproject.toml)
scikit-learn==1.5.0
Expand Down Expand Up @@ -557,6 +571,8 @@ tzdata==2023.3
# via pandas
tzlocal==5.2
# via dateparser
tzlocal==5.2
# via dateparser
urllib3==2.2.2
# via
# requests
Expand Down Expand Up @@ -584,6 +600,7 @@ wrapt==1.16.0
# via deprecated
xxhash==3.4.1
# via datasets
yarl==1.17.2
yarl==1.17.2
# via aiohttp
zipp==3.19.1
Expand Down
11 changes: 11 additions & 0 deletions requirements/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
annotated-types==0.6.0
# via pydantic
appnope==0.1.4
appnope==0.1.4
# via
# ipykernel
Expand Down Expand Up @@ -52,6 +53,8 @@ cymem==2.0.8
# thinc
dateparser==1.2.0
# via sec-certs (./../pyproject.toml)
dateparser==1.2.0
# via sec-certs (./../pyproject.toml)
debugpy==1.8.0
# via ipykernel
decorator==5.1.1
Expand Down Expand Up @@ -213,6 +216,7 @@ pytesseract==0.3.10
python-dateutil==2.8.2
# via
# dateparser
# dateparser
# jupyter-client
# matplotlib
# pandas
Expand All @@ -223,6 +227,9 @@ pytz==2023.3.post1
# via
# dateparser
# pandas
# via
# dateparser
# pandas
pyyaml==6.0.1
# via sec-certs (./../pyproject.toml)
pyzmq==25.1.1
Expand All @@ -237,6 +244,8 @@ referencing==0.31.0
# jsonschema-specifications
regex==2024.9.11
# via dateparser
regex==2024.9.11
# via dateparser
requests==2.32.0
# via
# sec-certs (./../pyproject.toml)
Expand Down Expand Up @@ -320,6 +329,8 @@ tzdata==2023.3
# via pandas
tzlocal==5.2
# via dateparser
tzlocal==5.2
# via dateparser
urllib3==2.2.2
# via requests
wasabi==1.1.2
Expand Down
11 changes: 11 additions & 0 deletions requirements/test_requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
annotated-types==0.6.0
# via pydantic
appnope==0.1.4
appnope==0.1.4
# via
# ipykernel
Expand Down Expand Up @@ -56,6 +57,8 @@ cymem==2.0.8
# thinc
dateparser==1.2.0
# via sec-certs (./../pyproject.toml)
dateparser==1.2.0
# via sec-certs (./../pyproject.toml)
debugpy==1.8.0
# via ipykernel
decorator==5.1.1
Expand Down Expand Up @@ -228,6 +231,7 @@ pytest-cov==4.1.0
python-dateutil==2.8.2
# via
# dateparser
# dateparser
# jupyter-client
# matplotlib
# pandas
Expand All @@ -238,6 +242,9 @@ pytz==2023.3.post1
# via
# dateparser
# pandas
# via
# dateparser
# pandas
pyyaml==6.0.1
# via sec-certs (./../pyproject.toml)
pyzmq==25.1.1
Expand All @@ -252,6 +259,8 @@ referencing==0.31.0
# jsonschema-specifications
regex==2024.9.11
# via dateparser
regex==2024.9.11
# via dateparser
requests==2.32.0
# via
# sec-certs (./../pyproject.toml)
Expand Down Expand Up @@ -335,6 +344,8 @@ tzdata==2023.3
# via pandas
tzlocal==5.2
# via dateparser
tzlocal==5.2
# via dateparser
urllib3==2.2.2
# via requests
wasabi==1.1.2
Expand Down
25 changes: 0 additions & 25 deletions src/sec_certs/dataset/cc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,8 @@
link_to_protection_profiles,
)
from sec_certs.heuristics.common import compute_cpe_heuristics, compute_related_cves, compute_transitive_vulnerabilities
from sec_certs.model.cc_matching import CCSchemeMatcher
from sec_certs.sample.cc import CCCertificate
from sec_certs.sample.cc_maintenance_update import CCMaintenanceUpdate
from sec_certs.sample.cc_scheme import EntryType
from sec_certs.sample.protection_profile import ProtectionProfile
from sec_certs.serialization.json import ComplexSerializableType, serialize
from sec_certs.utils import helpers, sanitization
Expand Down Expand Up @@ -808,29 +806,6 @@ def extract_data(self) -> None:
self._extract_pdf_frontpage()
self._extract_pdf_keywords()

@staged(
logger,
"Computing heuristics: Deriving information about laboratories involved in certification.",
)
def _compute_cert_labs(self) -> None:
certs_to_process = [x for x in self if x.state.report.is_ok_to_analyze()]
for cert in certs_to_process:
cert.compute_heuristics_cert_lab()

@staged(logger, "Computing heuristics: Matching scheme data.")
def _compute_scheme_data(self):
for scheme in self.aux_handlers[CCSchemeDatasetHandler].dset:
if certified := scheme.lists.get(EntryType.Certified):
certs = [cert for cert in self if cert.status == "active"]
matches = CCSchemeMatcher.match_all(certified, scheme.country, certs)
for dgst, match in matches.items():
self[dgst].heuristics.scheme_data = match
if archived := scheme.lists.get(EntryType.Archived):
certs = [cert for cert in self if cert.status == "archived"]
matches = CCSchemeMatcher.match_all(archived, scheme.country, certs)
for dgst, match in matches.items():
self[dgst].heuristics.scheme_data = match

def _compute_heuristics_body(self, skip_schemes: bool = False) -> None:
link_to_protection_profiles(self.aux_handlers[ProtectionProfileDatasetHandler].dset, self.certs.values())
compute_cpe_heuristics(self.aux_handlers[CPEDatasetHandler].dset, self.certs.values())
Expand Down
6 changes: 5 additions & 1 deletion src/sec_certs/dataset/fips.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@
FIPSAlgorithmDatasetHandler,
)
from sec_certs.dataset.dataset import Dataset
from sec_certs.heuristics.common import compute_cpe_heuristics, compute_related_cves, compute_transitive_vulnerabilities
from sec_certs.heuristics.common import (
compute_cpe_heuristics,
compute_related_cves,
compute_transitive_vulnerabilities,
)
from sec_certs.heuristics.fips import compute_references
from sec_certs.sample.fips import FIPSCertificate
from sec_certs.serialization.json import ComplexSerializableType, serialize
Expand Down
Loading
You are viewing a condensed version of this merge commit. You can view the full changes here.