From dcff5b5593468fd7588885b2e7f7da6af86ce3cc Mon Sep 17 00:00:00 2001 From: Alexey Ovchinnikov Date: Mon, 13 Jan 2025 18:17:39 -0600 Subject: [PATCH] SPDX: adding basic models Adding models of SODX elements to introduce proper SPDX support in a later commit. Signed-off-by: Alexey Ovchinnikov --- cachi2/core/models/sbom.py | 287 ++++++++++++++++++++++++++++++++++++- cachi2/interface/cli.py | 2 +- 2 files changed, 286 insertions(+), 3 deletions(-) diff --git a/cachi2/core/models/sbom.py b/cachi2/core/models/sbom.py index 9a153f362..fd8fc90b2 100644 --- a/cachi2/core/models/sbom.py +++ b/cachi2/core/models/sbom.py @@ -1,12 +1,20 @@ +import datetime +import hashlib +import json +import logging from functools import reduce from itertools import groupby -from typing import Any, Iterable, Literal, Optional +from typing import Annotated, Any, Dict, Iterable, Literal, Optional, Union +from urllib.parse import urlparse import pydantic +from packageurl import PackageURL from cachi2.core.models.property_semantics import Property, PropertySet from cachi2.core.models.validators import unique_sorted +log = logging.getLogger(__name__) + class ExternalReference(pydantic.BaseModel): """An ExternalReference inside an SBOM component.""" @@ -54,7 +62,7 @@ def from_package_dict(cls, package: dict[str, Any]) -> "Component": A Cachi2 package has extra fields which are unnecessary and can cause validation errors. """ - return Component( + return cls( name=package.get("name", None), version=package.get("version", None), purl=package.get("purl", None), @@ -74,6 +82,11 @@ class Metadata(pydantic.BaseModel): tools: list[Tool] = [Tool(vendor="red hat", name="cachi2")] +def spdx_now() -> str: + """Return a time stamp in SPDX-compliant format.""" + return datetime.datetime.now().isoformat()[:-7] + "Z" + + class Sbom(pydantic.BaseModel): """Software bill of materials in the CycloneDX format. @@ -93,6 +106,276 @@ def _unique_components(cls, components: list[Component]) -> list[Component]: return unique_sorted(components, by=lambda component: component.key()) +class SPDXPackageExternalRefReferenceLocatorURI(pydantic.BaseModel): + """SPDX Package External Reference with URI reference locator.""" + + referenceLocator: str + + @pydantic.validator("referenceLocator") + @classmethod + def _validate_uri_reference_locator(cls, referenceLocator: str) -> str: + parsed = urlparse(referenceLocator) + if not (parsed.scheme and (parsed.path or parsed.netloc)): + raise ValueError("Invalid URI reference locator") + return referenceLocator + + +class SPDXPackageExternalRef(pydantic.BaseModel): + """SPDX Package External Reference. + + Compliant to the SPDX specification: + https://spdx.github.io/spdx-spec/v2.3/package-information/#721-external-reference-field + """ + + model_config = pydantic.ConfigDict(frozen=True) + + referenceLocator: str + referenceType: str + referenceCategory: str + + def __hash__(self) -> int: + return hash((self.referenceLocator, self.referenceType, self.referenceCategory)) + + +class SPDXPackageExternalRefSecurity(SPDXPackageExternalRef): + """SPDX Package External Reference for category package-manager. + + Compliant to the SPDX specification: + https://spdx.github.io/spdx-spec/v2.3/package-information/#721-external-reference-field + """ + + referenceCategory: Literal["SECURITY"] + + +class SPDXPackageExternalRefPackageManager(SPDXPackageExternalRef): + """SPDX Package External Reference for category package-manager. + + Compliant to the SPDX specification: + https://spdx.github.io/spdx-spec/v2.3/package-information/#721-external-reference-field + """ + + referenceCategory: Literal["PACKAGE-MANAGER"] + + +class SPDXPackageExternalRefPackageManagerPURL( + SPDXPackageExternalRefPackageManager, SPDXPackageExternalRefReferenceLocatorURI +): + """SPDX Package External Reference for category package-manager and type purl. + + Compliant to the SPDX specification: + https://spdx.github.io/spdx-spec/v2.3/package-information/#721-external-reference-field + """ + + referenceCategory: Literal["PACKAGE-MANAGER"] + referenceType: Literal["purl"] + + +class SPDXPackageExternalRefSecurityPURL( + SPDXPackageExternalRefSecurity, SPDXPackageExternalRefReferenceLocatorURI +): + """SPDX Package External Reference for category package-manager and type purl. + + Compliant to the SPDX specification: + https://spdx.github.io/spdx-spec/v2.3/package-information/#721-external-reference-field + """ + + referenceCategory: Literal["SECURITY"] + referenceType: Literal["cpe23Type"] + + +SPDXPackageExternalRefPackageManagerType = Annotated[ + SPDXPackageExternalRefPackageManagerPURL, + pydantic.Field(discriminator="referenceType"), +] + +SPDXPackageExternalRefSecurityType = Annotated[ + SPDXPackageExternalRefSecurityPURL, + pydantic.Field(discriminator="referenceType"), +] + + +SPDXPackageExternalRefType = Annotated[ + Union[SPDXPackageExternalRefPackageManagerType, SPDXPackageExternalRefSecurityType], + pydantic.Field(discriminator="referenceCategory"), +] + + +class SPDXPackageAnnotation(pydantic.BaseModel): + """SPDX Package Annotation. + + Compliant to the SPDX specification: + https://github.com/spdx/spdx-spec/blob/development/v2.3/schemas/spdx-schema.json#L237 + """ + + model_config = pydantic.ConfigDict(frozen=True) + + annotator: str + annotationDate: str + annotationType: Literal["OTHER", "REVIEW"] + comment: str + + def __hash__(self) -> int: + return hash((self.annotator, self.annotationDate, self.annotationType, self.comment)) + + +class SPDXChecksum(pydantic.BaseModel): + """A basic representation of a checksum entry.""" + + model_config = pydantic.ConfigDict(frozen=True) + + algorithm: str + checksumValue: str + + def __hash__(self) -> int: + return hash(self.algorithm + self.checksumValue) + + +class SPDXFile(pydantic.BaseModel): + """SPDX File. + + Compliant to the SPDX specification: + https://spdx.github.io/spdx-spec/v2.3/package-information/ + + An actual SPDX document generated from a directory or a container image + would usually contain "files" section. This section would contain file + entries of this shape. + """ + + model_config = pydantic.ConfigDict(frozen=True) + + SPDXID: Optional[str] = None + fileName: str + checksums: list[SPDXChecksum] = [] # TODO: flesh out proper cehcksums type + licenseConcluded: str = "NOASSERTION" + copyrightText: str = "" + comment: str = "" + + def __hash__(self) -> int: + return hash( + hash(self.SPDXID) + + hash(self.fileName) + + hash(self.licenseConcluded + self.copyrightText + self.comment) + + sum(hash(c) for c in self.checksums) + ) + + +def _extract_purls(from_refs: list[SPDXPackageExternalRefType]) -> list[str]: + return [ref.referenceLocator for ref in from_refs if ref.referenceType == "purl"] + + +def _parse_purls(purls: list[str]) -> list[PackageURL]: + return [PackageURL.from_string(purl) for purl in purls if purl] + + +class SPDXPackage(pydantic.BaseModel): + """SPDX Package. + + Compliant to the SPDX specification: + https://spdx.github.io/spdx-spec/v2.3/package-information/ + """ + + SPDXID: Optional[str] = None + name: str + versionInfo: Optional[str] = None + externalRefs: list[SPDXPackageExternalRefType] = [] + annotations: list[SPDXPackageAnnotation] = [] + downloadLocation: str = "NOASSERTION" + + def __lt__(self, other: "SPDXPackage") -> bool: + return (self.SPDXID or "") < (other.SPDXID or "") + + def __hash__(self) -> int: + return hash( + hash(self.SPDXID) + + hash(self.name) + + hash(self.versionInfo) + + hash(self.downloadLocation) + + sum(hash(e) for e in self.externalRefs) + + sum(hash(a) for a in self.annotations) + ) + + @staticmethod + def _calculate_package_hash_from_dict(package_dict: Dict[str, Any]) -> str: + return hashlib.sha256(json.dumps(package_dict, sort_keys=True).encode()).hexdigest() + + @pydantic.field_validator("externalRefs") + def _purls_validation( + cls, refs: list[SPDXPackageExternalRefType] + ) -> list[SPDXPackageExternalRefType]: + """Validate that SPDXPackage includes only one purl with the same type, name, version.""" + parsed_purls = _parse_purls(_extract_purls(from_refs=refs)) + unique_purls_parts = set([(p.type, p.name, p.version) for p in parsed_purls]) + if len(unique_purls_parts) > 1: + raise ValueError( + "SPDXPackage includes multiple purls with different (type,name,version) tuple: " + + f"{unique_purls_parts}" + ) + return refs + + @classmethod + def from_package_dict(cls, package: dict[str, Any]) -> "SPDXPackage": + """Create a SPDXPackage from a Cachi2 package dictionary.""" + external_refs = package.get("externalRefs", []) + annotations = [SPDXPackageAnnotation(**an) for an in package.get("annotations", [])] + if package.get("SPDXID") is None: + purls = sorted( + [ + ref["referenceLocator"] + for ref in package["externalRefs"] + if ref["referenceType"] == "purl" + ] + ) + package_hash = cls._calculate_package_hash_from_dict( + { + "name": package["name"], + "version": package.get("versionInfo", None), + "purls": purls, + } + ) + SPDXID = ( + f"SPDXRef-Package-{package['name']}-{package.get('versionInfo', '')}-{package_hash}" + ) + else: + SPDXID = package["SPDXID"] + return cls( + SPDXID=SPDXID, + name=package["name"], + versionInfo=package.get("versionInfo", None), + externalRefs=external_refs, + annotations=annotations, + ) + + +class SPDXCreationInfo(pydantic.BaseModel): + """SPDX Creation Information. + + Compliant to the SPDX specification: + https://spdx.github.io/spdx-spec/v2.3/document-creation-information/ + """ + + creators: list[str] = [] + created: str + + +class SPDXRelation(pydantic.BaseModel): + """SPDX Relationship. + + Compliant to the SPDX specification: + https://spdx.github.io/spdx-spec/v2.3/relationships-between-SPDX-elements/ + """ + + spdxElementId: str + comment: Optional[str] = None + relatedSpdxElement: str + relationshipType: str + + def __hash__(self) -> int: + return hash( + hash(self.spdxElementId + self.relatedSpdxElement + self.relationshipType) + + hash(self.comment) + ) + + def merge_component_properties(components: Iterable[Component]) -> list[Component]: """Sort and de-duplicate components while merging their `properties`.""" components = sorted(components, key=Component.key) diff --git a/cachi2/interface/cli.py b/cachi2/interface/cli.py index 50cffa875..8bba86a22 100644 --- a/cachi2/interface/cli.py +++ b/cachi2/interface/cli.py @@ -16,7 +16,7 @@ from cachi2.core.extras.envfile import EnvFormat, generate_envfile from cachi2.core.models.input import Flag, PackageInput, Request, parse_user_input from cachi2.core.models.output import BuildConfig -from cachi2.core.models.sbom import merge_component_properties, Sbom +from cachi2.core.models.sbom import Sbom, merge_component_properties from cachi2.core.resolver import inject_files_post, resolve_packages, supported_package_managers from cachi2.core.rooted_path import RootedPath from cachi2.interface.logging import LogLevel, setup_logging