feat: basic service to scan vulnerabilities from OSV [experimental] (#…

…2483)

* feat: scan vulnerabilities from OSV

* chore: code quality

* feat: support Debian

* chore: fix existing unittests

* chore: update layers

* chore: code quality

* feat: write vulnerability_check

* chore: refactoring

* chore: 2 small changes

* feat: return numbers

* chore: fix unittests after rebase

* chore: lots of changes

* chore: documentation integrations overview

* chore: fix existing unittests

* chore: documentation (start)

* chore: documentation

* chore: rename Liunx ecosystem to Linux distribution

* chore: fix existing unittests

* fix: slicing for requests and namespace for maven

* chore: name change after code review

* feat: add 2 vulnerability urls

backend/application/access_control/services/roles_permissions.py

@@ -29,6 +29,7 @@ class Permissions(IntEnum):
     Product_Delete = 1103
     Product_Create = 1104
     Product_Import_Observations = 1105
+    Product_Scan_OSV = 1106
 
     Product_Member_View = 1201
     Product_Member_Edit = 1202
@@ -206,6 +207,7 @@ def get_roles_with_permissions():
             Permissions.Product_Group_View,
             Permissions.Product_View,
             Permissions.Product_Import_Observations,
+            Permissions.Product_Scan_OSV,
             Permissions.Product_Member_View,
             Permissions.Product_Authorization_Group_Member_View,
             Permissions.Product_Rule_View,
@@ -225,6 +227,7 @@ def get_roles_with_permissions():
             Permissions.Product_View,
             Permissions.Product_Edit,
             Permissions.Product_Import_Observations,
+            Permissions.Product_Scan_OSV,
             Permissions.Product_Member_View,
             Permissions.Product_Member_Edit,
             Permissions.Product_Member_Delete,
@@ -268,6 +271,7 @@ def get_roles_with_permissions():
             Permissions.Product_Edit,
             Permissions.Product_Delete,
             Permissions.Product_Import_Observations,
+            Permissions.Product_Scan_OSV,
             Permissions.Product_Member_View,
             Permissions.Product_Member_Edit,
             Permissions.Product_Member_Delete,

backend/application/core/api/serializers_product.py

@@ -411,6 +411,11 @@ def validate(self, attrs: dict):  # pylint: disable=too-many-branches
                     "Closed status must not be set when issue tracker type is not Jira"
                 )
 
+        if attrs.get("osv_linux_release") and not attrs.get("osv_linux_distribution"):
+            raise ValidationError(
+                "osv_linux_release cannot be set without osv_linux_distribution"
+            )
+
         return super().validate(attrs)
 
     def validate_product_group(self, product: Product) -> Product:
@@ -608,10 +613,6 @@ class BranchSerializer(ModelSerializer):
     allowed_licenses_count = SerializerMethodField()
     ignored_licenses_count = SerializerMethodField()
 
-    class Meta:
-        model = Branch
-        fields = "__all__"
-
     def validate_purl(self, purl: str) -> str:
         return validate_purl(purl)
 
@@ -657,6 +658,18 @@ def get_allowed_licenses_count(self, obj: Branch) -> int:
     def get_ignored_licenses_count(self, obj: Branch) -> int:
         return obj.ignored_licenses_count
 
+    class Meta:
+        model = Branch
+        fields = "__all__"
+
+    def validate(self, attrs: dict):  # pylint: disable=too-many-branches
+        if attrs.get("osv_linux_release") and not attrs.get("osv_linux_distribution"):
+            raise ValidationError(
+                "osv_linux_release cannot be set without osv_linux_distribution"
+            )
+
+        return super().validate(attrs)
+
 
 class BranchNameSerializer(ModelSerializer):
     name_with_product = SerializerMethodField()

backend/application/core/migrations/0059_branch_osv_linux_distribution_and_more.py

@@ -0,0 +1,68 @@
+# Generated by Django 5.1.5 on 2025-01-29 07:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0058_observation_vulnerability_id_aliases"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="branch",
+            name="osv_linux_distribution",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("AlmaLinux", "AlmaLinux"),
+                    ("Alpine", "Alpine"),
+                    ("Debian", "Debian"),
+                    ("Mageia", "Mageia"),
+                    ("openSUSE", "openSUSE"),
+                    ("Photon OS", "Photon OS"),
+                    ("Red Hat", "Red Hat"),
+                    ("Rocky Linux", "Rocky Linux"),
+                    ("SUSE", "SUSE"),
+                    ("Ubuntu", "Ubuntu"),
+                ],
+                max_length=12,
+            ),
+        ),
+        migrations.AddField(
+            model_name="branch",
+            name="osv_linux_release",
+            field=models.CharField(blank=True, max_length=255),
+        ),
+        migrations.AddField(
+            model_name="product",
+            name="osv_enabled",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="product",
+            name="osv_linux_distribution",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("AlmaLinux", "AlmaLinux"),
+                    ("Alpine", "Alpine"),
+                    ("Debian", "Debian"),
+                    ("Mageia", "Mageia"),
+                    ("openSUSE", "openSUSE"),
+                    ("Photon OS", "Photon OS"),
+                    ("Red Hat", "Red Hat"),
+                    ("Rocky Linux", "Rocky Linux"),
+                    ("SUSE", "SUSE"),
+                    ("Ubuntu", "Ubuntu"),
+                ],
+                max_length=12,
+            ),
+        ),
+        migrations.AddField(
+            model_name="product",
+            name="osv_linux_release",
+            field=models.CharField(blank=True, max_length=255),
+        ),
+    ]

backend/application/core/models.py

@@ -26,7 +26,13 @@
     normalize_observation_fields,
     set_product_flags,
 )
-from application.core.types import Assessment_Status, Severity, Status, VexJustification
+from application.core.types import (
+    Assessment_Status,
+    OSVLinuxDistribution,
+    Severity,
+    Status,
+    VexJustification,
+)
 from application.issue_tracker.types import Issue_Tracker
 from application.licenses.types import License_Policy_Evaluation_Result
 
@@ -108,23 +114,36 @@ class Product(Model):
     issue_tracker_minimum_severity = CharField(
         max_length=12, choices=Severity.SEVERITY_CHOICES, blank=True
     )
+
     last_observation_change = DateTimeField(default=timezone.now)
+
     assessments_need_approval = BooleanField(default=False)
     new_observations_in_review = BooleanField(default=False)
     product_rules_need_approval = BooleanField(default=False)
+
     risk_acceptance_expiry_active = BooleanField(null=True)
     risk_acceptance_expiry_days = IntegerField(
         null=True,
         validators=[MinValueValidator(0), MaxValueValidator(999999)],
         help_text="Days before risk acceptance expires, 0 means no expiry",
     )
+
     license_policy = ForeignKey(
         "licenses.License_Policy",
         on_delete=PROTECT,
         related_name="product",
         null=True,
         blank=True,
     )
+
+    osv_enabled = BooleanField(default=False)
+    osv_linux_distribution = CharField(
+        max_length=12,
+        choices=OSVLinuxDistribution.OSV_LINUX_DISTRIBUTION_CHOICES,
+        blank=True,
+    )
+    osv_linux_release = CharField(max_length=255, blank=True)
+
     has_cloud_resource = BooleanField(default=False)
     has_component = BooleanField(default=False)
     has_docker_image = BooleanField(default=False)
@@ -149,6 +168,12 @@ class Branch(Model):
     housekeeping_protect = BooleanField(default=False)
     purl = CharField(max_length=255, blank=True)
     cpe23 = CharField(max_length=255, blank=True)
+    osv_linux_distribution = CharField(
+        max_length=12,
+        choices=OSVLinuxDistribution.OSV_LINUX_DISTRIBUTION_CHOICES,
+        blank=True,
+    )
+    osv_linux_release = CharField(max_length=255, blank=True)
 
     class Meta:
         unique_together = (

backend/application/core/services/observation.py

@@ -1,6 +1,7 @@
 import hashlib
 from urllib.parse import urlparse
 
+from cvss import CVSS3, CVSS4
 from packageurl import PackageURL
 
 from application.core.types import Severity, Status
@@ -62,6 +63,12 @@ def _get_string_to_hash(observation):  # pylint: disable=too-many-branches
 
 
 def get_current_severity(observation) -> str:
+    if observation.cvss3_vector:
+        observation.cvss3_score = CVSS3(observation.cvss3_vector).base_score
+
+    if observation.cvss4_vector:
+        observation.cvss4_score = CVSS4(observation.cvss4_vector).base_score
+
     if observation.assessment_severity:
         return observation.assessment_severity
 
@@ -212,10 +219,15 @@ def normalize_origin_component(observation):  # pylint: disable=too-many-branche
     else:
         component_parts = observation.origin_component_name_version.split(":")
         if len(component_parts) == 3:
-            observation.origin_component_name = (
-                f"{component_parts[0]}:{component_parts[1]}"
-            )
-            observation.origin_component_version = component_parts[2]
+            if component_parts[0] == observation.origin_component_name:
+                observation.origin_component_version = (
+                    f"{component_parts[1]}:{component_parts[2]}"
+                )
+            else:
+                observation.origin_component_name = (
+                    f"{component_parts[0]}:{component_parts[1]}"
+                )
+                observation.origin_component_version = component_parts[2]
         elif len(component_parts) == 2:
             observation.origin_component_name = component_parts[0]
             observation.origin_component_version = component_parts[1]

backend/application/core/types.py

@@ -127,3 +127,29 @@ class PURL_Type:
         "swid": "SWID",
         "swift": "Swift",
     }
+
+
+class OSVLinuxDistribution:
+    DISTRIBUTION_ALMALINUX = "AlmaLinux"
+    DISTRIBUTION_ALPINE = "Alpine"
+    DISTRIBUTION_DEBIAN = "Debian"
+    DISTRIBUTION_MAGEIA = "Mageia"
+    DISTRIBUTION_OPENSUSE = "openSUSE"
+    DISTRIBUTION_PHOTON_OS = "Photon OS"
+    DISTRIBUTION_REDHAT = "Red Hat"
+    DISTRIBUTION_ROCKY_LINUX = "Rocky Linux"
+    DISTRIBUTION_SUSE = "SUSE"
+    DISTRIBUTION_UBUNTU = "Ubuntu"
+
+    OSV_LINUX_DISTRIBUTION_CHOICES = [
+        (DISTRIBUTION_ALMALINUX, DISTRIBUTION_ALMALINUX),
+        (DISTRIBUTION_ALPINE, DISTRIBUTION_ALPINE),
+        (DISTRIBUTION_DEBIAN, DISTRIBUTION_DEBIAN),
+        (DISTRIBUTION_MAGEIA, DISTRIBUTION_MAGEIA),
+        (DISTRIBUTION_OPENSUSE, DISTRIBUTION_OPENSUSE),
+        (DISTRIBUTION_PHOTON_OS, DISTRIBUTION_PHOTON_OS),
+        (DISTRIBUTION_REDHAT, DISTRIBUTION_REDHAT),
+        (DISTRIBUTION_ROCKY_LINUX, DISTRIBUTION_ROCKY_LINUX),
+        (DISTRIBUTION_SUSE, DISTRIBUTION_SUSE),
+        (DISTRIBUTION_UBUNTU, DISTRIBUTION_UBUNTU),
+    ]

backend/application/import_observations/api/serializers.py

@@ -63,7 +63,7 @@ class ApiImportObservationsByNameRequestSerializer(Serializer):
     kubernetes_cluster = CharField(max_length=255, required=False)
 
 
-class ImportObservationsResponseSerializer(Serializer):
+class FileImportObservationsResponseSerializer(Serializer):
     observations_new = IntegerField()
     observations_updated = IntegerField()
     observations_resolved = IntegerField()
@@ -72,6 +72,12 @@ class ImportObservationsResponseSerializer(Serializer):
     license_components_deleted = IntegerField()
 
 
+class APIImportObservationsResponseSerializer(Serializer):
+    observations_new = IntegerField()
+    observations_updated = IntegerField()
+    observations_resolved = IntegerField()
+
+
 class ApiConfigurationSerializer(ModelSerializer):
     product_data = NestedProductSerializer(source="product", read_only=True)
     test_connection = BooleanField(write_only=True, required=False, default=False)