updating performance and new script id column for problems

dojo/db_migrations/0219_problem_finding_problem.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.0.8 on 2024-11-26 23:24
+# Generated by Django 5.0.8 on 2024-12-22 19:33
 
 import django.db.models.deletion
 from django.db import migrations, models
@@ -16,7 +16,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('name', models.CharField(help_text='A short name or title for the problem.', max_length=255, verbose_name='Name')),
-                ('description', models.TextField(help_text='Detailed description of the problem.', verbose_name='Description')),
+                ('problem_id', models.TextField(help_text='Problem identifier. This field is used to uniquely identify the problem.', verbose_name='Problem ID')),
                 ('created_at', models.DateTimeField(auto_now_add=True, help_text='Timestamp when this problem was created.', verbose_name='Created At')),
                 ('updated_at', models.DateTimeField(auto_now=True, help_text='Timestamp when this problem was last updated.', verbose_name='Updated At')),
                 ('severity', models.CharField(choices=[('Critical', 'Critical'), ('High', 'High'), ('Medium', 'Medium'), ('Low', 'Low'), ('Info', 'Info')], help_text='The severity level of this problem.', max_length=50, verbose_name='Severity')),

dojo/models.py

@@ -2261,9 +2261,9 @@ class Problem(models.Model):
     name = models.CharField(max_length=255,
                             verbose_name=_("Name"),
                             help_text=_("A short name or title for the problem."))
-    description = models.TextField(
-        verbose_name=_("Description"),
-        help_text=_("Detailed description of the problem."))
+    problem_id = models.TextField(
+        verbose_name=_("Problem ID"),
+        help_text=_("Problem identifier. This field is used to uniquely identify the problem."))
     created_at = models.DateTimeField(auto_now_add=True,
                                        verbose_name=_("Created At"),
                                        help_text=_("Timestamp when this problem was created."))

dojo/problem/helper.py

@@ -3,13 +3,15 @@
 import requests
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
+from django.conf import settings
+
 from dojo.models import Problem, Finding
 
 import logging
 logger = logging.getLogger(__name__)
 
-CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'config.json')
-CACHED_JSON_FILE = os.path.join('/app/media', 'cached_disambiguator.json')
+MEDIA_ROOT = os.getenv('DD_MEDIA_ROOT', '/app/media')
+CACHED_JSON_FILE = os.path.join(MEDIA_ROOT, 'cached_disambiguator.json')
 
 SEVERITY_ORDER = {
     'Critical': 5,
@@ -19,10 +21,6 @@
     'Info': 1
 }
 
-def load_config():
-    with open(CONFIG_FILE, 'r') as f:
-        return json.load(f)
-
 def validate_json(data):
     if not isinstance(data, dict):
         return False
@@ -54,84 +52,68 @@ def save_json_to_cache(data):
     with open(CACHED_JSON_FILE, 'w') as f:
         json.dump(data, f)
 
-def load_json():
-    try:
-        # Disable SSL warnings
-        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
-
-        cached_data = load_cached_json()
-        if cached_data:
-            return cached_data
-
-        # Cache is missing or invalid, download and validate
-        config = load_config()
-        json_url = config.get('json_url')
+def mapping_script_problem_id(mappings_json_findings):
+    script_to_problem_mapping = {
+        script_id: key
+        for key, script_ids in mappings_json_findings.items()
+        for script_id in script_ids
+    }
+    return script_to_problem_mapping
 
-        if json_url:
-            data = download_json(json_url)
-            if validate_json(data):
-                save_json_to_cache(data)
-                return data
+def load_json(check_cash=True):
+    try:
+        if check_cash:
+            cached_data = load_cached_json()
+            if cached_data and validate_json(cached_data):
+                return mapping_script_problem_id(cached_data)
 
-        return {}
+        data = download_json(settings.PROBLEM_MAPPINGS_JSON_URL)
+        if validate_json(data):
+            save_json_to_cache(data)
+            return mapping_script_problem_id(data)
 
     except (requests.RequestException, ValueError, json.JSONDecodeError) as e:
         logger.error('Error loading disambiguator JSON: %s', e)
-        return {}
-
-def extract_script_id(full_id):
-    parts = full_id.split('____')
-    return parts[0] if len(parts) == 2 else None
-
-def find_or_create_problem(finding):
-    data = load_json()
-    script_id = finding.vuln_id_from_tool
+        pass
 
-    valid_ids_mapping = {
-        key: [extract_script_id(full_id) for full_id in script_ids if extract_script_id(full_id)]
-        for key, script_ids in data.items()
-    }
+    return {}
 
-    for key, valid_ids in valid_ids_mapping.items():
-        if script_id in valid_ids:
-            problem = _get_or_update_problem(valid_ids, finding, script_id)
-            if problem:
-                return problem
+def find_or_create_problem(finding, script_to_problem_mapping):
+    problem_id = script_to_problem_mapping.get(finding.vuln_id_from_tool)
+    if problem_id:
+        return _get_or_update_problem(finding, problem_id)
 
     # if the script_id is not in the mapping, create a new one
-    return _get_or_create_problem_by_script_id(script_id, finding)
+    return _get_or_create_problem_by_script_id(finding)
 
-def _get_or_update_problem(valid_ids, finding, script_id):
-    for valid_id in valid_ids:
-        related_finding = Finding.objects.filter(vuln_id_from_tool=valid_id).first()
-        if related_finding and related_finding.problem:
-            problem = related_finding.problem
-            if SEVERITY_ORDER[finding.severity] > SEVERITY_ORDER[problem.severity]:
-                _update_problem(problem, finding.title, finding.description, finding.severity)
-            return problem
+def _get_or_update_problem(finding, problem_id):
+    problem = Problem.objects.filter(problem_id=problem_id).first()
+    if problem:
+        if SEVERITY_ORDER[finding.severity] > SEVERITY_ORDER[problem.severity]:
+            _update_problem(problem, finding.title, finding.severity)
+        return problem
 
     return Problem.objects.create(
         name=finding.title,
-        description=finding.description,
+        problem_id=problem_id,
         severity=finding.severity
     )
 
-def _get_or_create_problem_by_script_id(script_id, finding):
-    related_finding = Finding.objects.filter(vuln_id_from_tool=script_id).first()
+def _get_or_create_problem_by_script_id(finding):
+    related_finding = Finding.objects.filter(vuln_id_from_tool=finding.vuln_id_from_tool).first()
     if related_finding and related_finding.problem:
         problem = related_finding.problem
         if SEVERITY_ORDER[finding.severity] > SEVERITY_ORDER[problem.severity]:
-            _update_problem(problem, finding.title, finding.description, finding.severity)
+            _update_problem(problem, finding.title, finding.severity)
         return problem
 
     return Problem.objects.create(
         name=finding.title,
-        description=finding.description,
+        problem_id=finding.description,
         severity=finding.severity
     )
 
-def _update_problem(problem, name, description, severity):
+def _update_problem(problem, name, severity):
     problem.name = name
-    problem.description = description
     problem.severity = severity
     problem.save()

dojo/problem/update_mappings.py

@@ -0,0 +1,18 @@
+import json
+import os
+import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
+from dojo.celery import app
+from dojo.decorators import dojo_async_task
+from dojo.problem.helper import load_json
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+@dojo_async_task
+@app.task
+def daily_cache_update(**kwargs):
+    logger.info("Starting daily cache update")
+    load_json(check_cash=False)

dojo/problem/views.py

@@ -40,17 +40,13 @@ def add_breadcrumbs(self, request: HttpRequest, context: dict):
 
         return request, context
 
-    def get_ordered_queryset(self, queryset, order_field):
-        if order_field == "findings.count":
-            order_field = "findings_count"
-        elif order_field == "-findings.count":
-            order_field = "-findings_count"
-        return queryset.order_by(order_field) if order_field else queryset.order_by("id")
-
     def get_problems(self, request: HttpRequest):
-        queryset = Problem.objects.all().annotate(findings_count=Count('findings'))
+        queryset = Problem.objects.all().annotate(
+            findings_count=Count('findings'),
+            total_script_ids=Count('findings__vuln_id_from_tool', distinct=True)
+        ).distinct()
         order_field = request.GET.get('o')
-        return self.get_ordered_queryset(queryset, order_field)
+        return queryset.order_by(order_field) if order_field else queryset.order_by("id")
 
     def paginate_queryset(self, queryset, request: HttpRequest):
         page_size = request.GET.get('page_size', 25)  # Default is 25
@@ -77,10 +73,12 @@ class ListOpenProblems(ListProblems):
     def get_problems(self, request: HttpRequest):
         queryset = Problem.objects.filter(
             findings__active=True
-        ).annotate(findings_count=Count('findings')).distinct()
-
+        ).annotate(
+            findings_count=Count('findings'),
+            total_script_ids=Count('findings__vuln_id_from_tool', distinct=True)
+        ).distinct()
         order_field = request.GET.get('o')
-        return self.get_ordered_queryset(queryset, order_field)
+        return queryset.order_by(order_field) if order_field else queryset.order_by("id")
 
 
 class ListClosedProblems(ListProblems):
@@ -89,10 +87,12 @@ class ListClosedProblems(ListProblems):
     def get_problems(self, request: HttpRequest):
         queryset = Problem.objects.annotate(
             active_findings=Count('findings', filter=Q(findings__active=True))
-        ).filter(active_findings=0).annotate(findings_count=Count('findings')).distinct()
-
+        ).filter(active_findings=0).annotate(
+            findings_count=Count('findings'),
+            total_script_ids=Count('findings__vuln_id_from_tool', distinct=True)
+        ).distinct()
         order_field = request.GET.get('o')
-        return self.get_ordered_queryset(queryset, order_field)
+        return queryset.order_by(order_field) if order_field else queryset.order_by("id")
 
 
 
@@ -104,7 +104,7 @@ def get_findings(self, request: HttpRequest):
         problem = Problem.objects.get(pk=self.problem_id)
         queryset = problem.findings.all()
         order_field = request.GET.get('o')
-        return problem.name, self.get_ordered_queryset(queryset, order_field)
+        return problem.name, queryset.order_by(order_field) if order_field else queryset.order_by("id")
 
     def get(self, request: HttpRequest, problem_id: int):
         self.problem_id = problem_id

dojo/settings/settings.dist.py

@@ -1109,7 +1109,7 @@ def saml2_attrib_map_format(dict):
 if len(env("DD_CELERY_BROKER_TRANSPORT_OPTIONS")) > 0:
     CELERY_BROKER_TRANSPORT_OPTIONS = json.loads(env("DD_CELERY_BROKER_TRANSPORT_OPTIONS"))
 
-CELERY_IMPORTS = ("dojo.tools.tool_issue_updater", "dojo.problem.tasks")
+CELERY_IMPORTS = ("dojo.tools.tool_issue_updater", "dojo.problem.update_mappings")
 
 # Celery beat scheduled tasks
 CELERY_BEAT_SCHEDULE = {
@@ -1148,7 +1148,7 @@ def saml2_attrib_map_format(dict):
         "schedule": timedelta(minutes=1),
     },
     "daily-cache-update": {
-        "task": "dojo.problem.tasks.daily_cache_update",
+        "task": "dojo.problem.update_mappings.daily_cache_update",
         "schedule": crontab(minute=0, hour=0),  # every day at midnight
     },
     # 'jira_status_reconciliation': {
@@ -1748,6 +1748,15 @@ def saml2_attrib_map_format(dict):
 # see https://github.com/laymonage/django-jsonfield-backport
 SILENCED_SYSTEM_CHECKS = ["django_jsonfield_backport.W001"]
 
+# Problem utilizes mappings from a JSON file to disambiguate Findings.
+# The JSON file should have the following structure:
+# {
+#     "problem_id_1": ["script_id_1", "script_id_2", "script_id_3"],
+#     "problem_id_2": ["script_id_4", "script_id_5"],
+#     "problem_id_3": ["script_id_6"]
+# }
+PROBLEM_MAPPINGS_JSON_URL = "https://homepages.dcc.ufmg.br/~leonardooliveira/disambiguator.json"
+
 VULNERABILITY_URLS = {
     "CVE": "https://nvd.nist.gov/vuln/detail/",
     "GHSA": "https://github.com/advisories/",

dojo/templates/dojo/problem_findings_list_snippet.html

@@ -119,7 +119,6 @@ <h3 class="has-filters">
         {% endblock %}
     </script>
     <script type="application/javascript">
-        // DataTables Setup
         $.fn.dataTable.ext.order['severity-asc'] = function (settings, col) {
             return this.api().column(col, { order: 'index' }).nodes().map(function (td, i) {
                 var severity = $(td).data('severity');
@@ -141,41 +140,34 @@ <h3 class="has-filters">
         };
 
         $(document).ready(function() {
-            // Inicializa a DataTable para a tabela com o ID 'open_problems'
             $('#open_problems').DataTable({
-                // Callback para mostrar e esconder popovers ao passar o mouse
                 drawCallback: function(){
                     $('#open_findings .has-popover').hover(
-                        function() { $(this).popover('show'); }, // hover
-                        function() { $(this).popover('hide'); } // unhover
+                        function() { $(this).popover('show'); },
+                        function() { $(this).popover('hide'); }
                     );
                 },
-                // Reordenação das colunas
                 colReorder: true,
-                // Definição das colunas a partir da variável datatables_columns
                 columns: datatables_columns,
-                // Desativa a ordenação automática
                 order: [],
-                // Configurações de colunas
                 columnDefs: [
                     {
                         targets: 'severity-sort',
-                        orderDataType: 'severity-asc' // Aplica a ordenação personalizada na coluna de severidade
+                        orderDataType: 'severity-asc'
                     },
                 ],
-                // Configurações de exibição da DataTable
                 dom: 'Bfrtip',
-                paging: false, // Desativa a paginação
-                info: false,   // Desativa as informações de página
+                paging: false,
+                info: false,
                 buttons: [
                     {
                         extend: 'colvis',
-                        columns: ':not(.noVis)' // Permite visibilidade das colunas, exceto as que têm a classe 'noVis'
+                        columns: ':not(.noVis)'
                     },
                     {
                         extend: 'copy',
                         exportOptions: {
-                            columns: [0, 1, 2, 3, 4, 5], // Define as colunas que serão exportadas
+                            columns: [0, 1, 2, 3, 4, 5],
                             stripHtml: true,
                             stripNewlines: true,
                             trim: true,