From fe44aaa44d591762b81f46a2ea7538d202d143d5 Mon Sep 17 00:00:00 2001
From: magajh <maria.jaimes@edunext.co>
Date: Tue, 14 Jan 2025 08:12:33 -0400
Subject: [PATCH 01/60] feat: add API to collect data
---
eox_core/api/data/data_collector/__init__.py | 0
eox_core/api/data/data_collector/tasks.py | 72 ++++++++++++++++
eox_core/api/data/data_collector/urls.py | 11 +++
eox_core/api/data/data_collector/utils.py | 82 ++++++++++++++++++
.../api/data/data_collector/v1/__init__.py | 0
.../api/data/data_collector/v1/serializers.py | 50 +++++++++++
eox_core/api/data/data_collector/v1/urls.py | 9 ++
eox_core/api/data/data_collector/v1/views.py | 84 +++++++++++++++++++
eox_core/api/data/v1/urls.py | 1 +
eox_core/utils.py | 28 +++++++
10 files changed, 337 insertions(+)
create mode 100644 eox_core/api/data/data_collector/__init__.py
create mode 100644 eox_core/api/data/data_collector/tasks.py
create mode 100644 eox_core/api/data/data_collector/urls.py
create mode 100644 eox_core/api/data/data_collector/utils.py
create mode 100644 eox_core/api/data/data_collector/v1/__init__.py
create mode 100644 eox_core/api/data/data_collector/v1/serializers.py
create mode 100644 eox_core/api/data/data_collector/v1/urls.py
create mode 100644 eox_core/api/data/data_collector/v1/views.py
diff --git a/eox_core/api/data/data_collector/__init__.py b/eox_core/api/data/data_collector/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/eox_core/api/data/data_collector/tasks.py b/eox_core/api/data/data_collector/tasks.py
new file mode 100644
index 000000000..894e687ec
--- /dev/null
+++ b/eox_core/api/data/data_collector/tasks.py
@@ -0,0 +1,72 @@
+"""
+Async task for generating reports by executing database queries
+and posting the results to the Shipyard API.
+"""
+
+from celery import shared_task, Task
+from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, serialize_data
+import yaml
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ReportTask(Task):
+ """
+ Custom task class to handle report generation with an on_failure hook.
+ """
+ def on_failure(self, exc, task_id, args, kwargs, einfo):
+ """
+ Called when the task has exhausted all retries.
+
+ Args:
+ exc (Exception): The exception raised.
+ task_id (str): The ID of the failed task.
+ args (tuple): The positional arguments for the task.
+ kwargs (dict): The keyword arguments for the task.
+ einfo (ExceptionInfo): Exception information.
+ """
+ logger.error(f"Task {task_id} failed after retries. Exception: {exc}. Could not collect data.")
+
+
+@shared_task(bind=True)
+def generate_report(self, destination_url, query_file_content, token_generation_url, current_host):
+ """
+ Async task to generate a report:
+ 1. Reads queries from the provided query file content.
+ 2. Executes each query against the database.
+ 3. Sends the results to the Shipyard API.
+
+ Args:
+ self (Task): The Celery task instance.
+ query_file_content (str): The content of the query file in YAML format.
+
+ Raises:
+ Retry: If an error occurs, the task retries up to 3 times with a 60-second delay.
+ """
+ try:
+ queries = yaml.safe_load(query_file_content).get("queries", [])
+ if not queries:
+ logger.warning("No queries found in the provided file. Task will exit.")
+ return
+
+ report_data = {}
+ for query in queries:
+ query_name = query.get("name")
+ query_sql = query.get("query")
+ logger.info(f"Executing query: {query_name}")
+ try:
+ result = execute_query(query_sql)
+
+ serialized_result = serialize_data(result)
+ report_data[query_name] = serialized_result
+ except Exception as e:
+ logger.error(f"Failed to execute query '{query_name}': {e}")
+ continue
+
+ post_data_to_api(destination_url, report_data, token_generation_url, current_host)
+
+ logger.info("Report generation task completed successfully.")
+ except Exception as e:
+ logger.error(f"An error occurred in the report generation task: {e}. Retrying")
+ raise self.retry(exc=e, countdown=60, max_retries=3)
diff --git a/eox_core/api/data/data_collector/urls.py b/eox_core/api/data/data_collector/urls.py
new file mode 100644
index 000000000..2d71ac519
--- /dev/null
+++ b/eox_core/api/data/data_collector/urls.py
@@ -0,0 +1,11 @@
+"""
+URLs for the Microsite API
+"""
+from django.urls import include, re_path
+
+app_name = 'eox_core' # pylint: disable=invalid-name
+
+
+urlpatterns = [ # pylint: disable=invalid-name
+ re_path(r'^v1/', include('eox_core.api.data.data_collector.v1.urls', namespace='eox-data-api-collector-v1')),
+]
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
new file mode 100644
index 000000000..d51e81a23
--- /dev/null
+++ b/eox_core/api/data/data_collector/utils.py
@@ -0,0 +1,82 @@
+"""
+Utility functions for report generation, including query execution
+and integration with the Shipyard API.
+"""
+
+import yaml
+from django.db import connection
+import requests
+from django.conf import settings
+from datetime import datetime
+import logging
+
+from eox_core.utils import get_access_token
+
+logger = logging.getLogger(__name__)
+
+
+def execute_query(sql_query):
+ """
+ Execute a raw SQL query and return the results in a structured format.
+
+ Args:
+ sql_query (str): The raw SQL query to execute.
+
+ Returns:
+ list or dict: Structured query results.
+ """
+ with connection.cursor() as cursor:
+ cursor.execute(sql_query)
+ rows = cursor.fetchall()
+ # If the query returns more than one column, return rows as is.
+ if cursor.description:
+ columns = [col[0] for col in cursor.description]
+ if len(columns) == 1:
+ return [row[0] for row in rows] # Return single-column results as a list
+ return [dict(zip(columns, row)) for row in rows] # Multi-column results as a list of dicts
+ return rows
+
+
+def serialize_data(data):
+ """
+ Recursively serialize data, converting datetime objects to strings.
+
+ Args:
+ data (dict or list): The data to serialize.
+
+ Returns:
+ dict or list: The serialized data with datetime objects as strings.
+ """
+ if isinstance(data, dict):
+ return {key: serialize_data(value) for key, value in data.items()}
+ elif isinstance(data, list):
+ return [serialize_data(item) for item in data]
+ elif isinstance(data, datetime):
+ return data.isoformat()
+ return data
+
+
+def post_data_to_api(api_url, report_data, token_generation_url, current_host):
+ """
+ Sends the generated report data to the Shipyard API.
+
+ Args:
+ report_data (dict): The data to be sent to the Shipyard API.
+
+ Raises:
+ Exception: If the API request fails.
+ """
+ token = get_access_token(
+ token_generation_url,
+ settings.EOX_CORE_SAVE_DATA_API_CLIENT_ID,
+ settings.EOX_CORE_SAVE_DATA_API_CLIENT_SECRET,
+ )
+ headers = {
+ "Authorization": f"Bearer {token}",
+ "Content-Type": "application/json",
+ }
+ payload = {"instance_domain":current_host, "data": report_data}
+ response = requests.post(api_url, json=payload, headers=headers)
+
+ if not response.ok:
+ raise Exception(f"Failed to post data to Shipyard API: {response.content}")
diff --git a/eox_core/api/data/data_collector/v1/__init__.py b/eox_core/api/data/data_collector/v1/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/eox_core/api/data/data_collector/v1/serializers.py b/eox_core/api/data/data_collector/v1/serializers.py
new file mode 100644
index 000000000..5109b26e0
--- /dev/null
+++ b/eox_core/api/data/data_collector/v1/serializers.py
@@ -0,0 +1,50 @@
+from rest_framework import serializers
+
+class DataCollectorSerializer(serializers.Serializer):
+ """
+ Serializer for the DataCollectorView API.
+
+ Validates the incoming payload for the data collection endpoint.
+
+ Fields:
+ query_file_content (str): The content of the query file in YAML format.
+ query_file_url (str): A public URL pointing to the query file.
+ destination_url (str): The URL where the results should be sent.
+ """
+ query_file_content = serializers.CharField(
+ required=False,
+ allow_blank=True,
+ help_text="Content of the query file in YAML format."
+ )
+ query_file_url = serializers.URLField(
+ required=False,
+ allow_blank=True,
+ help_text="Public URL pointing to the query file."
+ )
+ destination_url = serializers.URLField(
+ required=True,
+ help_text="The API endpoint where the results will be sent."
+ )
+ token_generation_url = serializers.URLField(
+ required=True,
+ help_text="The API endpoint where the results will be sent."
+ )
+
+ def validate(self, data):
+ """
+ Custom validation to ensure either 'query_file_content' or 'query_file_url' is provided.
+
+ Args:
+ data (dict): The validated data.
+
+ Returns:
+ dict: The validated data if valid.
+
+ Raises:
+ serializers.ValidationError: If neither 'query_file_content' nor 'query_file_url' is provided.
+ """
+ if not data.get("query_file_content") and not data.get("query_file_url"):
+ raise serializers.ValidationError(
+ "Either 'query_file_content' or 'query_file_url' must be provided."
+ )
+ return data
diff --git a/eox_core/api/data/data_collector/v1/urls.py b/eox_core/api/data/data_collector/v1/urls.py
new file mode 100644
index 000000000..a815d1d03
--- /dev/null
+++ b/eox_core/api/data/data_collector/v1/urls.py
@@ -0,0 +1,9 @@
+"""_"""
+from django.urls import path
+from eox_core.api.data.data_collector.v1.views import DataCollectorView
+
+app_name = "data_collector"
+
+urlpatterns = [
+ path("collect-data/", DataCollectorView.as_view(), name="collect_data"),
+]
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
new file mode 100644
index 000000000..84b0cafe4
--- /dev/null
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -0,0 +1,84 @@
+import logging
+import requests
+import yaml
+from rest_framework.views import APIView
+from rest_framework.response import Response
+from rest_framework import status, permissions
+from django.conf import settings
+from eox_core.api.data.data_collector.tasks import generate_report
+
+from rest_framework.permissions import BasePermission
+from rest_framework.authentication import get_authorization_header
+from django.conf import settings
+from eox_core.api.data.data_collector.v1.serializers import DataCollectorSerializer
+
+logger = logging.getLogger(__name__)
+
+
+class IsGitHubAction(BasePermission):
+ """
+ Permission class to allow access only if the request contains a valid GitHub Action token.
+ """
+ def has_permission(self, request, view):
+ auth_header = get_authorization_header(request).decode('utf-8')
+ auth_token = settings.EOX_CORE_DATA_COLLECT_AUTH_TOKEN
+ if auth_header and auth_header == f"Bearer {auth_token}":
+ return True
+ return False
+
+
+class DataCollectorView(APIView):
+ """
+ API view to handle data collection requests.
+
+ This view:
+ - Validates input using DataCollectorSerializer.
+ - Triggers an async task to execute queries and send results to a specified destination.
+ """
+ # Allow JWT Auth
+ permission_classes = [IsGitHubAction]
+
+ def post(self, request):
+ """
+ Handles POST requests to collect data.
+
+ Args:
+ request (HttpRequest): The incoming request.
+
+ Returns:
+ Response: A success or error message.
+ """
+ serializer = DataCollectorSerializer(data=request.data)
+
+ if serializer.is_valid():
+ validated_data = serializer.validated_data
+ query_file_content = validated_data.get("query_file_content")
+ query_file_url = validated_data.get("query_file_url")
+ destination_url = validated_data.get("destination_url")
+ token_generation_url = validated_data.get("token_generation_url")
+ current_host = request.get_host() #Remove trailing slash and http
+
+ # If the query file content is not provided, fetch it from the URL
+ if not query_file_content and query_file_url:
+ try:
+ response = requests.get(query_file_url)
+ if response.status_code == 200:
+ query_file_content = response.text
+ else:
+ return Response(
+ {"error": "Failed to fetch query file from the provided URL."},
+ status=status.HTTP_400_BAD_REQUEST
+ )
+ except Exception as e:
+ return Response(
+ {"error": f"An error occurred while fetching the query file: {str(e)}"},
+ status=status.HTTP_400_BAD_REQUEST
+ )
+
+ generate_report.delay(destination_url, query_file_content, token_generation_url, current_host)
+ return Response(
+ {"message": "Data collection task has been initiated successfully."},
+ status=status.HTTP_202_ACCEPTED
+ )
+
+ return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
diff --git a/eox_core/api/data/v1/urls.py b/eox_core/api/data/v1/urls.py
index 8ca7a9f2d..292589f20 100644
--- a/eox_core/api/data/v1/urls.py
+++ b/eox_core/api/data/v1/urls.py
@@ -11,4 +11,5 @@
urlpatterns = [ # pylint: disable=invalid-name
re_path(r'^v1/', include((ROUTER.urls, 'eox_core'), namespace='eox-data-api-v1')),
re_path(r'^v1/tasks/(?P<task_id>.*)$', CeleryTasksStatus.as_view(), name="celery-data-api-tasks"),
+ re_path(r'^', include('eox_core.api.data.data_collector.urls', namespace='eox-data-api-collector')),
]
diff --git a/eox_core/utils.py b/eox_core/utils.py
index 8a3764d0c..0f7b5a346 100644
--- a/eox_core/utils.py
+++ b/eox_core/utils.py
@@ -5,6 +5,7 @@
import datetime
import hashlib
import re
+import requests
from django.conf import settings
from django.contrib.sites.models import Site
@@ -165,3 +166,30 @@ def get_or_create_site_from_oauth_app_uris(redirect_uris):
return sites_qs.first()
return Site.objects.create(domain=domain, name=domain)
+
+
+def get_access_token(
+ token_url,
+ client_id,
+ client_secret,
+ grant_type="client_credentials"
+ ):
+ """
+ Fetch an access token from a service OAuth2 API.
+
+ Returns:
+ str: The access token.
+ Raises:
+ Exception: If the token request fails.
+ """
+ response = requests.post(
+ token_url,
+ data={
+ "grant_type": grant_type,
+ "client_id": client_id,
+ "client_secret": client_secret,
+ },
+ )
+ if response.ok:
+ return response.json().get("access_token")
+ raise Exception("Failed to obtain access token for API.")
From 53c01c689b06d19e0fe843811a5d673fc5f9f12d Mon Sep 17 00:00:00 2001
From: magajh <maria.jaimes@edunext.co>
Date: Wed, 22 Jan 2025 01:42:50 -0400
Subject: [PATCH 02/60] feat: update version and changelog
---
CHANGELOG.md | 6 ++++++
eox_core/__init__.py | 2 +-
setup.cfg | 2 +-
3 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b19d2c505..81d134526 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,12 @@ Please do not update the unreleased notes.
<!-- Content should be placed here -->
+## [v11.2.0](https://github.com/eduNEXT/eox-core/compare/v11.1.0...v11.2.0) - (2025-01-20)
+
+### Added
+
+- API to collect data and generate reports.
+
## [v11.1.0](https://github.com/eduNEXT/eox-core/compare/v11.0.0...v11.1.0) - (2024-11-21)
### Changed
diff --git a/eox_core/__init__.py b/eox_core/__init__.py
index d3fa81ffd..1ff125f2a 100644
--- a/eox_core/__init__.py
+++ b/eox_core/__init__.py
@@ -1,4 +1,4 @@
"""
Init for main eox-core app
"""
-__version__ = '11.1.0'
+__version__ = '11.2.0'
diff --git a/setup.cfg b/setup.cfg
index 796ff33d8..82d5ef267 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 11.1.0
+current_version = 11.2.0
commit = False
tag = False
From 12ed527688eb8b14f786084221bcfaa29c41360c Mon Sep 17 00:00:00 2001
From: magajh <maria.jaimes@edunext.co>
Date: Wed, 22 Jan 2025 06:42:49 -0400
Subject: [PATCH 03/60] fix: process results
---
eox_core/api/data/data_collector/tasks.py | 5 +++--
eox_core/api/data/data_collector/utils.py | 16 ++++++++++++++++
2 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/eox_core/api/data/data_collector/tasks.py b/eox_core/api/data/data_collector/tasks.py
index 894e687ec..00cc7ecc8 100644
--- a/eox_core/api/data/data_collector/tasks.py
+++ b/eox_core/api/data/data_collector/tasks.py
@@ -4,7 +4,7 @@
"""
from celery import shared_task, Task
-from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, serialize_data
+from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, serialize_data, process_query_results
import yaml
import logging
@@ -59,7 +59,8 @@ def generate_report(self, destination_url, query_file_content, token_generation_
result = execute_query(query_sql)
serialized_result = serialize_data(result)
- report_data[query_name] = serialized_result
+ processed_result = process_query_results(serialized_result)
+ report_data[query_name] = processed_result
except Exception as e:
logger.error(f"Failed to execute query '{query_name}': {e}")
continue
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index d51e81a23..e4f7bf9f5 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -56,6 +56,22 @@ def serialize_data(data):
return data
+def process_query_results(raw_result):
+ """
+ Process the raw result of a query.
+
+ Args:
+ raw_result: The result from the SQL query (list, scalar, or dictionary).
+
+ Returns:
+ The processed result, extracting scalar values from single-item lists,
+ or returning the original value for more complex data structures.
+ """
+ if isinstance(raw_result, list) and len(raw_result) == 1:
+ return raw_result[0]
+ return raw_result
+
+
def post_data_to_api(api_url, report_data, token_generation_url, current_host):
"""
Sends the generated report data to the Shipyard API.
From 38587d1f353fd38df911a69e2a4b26027680beca Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 15:08:24 -0500
Subject: [PATCH 04/60] chore: change IsGitHubAction class name
---
eox_core/api/data/data_collector/v1/views.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index 84b0cafe4..f3ec27eb8 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -15,7 +15,7 @@
logger = logging.getLogger(__name__)
-class IsGitHubAction(BasePermission):
+class DatacollectorPermission(BasePermission):
"""
Permission class to allow access only if the request contains a valid GitHub Action token.
"""
@@ -36,7 +36,7 @@ class DataCollectorView(APIView):
- Triggers an async task to execute queries and send results to a specified destination.
"""
# Allow JWT Auth
- permission_classes = [IsGitHubAction]
+ permission_classes = [DatacollectorPermission]
def post(self, request):
"""
From 37ca44ec312d25f617a21a1470b666d1613b8a4a Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 15:10:12 -0500
Subject: [PATCH 05/60] chore: block non-SELECT queries
---
eox_core/api/data/data_collector/utils.py | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index e4f7bf9f5..03e509845 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -9,6 +9,7 @@
from django.conf import settings
from datetime import datetime
import logging
+import re
from eox_core.utils import get_access_token
@@ -24,7 +25,17 @@ def execute_query(sql_query):
Returns:
list or dict: Structured query results.
+
+ Raises:
+ ValueError: If the query is not a SELECT statement.
"""
+ # Normalize query (remove whitespace and convert to uppercase)
+ normalized_query = sql_query.strip().upper()
+
+ # Verify that the query begins with "SELECT"
+ if not re.match(r"^SELECT\s", normalized_query):
+ raise ValueError("Only SELECT queries are allowed.")
+
with connection.cursor() as cursor:
cursor.execute(sql_query)
rows = cursor.fetchall()
From 88081c295bbcf1bc907ffc4f62fa79cc4ace4389 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 15:11:24 -0500
Subject: [PATCH 06/60] feat: add setting AGGREGATED_DATA_COLLECTOR_API_ENABLED
to disable data collector endpoint
---
eox_core/api/data/data_collector/v1/views.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index f3ec27eb8..b2a8f90c5 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -48,6 +48,12 @@ def post(self, request):
Returns:
Response: A success or error message.
"""
+ if not getattr(settings, "EOX_CORE_DATA_COLLECTOR_ENABLED", False):
+ return Response(
+ {"error": "This endpoint is currently disabled."},
+ status=status.HTTP_403_FORBIDDEN
+ )
+
serializer = DataCollectorSerializer(data=request.data)
if serializer.is_valid():
From cebbc1eec232b1818bb995fd28536ea74e08f8da Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 15:19:12 -0500
Subject: [PATCH 07/60] refactor: execute all predefined queries and remove
query selection
---
eox_core/api/data/data_collector/queries.py | 58 +++++++++++++++++++
eox_core/api/data/data_collector/tasks.py | 19 ++----
.../api/data/data_collector/v1/serializers.py | 31 ----------
eox_core/api/data/data_collector/v1/views.py | 22 +------
4 files changed, 65 insertions(+), 65 deletions(-)
create mode 100644 eox_core/api/data/data_collector/queries.py
diff --git a/eox_core/api/data/data_collector/queries.py b/eox_core/api/data/data_collector/queries.py
new file mode 100644
index 000000000..09034d990
--- /dev/null
+++ b/eox_core/api/data/data_collector/queries.py
@@ -0,0 +1,58 @@
+PREDEFINED_QUERIES = {
+ "Usuarios total": """
+ SELECT COUNT(*) FROM auth_user as au;
+ """,
+ "Usuarios activos mes anterior": """
+ SELECT
+ YEAR(cs.modified) AS 'Year',
+ MONTH(cs.modified) AS 'Month',
+ COUNT(DISTINCT cs.student_id) AS 'Active Users'
+ FROM
+ courseware_studentmodule AS cs
+ WHERE
+ cs.modified >= DATE_FORMAT(DATE_SUB(CURDATE(), INTERVAL 1 MONTH), '%Y-%m-01')
+ AND cs.modified < DATE_FORMAT(CURDATE(), '%Y-%m-01')
+ GROUP BY
+ YEAR(cs.modified), MONTH(cs.modified);
+ """,
+ "Usuarios activos 7 dias": """
+ SELECT COUNT(DISTINCT cs.student_id)
+ FROM courseware_studentmodule AS cs
+ WHERE cs.modified >= DATE_SUB(CURDATE(), INTERVAL 7 DAY);
+ """,
+ "Cantidad de cursos creados": """
+ SELECT COUNT(*) FROM course_overviews_courseoverview as coc;
+ """,
+ "Cursos activos": """
+ SELECT COUNT(*)
+ FROM course_overviews_courseoverview as coc
+ WHERE coc.start <= NOW() AND (coc.end >= NOW() OR coc.end IS NULL);
+ """,
+ "Cursos con certificado activo": """
+ SELECT COUNT(DISTINCT coc.id)
+ FROM course_overviews_courseoverview AS coc
+ JOIN certificates_generatedcertificate AS cg
+ ON coc.id = cg.course_id;
+ """,
+ "Inscripciones mes anterior": """
+ SELECT
+ YEAR(sc.created) AS 'Year',
+ MONTH(sc.created) AS 'Month',
+ COUNT(DISTINCT sc.id) AS 'Enrollments'
+ FROM
+ student_courseenrollment AS sc
+ WHERE
+ sc.created >= DATE_FORMAT(DATE_SUB(CURDATE(), INTERVAL 1 MONTH), '%Y-%m-01')
+ AND sc.created < DATE_FORMAT(CURDATE(), '%Y-%m-01')
+ GROUP BY
+ YEAR(sc.created), MONTH(sc.created);
+ """,
+ "Inscripciones 7 dias": """
+ SELECT COUNT(DISTINCT sc.id)
+ FROM student_courseenrollment AS sc
+ WHERE sc.created >= DATE_SUB(CURDATE(), INTERVAL 7 DAY);
+ """,
+ "Certificados generados": """
+ SELECT COUNT(*) FROM certificates_generatedcertificate as cg;
+ """,
+}
diff --git a/eox_core/api/data/data_collector/tasks.py b/eox_core/api/data/data_collector/tasks.py
index 00cc7ecc8..9ad5d3a1a 100644
--- a/eox_core/api/data/data_collector/tasks.py
+++ b/eox_core/api/data/data_collector/tasks.py
@@ -5,6 +5,7 @@
from celery import shared_task, Task
from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, serialize_data, process_query_results
+from eox_core.api.data.data_collector.queries import PREDEFINED_QUERIES
import yaml
import logging
@@ -30,30 +31,22 @@ def on_failure(self, exc, task_id, args, kwargs, einfo):
@shared_task(bind=True)
-def generate_report(self, destination_url, query_file_content, token_generation_url, current_host):
+def generate_report(self, destination_url, token_generation_url, current_host):
"""
Async task to generate a report:
- 1. Reads queries from the provided query file content.
- 2. Executes each query against the database.
- 3. Sends the results to the Shipyard API.
+ 1. Executes all predefined queries.
+ 2. Sends the results to the Shipyard API.
Args:
self (Task): The Celery task instance.
- query_file_content (str): The content of the query file in YAML format.
+ destination_url (str): URL to send the results.
Raises:
Retry: If an error occurs, the task retries up to 3 times with a 60-second delay.
"""
try:
- queries = yaml.safe_load(query_file_content).get("queries", [])
- if not queries:
- logger.warning("No queries found in the provided file. Task will exit.")
- return
-
report_data = {}
- for query in queries:
- query_name = query.get("name")
- query_sql = query.get("query")
+ for query_name, query_sql in PREDEFINED_QUERIES.items():
logger.info(f"Executing query: {query_name}")
try:
result = execute_query(query_sql)
diff --git a/eox_core/api/data/data_collector/v1/serializers.py b/eox_core/api/data/data_collector/v1/serializers.py
index 5109b26e0..59e24e5c1 100644
--- a/eox_core/api/data/data_collector/v1/serializers.py
+++ b/eox_core/api/data/data_collector/v1/serializers.py
@@ -7,20 +7,8 @@ class DataCollectorSerializer(serializers.Serializer):
Validates the incoming payload for the data collection endpoint.
Fields:
- query_file_content (str): The content of the query file in YAML format.
- query_file_url (str): A public URL pointing to the query file.
destination_url (str): The URL where the results should be sent.
"""
- query_file_content = serializers.CharField(
- required=False,
- allow_blank=True,
- help_text="Content of the query file in YAML format."
- )
- query_file_url = serializers.URLField(
- required=False,
- allow_blank=True,
- help_text="Public URL pointing to the query file."
- )
destination_url = serializers.URLField(
required=True,
help_text="The API endpoint where the results will be sent."
@@ -29,22 +17,3 @@ class DataCollectorSerializer(serializers.Serializer):
required=True,
help_text="The API endpoint where the results will be sent."
)
-
- def validate(self, data):
- """
- Custom validation to ensure either 'query_file_content' or 'query_file_url' is provided.
-
- Args:
- data (dict): The validated data.
-
- Returns:
- dict: The validated data if valid.
-
- Raises:
- serializers.ValidationError: If neither 'query_file_content' nor 'query_file_url' is provided.
- """
- if not data.get("query_file_content") and not data.get("query_file_url"):
- raise serializers.ValidationError(
- "Either 'query_file_content' or 'query_file_url' must be provided."
- )
- return data
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index b2a8f90c5..eac4291b8 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -35,7 +35,6 @@ class DataCollectorView(APIView):
- Validates input using DataCollectorSerializer.
- Triggers an async task to execute queries and send results to a specified destination.
"""
- # Allow JWT Auth
permission_classes = [DatacollectorPermission]
def post(self, request):
@@ -58,30 +57,11 @@ def post(self, request):
if serializer.is_valid():
validated_data = serializer.validated_data
- query_file_content = validated_data.get("query_file_content")
- query_file_url = validated_data.get("query_file_url")
destination_url = validated_data.get("destination_url")
token_generation_url = validated_data.get("token_generation_url")
current_host = request.get_host() #Remove trailing slash and http
- # If the query file content is not provided, fetch it from the URL
- if not query_file_content and query_file_url:
- try:
- response = requests.get(query_file_url)
- if response.status_code == 200:
- query_file_content = response.text
- else:
- return Response(
- {"error": "Failed to fetch query file from the provided URL."},
- status=status.HTTP_400_BAD_REQUEST
- )
- except Exception as e:
- return Response(
- {"error": f"An error occurred while fetching the query file: {str(e)}"},
- status=status.HTTP_400_BAD_REQUEST
- )
-
- generate_report.delay(destination_url, query_file_content, token_generation_url, current_host)
+ generate_report.delay(destination_url, token_generation_url, current_host)
return Response(
{"message": "Data collection task has been initiated successfully."},
status=status.HTTP_202_ACCEPTED
From ad988d1b340e51d96ebdc1d5dbfb582ad744aedd Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 15:21:21 -0500
Subject: [PATCH 08/60] docs: add docstring
---
eox_core/api/data/data_collector/v1/views.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index eac4291b8..e3f531113 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -31,6 +31,9 @@ class DataCollectorView(APIView):
"""
API view to handle data collection requests.
+ This feature is **100% opt-in**, meaning that it will only work if explicitly enabled
+ by the system administrator. No data is extracted or sent without consent.
+
This view:
- Validates input using DataCollectorSerializer.
- Triggers an async task to execute queries and send results to a specified destination.
From 3219c932fe9a580699f8a7b40539136cfe87d207 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 18:01:27 -0500
Subject: [PATCH 09/60] chore: rename EOX_CORE_DATA_COLLECTOR_ENABLED setting
to AGGREGATED_DATA_COLLECTOR_API_ENABLED
---
eox_core/api/data/data_collector/v1/views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index e3f531113..8ee192913 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -50,7 +50,7 @@ def post(self, request):
Returns:
Response: A success or error message.
"""
- if not getattr(settings, "EOX_CORE_DATA_COLLECTOR_ENABLED", False):
+ if not getattr(settings, "AGGREGATED_DATA_COLLECTOR_API_ENABLED", False):
return Response(
{"error": "This endpoint is currently disabled."},
status=status.HTTP_403_FORBIDDEN
From b9e0f25d522e0db85e2cd456417f7bdb1fe04685 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 18:13:28 -0500
Subject: [PATCH 10/60] chore: fix indentation and formatting issues
---
eox_core/api/data/data_collector/utils.py | 8 ++++----
eox_core/api/data/data_collector/v1/serializers.py | 1 +
eox_core/api/data/data_collector/v1/views.py | 2 +-
eox_core/utils.py | 8 ++------
4 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index 03e509845..9ae2f5a4c 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -19,13 +19,13 @@
def execute_query(sql_query):
"""
Execute a raw SQL query and return the results in a structured format.
-
+
Args:
sql_query (str): The raw SQL query to execute.
-
+
Returns:
list or dict: Structured query results.
-
+
Raises:
ValueError: If the query is not a SELECT statement.
"""
@@ -102,7 +102,7 @@ def post_data_to_api(api_url, report_data, token_generation_url, current_host):
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
}
- payload = {"instance_domain":current_host, "data": report_data}
+ payload = {"instance_domain": current_host, "data": report_data}
response = requests.post(api_url, json=payload, headers=headers)
if not response.ok:
diff --git a/eox_core/api/data/data_collector/v1/serializers.py b/eox_core/api/data/data_collector/v1/serializers.py
index 59e24e5c1..7df6f29e3 100644
--- a/eox_core/api/data/data_collector/v1/serializers.py
+++ b/eox_core/api/data/data_collector/v1/serializers.py
@@ -1,5 +1,6 @@
from rest_framework import serializers
+
class DataCollectorSerializer(serializers.Serializer):
"""
Serializer for the DataCollectorView API.
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index 8ee192913..4e6199b4d 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -62,7 +62,7 @@ def post(self, request):
validated_data = serializer.validated_data
destination_url = validated_data.get("destination_url")
token_generation_url = validated_data.get("token_generation_url")
- current_host = request.get_host() #Remove trailing slash and http
+ current_host = request.get_host() # Remove trailing slash and http
generate_report.delay(destination_url, token_generation_url, current_host)
return Response(
diff --git a/eox_core/utils.py b/eox_core/utils.py
index 0f7b5a346..1e3a1e42c 100644
--- a/eox_core/utils.py
+++ b/eox_core/utils.py
@@ -168,12 +168,8 @@ def get_or_create_site_from_oauth_app_uris(redirect_uris):
return Site.objects.create(domain=domain, name=domain)
-def get_access_token(
- token_url,
- client_id,
- client_secret,
- grant_type="client_credentials"
- ):
+def get_access_token(token_url, client_id,
+ client_secret, grant_type="client_credentials"):
"""
Fetch an access token from a service OAuth2 API.
From d22fc41d9bb75d3c18c7bf7112163cf9c3aad0f1 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 18:23:01 -0500
Subject: [PATCH 11/60] chore: fix indentation and formatting issues
---
eox_core/utils.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/eox_core/utils.py b/eox_core/utils.py
index 1e3a1e42c..aab072ab0 100644
--- a/eox_core/utils.py
+++ b/eox_core/utils.py
@@ -168,8 +168,9 @@ def get_or_create_site_from_oauth_app_uris(redirect_uris):
return Site.objects.create(domain=domain, name=domain)
-def get_access_token(token_url, client_id,
- client_secret, grant_type="client_credentials"):
+def get_access_token(
+ token_url, client_id, client_secret, grant_type="client_credentials"
+ ):
"""
Fetch an access token from a service OAuth2 API.
From f71c0b12d6772a9c4c27f9801c2bfdeab0c285f9 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 18:26:43 -0500
Subject: [PATCH 12/60] chore: fix indentation and formatting issues
---
eox_core/utils.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/eox_core/utils.py b/eox_core/utils.py
index aab072ab0..0f7b5a346 100644
--- a/eox_core/utils.py
+++ b/eox_core/utils.py
@@ -169,7 +169,10 @@ def get_or_create_site_from_oauth_app_uris(redirect_uris):
def get_access_token(
- token_url, client_id, client_secret, grant_type="client_credentials"
+ token_url,
+ client_id,
+ client_secret,
+ grant_type="client_credentials"
):
"""
Fetch an access token from a service OAuth2 API.
From 9594c5c268bceb9487f9bb0691f0ef39acf010d3 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 18:34:33 -0500
Subject: [PATCH 13/60] chore: fix indentation and formatting issues
---
eox_core/utils.py | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/eox_core/utils.py b/eox_core/utils.py
index 0f7b5a346..8befcdb9c 100644
--- a/eox_core/utils.py
+++ b/eox_core/utils.py
@@ -168,12 +168,7 @@ def get_or_create_site_from_oauth_app_uris(redirect_uris):
return Site.objects.create(domain=domain, name=domain)
-def get_access_token(
- token_url,
- client_id,
- client_secret,
- grant_type="client_credentials"
- ):
+def get_access_token(token_url, client_id, client_secret, grant_type = "client_credentials"):
"""
Fetch an access token from a service OAuth2 API.
From feffa89924e85c980f5518e32203a8bc351d159d Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 24 Feb 2025 18:39:22 -0500
Subject: [PATCH 14/60] chore: fix indentation and formatting issues
---
eox_core/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/utils.py b/eox_core/utils.py
index 8befcdb9c..0efa471d7 100644
--- a/eox_core/utils.py
+++ b/eox_core/utils.py
@@ -168,7 +168,7 @@ def get_or_create_site_from_oauth_app_uris(redirect_uris):
return Site.objects.create(domain=domain, name=domain)
-def get_access_token(token_url, client_id, client_secret, grant_type = "client_credentials"):
+def get_access_token(token_url, client_id, client_secret, grant_type="client_credentials"):
"""
Fetch an access token from a service OAuth2 API.
From f6f676ce31aad9eaa0186850a580ba0ca20cf8f2 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 17:43:36 -0500
Subject: [PATCH 15/60] chore: queries file refactor
---
eox_core/api/data/data_collector/queries.py | 149 ++++++++++++--------
eox_core/utils.py | 5 +-
2 files changed, 97 insertions(+), 57 deletions(-)
diff --git a/eox_core/api/data/data_collector/queries.py b/eox_core/api/data/data_collector/queries.py
index 09034d990..8085efea8 100644
--- a/eox_core/api/data/data_collector/queries.py
+++ b/eox_core/api/data/data_collector/queries.py
@@ -1,58 +1,95 @@
+TOTAL_USERS_QUERY = """
+SELECT
+ COUNT(*)
+FROM
+ auth_user as au;
+"""
+ACTIVE_USERS_LAST_MONTH_QUERY = """
+SELECT
+ YEAR(cs.modified) AS 'Year',
+ MONTH(cs.modified) AS 'Month',
+ COUNT(DISTINCT cs.student_id) AS 'Active Users'
+FROM
+ courseware_studentmodule AS cs
+WHERE
+ cs.modified >= DATE_FORMAT(DATE_SUB(CURDATE(), INTERVAL 1 MONTH), '%Y-%m-01')
+ AND cs.modified < DATE_FORMAT(CURDATE(), '%Y-%m-01')
+GROUP BY
+ YEAR(cs.modified), MONTH(cs.modified);
+"""
+ACTIVE_USERS_LAST_7_DAYS_QUERY = """
+SELECT
+ COUNT(DISTINCT cs.student_id)
+FROM
+ courseware_studentmodule AS cs
+WHERE
+ cs.modified >= DATE_SUB(CURDATE(), INTERVAL 7 DAY);
+"""
+TOTAL_COURSES_CREATED_QUERY = """
+SELECT
+ COUNT(*)
+FROM
+ course_overviews_courseoverview as coc;
+"""
+# This query counts the number of CourseOverviews objects that started before
+# now and have not yet ended
+ACTIVE_COURSES_COUNT_QUERY = """
+SELECT
+ COUNT(*)
+FROM
+ course_overviews_courseoverview as coc
+WHERE
+ coc.start <= NOW()
+ AND (
+ coc.end >= NOW()
+ OR coc.end IS NULL
+ );
+"""
+COURSES_WITH_ACTIVE_CERTIFICATES_QUERY = """
+SELECT
+ COUNT(DISTINCT coc.id)
+FROM
+ course_overviews_courseoverview AS coc
+JOIN
+ certificates_generatedcertificate AS cg
+ON
+ coc.id = cg.course_id;
+"""
+ENROLLMENTS_LAST_MONTH_QUERY = """
+SELECT
+ YEAR(sc.created) AS 'Year',
+ MONTH(sc.created) AS 'Month',
+ COUNT(DISTINCT sc.id) AS 'Enrollments'
+FROM
+ student_courseenrollment AS sc
+WHERE
+ sc.created >= DATE_FORMAT(DATE_SUB(CURDATE(), INTERVAL 1 MONTH), '%Y-%m-01')
+ AND sc.created < DATE_FORMAT(CURDATE(), '%Y-%m-01')
+GROUP BY
+ YEAR(sc.created), MONTH(sc.created);
+"""
+ENROLLMENTS_LAST_7_DAYS_QUERY = """
+SELECT
+ COUNT(DISTINCT sc.id)
+FROM
+ student_courseenrollment AS sc
+WHERE
+ sc.created >= DATE_SUB(CURDATE(), INTERVAL 7 DAY);
+"""
+CERTIFICATES_ISSUED_QUERY = """
+SELECT
+ COUNT(*)
+FROM
+ certificates_generatedcertificate as cg;
+"""
PREDEFINED_QUERIES = {
- "Usuarios total": """
- SELECT COUNT(*) FROM auth_user as au;
- """,
- "Usuarios activos mes anterior": """
- SELECT
- YEAR(cs.modified) AS 'Year',
- MONTH(cs.modified) AS 'Month',
- COUNT(DISTINCT cs.student_id) AS 'Active Users'
- FROM
- courseware_studentmodule AS cs
- WHERE
- cs.modified >= DATE_FORMAT(DATE_SUB(CURDATE(), INTERVAL 1 MONTH), '%Y-%m-01')
- AND cs.modified < DATE_FORMAT(CURDATE(), '%Y-%m-01')
- GROUP BY
- YEAR(cs.modified), MONTH(cs.modified);
- """,
- "Usuarios activos 7 dias": """
- SELECT COUNT(DISTINCT cs.student_id)
- FROM courseware_studentmodule AS cs
- WHERE cs.modified >= DATE_SUB(CURDATE(), INTERVAL 7 DAY);
- """,
- "Cantidad de cursos creados": """
- SELECT COUNT(*) FROM course_overviews_courseoverview as coc;
- """,
- "Cursos activos": """
- SELECT COUNT(*)
- FROM course_overviews_courseoverview as coc
- WHERE coc.start <= NOW() AND (coc.end >= NOW() OR coc.end IS NULL);
- """,
- "Cursos con certificado activo": """
- SELECT COUNT(DISTINCT coc.id)
- FROM course_overviews_courseoverview AS coc
- JOIN certificates_generatedcertificate AS cg
- ON coc.id = cg.course_id;
- """,
- "Inscripciones mes anterior": """
- SELECT
- YEAR(sc.created) AS 'Year',
- MONTH(sc.created) AS 'Month',
- COUNT(DISTINCT sc.id) AS 'Enrollments'
- FROM
- student_courseenrollment AS sc
- WHERE
- sc.created >= DATE_FORMAT(DATE_SUB(CURDATE(), INTERVAL 1 MONTH), '%Y-%m-01')
- AND sc.created < DATE_FORMAT(CURDATE(), '%Y-%m-01')
- GROUP BY
- YEAR(sc.created), MONTH(sc.created);
- """,
- "Inscripciones 7 dias": """
- SELECT COUNT(DISTINCT sc.id)
- FROM student_courseenrollment AS sc
- WHERE sc.created >= DATE_SUB(CURDATE(), INTERVAL 7 DAY);
- """,
- "Certificados generados": """
- SELECT COUNT(*) FROM certificates_generatedcertificate as cg;
- """,
+ "Total Users": TOTAL_USERS_QUERY,
+ "Active Users Last Month": ACTIVE_USERS_LAST_MONTH_QUERY,
+ "Active users in the last 7 days": ACTIVE_USERS_LAST_7_DAYS_QUERY,
+ "Total Courses Created": TOTAL_COURSES_CREATED_QUERY,
+ "Active Courses Count": ACTIVE_COURSES_COUNT_QUERY,
+ "Courses With Active Certificates": COURSES_WITH_ACTIVE_CERTIFICATES_QUERY,
+ "Enrollments Last Month": ENROLLMENTS_LAST_MONTH_QUERY,
+ "Enrollments Last 7 Days": ENROLLMENTS_LAST_7_DAYS_QUERY,
+ "Certificates Issued": CERTIFICATES_ISSUED_QUERY,
}
diff --git a/eox_core/utils.py b/eox_core/utils.py
index 0efa471d7..cf9261b4c 100644
--- a/eox_core/utils.py
+++ b/eox_core/utils.py
@@ -184,7 +184,10 @@ def get_access_token(token_url, client_id, client_secret, grant_type="client_cre
"client_id": client_id,
"client_secret": client_secret,
},
+ timeout=10
)
if response.ok:
return response.json().get("access_token")
- raise Exception("Failed to obtain access token for API.")
+ raise requests.exceptions.HTTPError(
+ f"Failed to obtain access token: {response.status_code} - {response.text}"
+ )
From 37594de5b5c79caf8b94bbd48ecb88dfc10e17e1 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 17:55:28 -0500
Subject: [PATCH 16/60] docs: update queries docstring
---
eox_core/api/data/data_collector/queries.py | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/eox_core/api/data/data_collector/queries.py b/eox_core/api/data/data_collector/queries.py
index 8085efea8..34cac92c4 100644
--- a/eox_core/api/data/data_collector/queries.py
+++ b/eox_core/api/data/data_collector/queries.py
@@ -1,9 +1,21 @@
+"""
+This module contains SQL queries to retrieve platform-related metrics.
+
+These queries are used to extract information about user activity, course
+engagement, and certificate issuance. The data retrieval is conditioned on
+the feature being enabled and the necessary authentication credentials
+being set.
+"""
+
+# This query counts the total number of users in the platform.
TOTAL_USERS_QUERY = """
SELECT
COUNT(*)
FROM
auth_user as au;
"""
+# This query counts the number of unique active users in the last month.
+# A user is considered active if they have modified a student module within the last month.
ACTIVE_USERS_LAST_MONTH_QUERY = """
SELECT
YEAR(cs.modified) AS 'Year',
@@ -17,6 +29,8 @@
GROUP BY
YEAR(cs.modified), MONTH(cs.modified);
"""
+# This query counts the number of unique active users in the last 7 days.
+# A user is considered active if they have modified a student module within the last 7 days.
ACTIVE_USERS_LAST_7_DAYS_QUERY = """
SELECT
COUNT(DISTINCT cs.student_id)
@@ -25,6 +39,7 @@
WHERE
cs.modified >= DATE_SUB(CURDATE(), INTERVAL 7 DAY);
"""
+# This query counts the total number of courses created on the platform.
TOTAL_COURSES_CREATED_QUERY = """
SELECT
COUNT(*)
@@ -45,6 +60,7 @@
OR coc.end IS NULL
);
"""
+# This query counts the number of courses that have at least one issued certificate.
COURSES_WITH_ACTIVE_CERTIFICATES_QUERY = """
SELECT
COUNT(DISTINCT coc.id)
@@ -55,6 +71,7 @@
ON
coc.id = cg.course_id;
"""
+# This query counts the number of new enrollments in the last month.
ENROLLMENTS_LAST_MONTH_QUERY = """
SELECT
YEAR(sc.created) AS 'Year',
@@ -68,6 +85,7 @@
GROUP BY
YEAR(sc.created), MONTH(sc.created);
"""
+# This query counts the number of new enrollments in the last 7 days.
ENROLLMENTS_LAST_7_DAYS_QUERY = """
SELECT
COUNT(DISTINCT sc.id)
@@ -82,6 +100,7 @@
FROM
certificates_generatedcertificate as cg;
"""
+# This query counts the total number of certificates issued on the platform.
PREDEFINED_QUERIES = {
"Total Users": TOTAL_USERS_QUERY,
"Active Users Last Month": ACTIVE_USERS_LAST_MONTH_QUERY,
From 8f75361b3ba00c9cb7937a1cc1a78d487ec78886 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 18:00:05 -0500
Subject: [PATCH 17/60] refactor: remove unused class
---
eox_core/api/data/data_collector/tasks.py | 20 +-------------------
1 file changed, 1 insertion(+), 19 deletions(-)
diff --git a/eox_core/api/data/data_collector/tasks.py b/eox_core/api/data/data_collector/tasks.py
index 9ad5d3a1a..9e2f309f4 100644
--- a/eox_core/api/data/data_collector/tasks.py
+++ b/eox_core/api/data/data_collector/tasks.py
@@ -3,7 +3,7 @@
and posting the results to the Shipyard API.
"""
-from celery import shared_task, Task
+from celery import shared_task
from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, serialize_data, process_query_results
from eox_core.api.data.data_collector.queries import PREDEFINED_QUERIES
import yaml
@@ -12,24 +12,6 @@
logger = logging.getLogger(__name__)
-class ReportTask(Task):
- """
- Custom task class to handle report generation with an on_failure hook.
- """
- def on_failure(self, exc, task_id, args, kwargs, einfo):
- """
- Called when the task has exhausted all retries.
-
- Args:
- exc (Exception): The exception raised.
- task_id (str): The ID of the failed task.
- args (tuple): The positional arguments for the task.
- kwargs (dict): The keyword arguments for the task.
- einfo (ExceptionInfo): Exception information.
- """
- logger.error(f"Task {task_id} failed after retries. Exception: {exc}. Could not collect data.")
-
-
@shared_task(bind=True)
def generate_report(self, destination_url, token_generation_url, current_host):
"""
From f72843219f10fa4a8fa105bb4da6110e965d4a30 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 18:03:25 -0500
Subject: [PATCH 18/60] chore: COUNTDOWN and MAX_RETRIES constants
---
eox_core/api/data/data_collector/tasks.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/eox_core/api/data/data_collector/tasks.py b/eox_core/api/data/data_collector/tasks.py
index 9e2f309f4..cf2fd3b52 100644
--- a/eox_core/api/data/data_collector/tasks.py
+++ b/eox_core/api/data/data_collector/tasks.py
@@ -11,6 +11,8 @@
logger = logging.getLogger(__name__)
+COUNTDOWN = 60
+MAX_RETRIES = 3
@shared_task(bind=True)
def generate_report(self, destination_url, token_generation_url, current_host):
@@ -45,4 +47,4 @@ def generate_report(self, destination_url, token_generation_url, current_host):
logger.info("Report generation task completed successfully.")
except Exception as e:
logger.error(f"An error occurred in the report generation task: {e}. Retrying")
- raise self.retry(exc=e, countdown=60, max_retries=3)
+ raise self.retry(exc=e, countdown=COUNTDOWN, max_retries=MAX_RETRIES)
From 36b31045c3c31312047f8f04e9775d07c7361c86 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 18:06:38 -0500
Subject: [PATCH 19/60] docs: update docstring urls.py
---
eox_core/api/data/data_collector/urls.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/eox_core/api/data/data_collector/urls.py b/eox_core/api/data/data_collector/urls.py
index 2d71ac519..ee575a587 100644
--- a/eox_core/api/data/data_collector/urls.py
+++ b/eox_core/api/data/data_collector/urls.py
@@ -1,5 +1,8 @@
"""
-URLs for the Microsite API
+URL configuration for the Microsite API.
+
+This module defines the URL patterns for the Microsite API,
+including versioned endpoints for data_collector.
"""
from django.urls import include, re_path
From b0a5b5190d1a59b62fa9184c0491e52214981b88 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 18:22:35 -0500
Subject: [PATCH 20/60] refactor: merge serialize_data and
process_query_results into post_process_query_results
---
eox_core/api/data/data_collector/tasks.py | 5 ++--
eox_core/api/data/data_collector/utils.py | 36 ++++++++---------------
2 files changed, 14 insertions(+), 27 deletions(-)
diff --git a/eox_core/api/data/data_collector/tasks.py b/eox_core/api/data/data_collector/tasks.py
index cf2fd3b52..549a8672e 100644
--- a/eox_core/api/data/data_collector/tasks.py
+++ b/eox_core/api/data/data_collector/tasks.py
@@ -4,7 +4,7 @@
"""
from celery import shared_task
-from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, serialize_data, process_query_results
+from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, post_process_query_results
from eox_core.api.data.data_collector.queries import PREDEFINED_QUERIES
import yaml
import logging
@@ -35,8 +35,7 @@ def generate_report(self, destination_url, token_generation_url, current_host):
try:
result = execute_query(query_sql)
- serialized_result = serialize_data(result)
- processed_result = process_query_results(serialized_result)
+ processed_result = post_process_query_results(result)
report_data[query_name] = processed_result
except Exception as e:
logger.error(f"Failed to execute query '{query_name}': {e}")
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index 9ae2f5a4c..62b35f6cb 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -47,42 +47,30 @@ def execute_query(sql_query):
return [dict(zip(columns, row)) for row in rows] # Multi-column results as a list of dicts
return rows
-
-def serialize_data(data):
+def post_process_query_results(data):
"""
- Recursively serialize data, converting datetime objects to strings.
+ Cleans and processes query results by:
+ - Serializing datetime objects into strings.
+ - Extracting scalar values from single-item lists.
+ - Returning structured data for further use.
Args:
- data (dict or list): The data to serialize.
+ data (dict, list, datetime, or scalar): The query result data.
Returns:
- dict or list: The serialized data with datetime objects as strings.
+ dict, list, or scalar: The processed query result.
"""
if isinstance(data, dict):
- return {key: serialize_data(value) for key, value in data.items()}
+ return {key: post_process_query_results(value) for key, value in data.items()}
elif isinstance(data, list):
- return [serialize_data(item) for item in data]
+ # If it's a list with one item, return just the item
+ if len(data) == 1:
+ return post_process_query_results(data[0])
+ return [post_process_query_results(item) for item in data]
elif isinstance(data, datetime):
return data.isoformat()
return data
-
-def process_query_results(raw_result):
- """
- Process the raw result of a query.
-
- Args:
- raw_result: The result from the SQL query (list, scalar, or dictionary).
-
- Returns:
- The processed result, extracting scalar values from single-item lists,
- or returning the original value for more complex data structures.
- """
- if isinstance(raw_result, list) and len(raw_result) == 1:
- return raw_result[0]
- return raw_result
-
-
def post_data_to_api(api_url, report_data, token_generation_url, current_host):
"""
Sends the generated report data to the Shipyard API.
From 9315b351e90d37ee0f31aadb0036e26f8373e4ba Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 18:26:49 -0500
Subject: [PATCH 21/60] refactor: remove validate Only SELECT queries are
allowed
---
eox_core/api/data/data_collector/utils.py | 10 ----------
1 file changed, 10 deletions(-)
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index 62b35f6cb..e6a251f8f 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -25,17 +25,7 @@ def execute_query(sql_query):
Returns:
list or dict: Structured query results.
-
- Raises:
- ValueError: If the query is not a SELECT statement.
"""
- # Normalize query (remove whitespace and convert to uppercase)
- normalized_query = sql_query.strip().upper()
-
- # Verify that the query begins with "SELECT"
- if not re.match(r"^SELECT\s", normalized_query):
- raise ValueError("Only SELECT queries are allowed.")
-
with connection.cursor() as cursor:
cursor.execute(sql_query)
rows = cursor.fetchall()
From e8bf0a0ea6371154c3b94906535bad10ac4bb0cb Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 18:32:40 -0500
Subject: [PATCH 22/60] docs: add execute_query example
---
eox_core/api/data/data_collector/utils.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index e6a251f8f..ddf7d5ca8 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -25,6 +25,13 @@ def execute_query(sql_query):
Returns:
list or dict: Structured query results.
+
+ Example:
+ >>> execute_query("SELECT id, username FROM auth_user WHERE is_active = 1;")
+ [
+ {"id": 1, "username": "john_doe"},
+ {"id": 2, "username": "jane_doe"}
+ ]
"""
with connection.cursor() as cursor:
cursor.execute(sql_query)
From e257d345ac091ade3d1b17ec04e9bf70f951cd49 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 18:35:21 -0500
Subject: [PATCH 23/60] chore: rename EOX_CORE_SAVE_DATA_API_CLIENT_ID and
EOX_CORE_SAVE_DATA_API_CLIENT_SECRET
---
eox_core/api/data/data_collector/utils.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index ddf7d5ca8..e3d7b0e67 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -80,8 +80,8 @@ def post_data_to_api(api_url, report_data, token_generation_url, current_host):
"""
token = get_access_token(
token_generation_url,
- settings.EOX_CORE_SAVE_DATA_API_CLIENT_ID,
- settings.EOX_CORE_SAVE_DATA_API_CLIENT_SECRET,
+ settings.EOX_CORE_AGGREGATED_DATA_API_CLIENT_ID,
+ settings.EOX_CORE_AGGREGATED_DATA_API_CLIENT_SECRET,
)
headers = {
"Authorization": f"Bearer {token}",
From 7b8ac8eeb51957ea76097782750c147ea17f0d29 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 18:37:48 -0500
Subject: [PATCH 24/60] docs: add serializers docstring
---
eox_core/api/data/data_collector/v1/serializers.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/eox_core/api/data/data_collector/v1/serializers.py b/eox_core/api/data/data_collector/v1/serializers.py
index 7df6f29e3..dcd49c0be 100644
--- a/eox_core/api/data/data_collector/v1/serializers.py
+++ b/eox_core/api/data/data_collector/v1/serializers.py
@@ -5,10 +5,13 @@ class DataCollectorSerializer(serializers.Serializer):
"""
Serializer for the DataCollectorView API.
- Validates the incoming payload for the data collection endpoint.
-
+ This serializer is used to validate the payload for the data collection endpoint.
+ It ensures that the required URLs are provided for sending collected data and
+ generating authentication tokens.
+
Fields:
destination_url (str): The URL where the results should be sent.
+ token_generation_url (str): The API endpoint used to generate authentication tokens.
"""
destination_url = serializers.URLField(
required=True,
From fab57936d55ca7c6d126bc68d82d1195300bd90d Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 22:36:05 -0500
Subject: [PATCH 25/60] chore: Updated DataCollectorView to retrieve
destination_url and token_generation_url from settings.
---
.../api/data/data_collector/v1/serializers.py | 23 ---------------
eox_core/api/data/data_collector/v1/views.py | 28 +++++++++----------
2 files changed, 14 insertions(+), 37 deletions(-)
delete mode 100644 eox_core/api/data/data_collector/v1/serializers.py
diff --git a/eox_core/api/data/data_collector/v1/serializers.py b/eox_core/api/data/data_collector/v1/serializers.py
deleted file mode 100644
index dcd49c0be..000000000
--- a/eox_core/api/data/data_collector/v1/serializers.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from rest_framework import serializers
-
-
-class DataCollectorSerializer(serializers.Serializer):
- """
- Serializer for the DataCollectorView API.
-
- This serializer is used to validate the payload for the data collection endpoint.
- It ensures that the required URLs are provided for sending collected data and
- generating authentication tokens.
-
- Fields:
- destination_url (str): The URL where the results should be sent.
- token_generation_url (str): The API endpoint used to generate authentication tokens.
- """
- destination_url = serializers.URLField(
- required=True,
- help_text="The API endpoint where the results will be sent."
- )
- token_generation_url = serializers.URLField(
- required=True,
- help_text="The API endpoint where the results will be sent."
- )
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index 4e6199b4d..e61719ac7 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -10,7 +10,6 @@
from rest_framework.permissions import BasePermission
from rest_framework.authentication import get_authorization_header
from django.conf import settings
-from eox_core.api.data.data_collector.v1.serializers import DataCollectorSerializer
logger = logging.getLogger(__name__)
@@ -35,7 +34,6 @@ class DataCollectorView(APIView):
by the system administrator. No data is extracted or sent without consent.
This view:
- - Validates input using DataCollectorSerializer.
- Triggers an async task to execute queries and send results to a specified destination.
"""
permission_classes = [DatacollectorPermission]
@@ -55,19 +53,21 @@ def post(self, request):
{"error": "This endpoint is currently disabled."},
status=status.HTTP_403_FORBIDDEN
)
+
+ destination_url = getattr(settings, "EOX_CORE_DATA_COLLECT_DESTINATION_URL", None)
+ token_generation_url = getattr(settings, "EOX_CORE_DATA_COLLECT_TOKEN_URL", None)
- serializer = DataCollectorSerializer(data=request.data)
-
- if serializer.is_valid():
- validated_data = serializer.validated_data
- destination_url = validated_data.get("destination_url")
- token_generation_url = validated_data.get("token_generation_url")
- current_host = request.get_host() # Remove trailing slash and http
-
- generate_report.delay(destination_url, token_generation_url, current_host)
+ if not destination_url or not token_generation_url:
+ logger.error("Data collection settings are missing.")
return Response(
- {"message": "Data collection task has been initiated successfully."},
- status=status.HTTP_202_ACCEPTED
+ {"error": "Data collection settings are not properly configured."},
+ status=status.HTTP_500_INTERNAL_SERVER_ERROR
)
- return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+ current_host = request.get_host() # Remove trailing slash and http
+
+ generate_report.delay(destination_url, token_generation_url, current_host)
+ return Response(
+ {"message": "Data collection task has been initiated successfully."},
+ status=status.HTTP_202_ACCEPTED
+ )
From 9e1d7cc75d59431334ba3c2fabc7b0b87a2d694d Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Tue, 25 Feb 2025 22:43:57 -0500
Subject: [PATCH 26/60] refactor: move DatacollectorPermission to a dedicated
permissions.py file
---
.../api/data/data_collector/v1/permissions.py | 18 +++++++++++++++++
eox_core/api/data/data_collector/v1/views.py | 20 ++-----------------
2 files changed, 20 insertions(+), 18 deletions(-)
create mode 100644 eox_core/api/data/data_collector/v1/permissions.py
diff --git a/eox_core/api/data/data_collector/v1/permissions.py b/eox_core/api/data/data_collector/v1/permissions.py
new file mode 100644
index 000000000..ff6398455
--- /dev/null
+++ b/eox_core/api/data/data_collector/v1/permissions.py
@@ -0,0 +1,18 @@
+import logging
+from rest_framework.permissions import BasePermission
+from rest_framework.authentication import get_authorization_header
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+class DatacollectorPermission(BasePermission):
+ """
+ Permission class to allow access only if the request contains a valid GitHub Action token.
+ """
+
+ def has_permission(self, request, view):
+ auth_header = get_authorization_header(request).decode('utf-8')
+ auth_token = settings.EOX_CORE_DATA_COLLECT_AUTH_TOKEN
+ if auth_header and auth_header == f"Bearer {auth_token}":
+ return True
+ return False
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index e61719ac7..30fc82e3e 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -1,31 +1,15 @@
import logging
-import requests
-import yaml
from rest_framework.views import APIView
from rest_framework.response import Response
-from rest_framework import status, permissions
+from rest_framework import status
from django.conf import settings
from eox_core.api.data.data_collector.tasks import generate_report
-
-from rest_framework.permissions import BasePermission
-from rest_framework.authentication import get_authorization_header
+from eox_core.api.data.data_collector.v1.permissions import DatacollectorPermission
from django.conf import settings
logger = logging.getLogger(__name__)
-class DatacollectorPermission(BasePermission):
- """
- Permission class to allow access only if the request contains a valid GitHub Action token.
- """
- def has_permission(self, request, view):
- auth_header = get_authorization_header(request).decode('utf-8')
- auth_token = settings.EOX_CORE_DATA_COLLECT_AUTH_TOKEN
- if auth_header and auth_header == f"Bearer {auth_token}":
- return True
- return False
-
-
class DataCollectorView(APIView):
"""
API view to handle data collection requests.
From 69ca02af9ca2c61c85dd49e519403b63f5419bd8 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Wed, 26 Feb 2025 00:18:47 -0500
Subject: [PATCH 27/60] chore: pycodestyle fixes
---
eox_core/api/data/data_collector/queries.py | 8 ++++----
eox_core/api/data/data_collector/tasks.py | 1 +
eox_core/api/data/data_collector/urls.py | 2 +-
eox_core/api/data/data_collector/utils.py | 4 +++-
eox_core/api/data/data_collector/v1/permissions.py | 1 +
eox_core/api/data/data_collector/v1/views.py | 2 +-
6 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/eox_core/api/data/data_collector/queries.py b/eox_core/api/data/data_collector/queries.py
index 34cac92c4..91e6f7497 100644
--- a/eox_core/api/data/data_collector/queries.py
+++ b/eox_core/api/data/data_collector/queries.py
@@ -1,9 +1,9 @@
"""
This module contains SQL queries to retrieve platform-related metrics.
-These queries are used to extract information about user activity, course
-engagement, and certificate issuance. The data retrieval is conditioned on
-the feature being enabled and the necessary authentication credentials
+These queries are used to extract information about user activity, course
+engagement, and certificate issuance. The data retrieval is conditioned on
+the feature being enabled and the necessary authentication credentials
being set.
"""
@@ -110,5 +110,5 @@
"Courses With Active Certificates": COURSES_WITH_ACTIVE_CERTIFICATES_QUERY,
"Enrollments Last Month": ENROLLMENTS_LAST_MONTH_QUERY,
"Enrollments Last 7 Days": ENROLLMENTS_LAST_7_DAYS_QUERY,
- "Certificates Issued": CERTIFICATES_ISSUED_QUERY,
+ "Certificates Issued": CERTIFICATES_ISSUED_QUERY,
}
diff --git a/eox_core/api/data/data_collector/tasks.py b/eox_core/api/data/data_collector/tasks.py
index 549a8672e..86ada98b4 100644
--- a/eox_core/api/data/data_collector/tasks.py
+++ b/eox_core/api/data/data_collector/tasks.py
@@ -14,6 +14,7 @@
COUNTDOWN = 60
MAX_RETRIES = 3
+
@shared_task(bind=True)
def generate_report(self, destination_url, token_generation_url, current_host):
"""
diff --git a/eox_core/api/data/data_collector/urls.py b/eox_core/api/data/data_collector/urls.py
index ee575a587..77b523045 100644
--- a/eox_core/api/data/data_collector/urls.py
+++ b/eox_core/api/data/data_collector/urls.py
@@ -1,7 +1,7 @@
"""
URL configuration for the Microsite API.
-This module defines the URL patterns for the Microsite API,
+This module defines the URL patterns for the Microsite API,
including versioned endpoints for data_collector.
"""
from django.urls import include, re_path
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index e3d7b0e67..703a63e25 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -25,7 +25,7 @@ def execute_query(sql_query):
Returns:
list or dict: Structured query results.
-
+
Example:
>>> execute_query("SELECT id, username FROM auth_user WHERE is_active = 1;")
[
@@ -44,6 +44,7 @@ def execute_query(sql_query):
return [dict(zip(columns, row)) for row in rows] # Multi-column results as a list of dicts
return rows
+
def post_process_query_results(data):
"""
Cleans and processes query results by:
@@ -68,6 +69,7 @@ def post_process_query_results(data):
return data.isoformat()
return data
+
def post_data_to_api(api_url, report_data, token_generation_url, current_host):
"""
Sends the generated report data to the Shipyard API.
diff --git a/eox_core/api/data/data_collector/v1/permissions.py b/eox_core/api/data/data_collector/v1/permissions.py
index ff6398455..c9ebbbbdb 100644
--- a/eox_core/api/data/data_collector/v1/permissions.py
+++ b/eox_core/api/data/data_collector/v1/permissions.py
@@ -5,6 +5,7 @@
logger = logging.getLogger(__name__)
+
class DatacollectorPermission(BasePermission):
"""
Permission class to allow access only if the request contains a valid GitHub Action token.
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index 30fc82e3e..ec2125b7b 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -37,7 +37,7 @@ def post(self, request):
{"error": "This endpoint is currently disabled."},
status=status.HTTP_403_FORBIDDEN
)
-
+
destination_url = getattr(settings, "EOX_CORE_DATA_COLLECT_DESTINATION_URL", None)
token_generation_url = getattr(settings, "EOX_CORE_DATA_COLLECT_TOKEN_URL", None)
From 95efdd5a271bc30596e45f6bd30a74342bfb592f Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Wed, 26 Feb 2025 18:20:21 -0500
Subject: [PATCH 28/60] chore: pylint fixes
---
eox_core/api/data/data_collector/tasks.py | 14 +++++------
eox_core/api/data/data_collector/utils.py | 24 ++++++++++---------
.../api/data/data_collector/v1/permissions.py | 4 ++++
eox_core/api/data/data_collector/v1/urls.py | 1 +
eox_core/api/data/data_collector/v1/views.py | 7 +++++-
5 files changed, 31 insertions(+), 19 deletions(-)
diff --git a/eox_core/api/data/data_collector/tasks.py b/eox_core/api/data/data_collector/tasks.py
index 86ada98b4..5abc49f34 100644
--- a/eox_core/api/data/data_collector/tasks.py
+++ b/eox_core/api/data/data_collector/tasks.py
@@ -3,11 +3,11 @@
and posting the results to the Shipyard API.
"""
+import logging
from celery import shared_task
-from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, post_process_query_results
+from django.db.utils import DatabaseError, OperationalError
from eox_core.api.data.data_collector.queries import PREDEFINED_QUERIES
-import yaml
-import logging
+from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, post_process_query_results
logger = logging.getLogger(__name__)
@@ -32,19 +32,19 @@ def generate_report(self, destination_url, token_generation_url, current_host):
try:
report_data = {}
for query_name, query_sql in PREDEFINED_QUERIES.items():
- logger.info(f"Executing query: {query_name}")
+ logger.info("Executing query: %s", query_name)
try:
result = execute_query(query_sql)
processed_result = post_process_query_results(result)
report_data[query_name] = processed_result
- except Exception as e:
- logger.error(f"Failed to execute query '{query_name}': {e}")
+ except (DatabaseError, OperationalError) as e:
+ logger.error("Failed to execute query '%s': %s", query_name, e)
continue
post_data_to_api(destination_url, report_data, token_generation_url, current_host)
logger.info("Report generation task completed successfully.")
except Exception as e:
- logger.error(f"An error occurred in the report generation task: {e}. Retrying")
+ logger.error("An error occurred in the report generation task: '%s'. Retrying", e)
raise self.retry(exc=e, countdown=COUNTDOWN, max_retries=MAX_RETRIES)
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index 703a63e25..8659d0c83 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -3,14 +3,11 @@
and integration with the Shipyard API.
"""
-import yaml
-from django.db import connection
+import logging
+from datetime import datetime
import requests
from django.conf import settings
-from datetime import datetime
-import logging
-import re
-
+from django.db import connection
from eox_core.utils import get_access_token
logger = logging.getLogger(__name__)
@@ -60,7 +57,7 @@ def post_process_query_results(data):
"""
if isinstance(data, dict):
return {key: post_process_query_results(value) for key, value in data.items()}
- elif isinstance(data, list):
+ if isinstance(data, list):
# If it's a list with one item, return just the item
if len(data) == 1:
return post_process_query_results(data[0])
@@ -90,7 +87,12 @@ def post_data_to_api(api_url, report_data, token_generation_url, current_host):
"Content-Type": "application/json",
}
payload = {"instance_domain": current_host, "data": report_data}
- response = requests.post(api_url, json=payload, headers=headers)
-
- if not response.ok:
- raise Exception(f"Failed to post data to Shipyard API: {response.content}")
+ response = requests.post(api_url, json=payload, headers=headers, timeout=10)
+
+ try:
+ response = requests.post(api_url, json=payload, headers=headers, timeout=10)
+ response.raise_for_status()
+ except requests.Timeout:
+ raise requests.Timeout("The request to Shipyard API timed out.")
+ except requests.RequestException as e:
+ raise requests.RequestException(f"Failed to post data to Shipyard API: {e}")
\ No newline at end of file
diff --git a/eox_core/api/data/data_collector/v1/permissions.py b/eox_core/api/data/data_collector/v1/permissions.py
index c9ebbbbdb..89331b109 100644
--- a/eox_core/api/data/data_collector/v1/permissions.py
+++ b/eox_core/api/data/data_collector/v1/permissions.py
@@ -1,3 +1,7 @@
+"""
+Custom permission classes for the Data Collector API.
+"""
+
import logging
from rest_framework.permissions import BasePermission
from rest_framework.authentication import get_authorization_header
diff --git a/eox_core/api/data/data_collector/v1/urls.py b/eox_core/api/data/data_collector/v1/urls.py
index a815d1d03..1cf78f1e7 100644
--- a/eox_core/api/data/data_collector/v1/urls.py
+++ b/eox_core/api/data/data_collector/v1/urls.py
@@ -2,6 +2,7 @@
from django.urls import path
from eox_core.api.data.data_collector.v1.views import DataCollectorView
+# pylint: disable=invalid-name
app_name = "data_collector"
urlpatterns = [
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index ec2125b7b..f1d6e542a 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -1,3 +1,9 @@
+"""
+Views for the Data Collector API (v1).
+
+This module defines the API views for collecting and processing data.
+"""
+
import logging
from rest_framework.views import APIView
from rest_framework.response import Response
@@ -5,7 +11,6 @@
from django.conf import settings
from eox_core.api.data.data_collector.tasks import generate_report
from eox_core.api.data.data_collector.v1.permissions import DatacollectorPermission
-from django.conf import settings
logger = logging.getLogger(__name__)
From 94d5aebff342e7ea4b8b4abd3d33929f099d611d Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Wed, 26 Feb 2025 18:23:05 -0500
Subject: [PATCH 29/60] chore: newline
---
eox_core/api/data/data_collector/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index 8659d0c83..da1de51c2 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -95,4 +95,4 @@ def post_data_to_api(api_url, report_data, token_generation_url, current_host):
except requests.Timeout:
raise requests.Timeout("The request to Shipyard API timed out.")
except requests.RequestException as e:
- raise requests.RequestException(f"Failed to post data to Shipyard API: {e}")
\ No newline at end of file
+ raise requests.RequestException(f"Failed to post data to Shipyard API: {e}")
From d344e6307f4ff51d59526a8a7794fe5391f4ae8f Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Wed, 26 Feb 2025 18:31:29 -0500
Subject: [PATCH 30/60] chore: newline
---
eox_core/api/data/data_collector/utils.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index da1de51c2..1331670c8 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -62,7 +62,7 @@ def post_process_query_results(data):
if len(data) == 1:
return post_process_query_results(data[0])
return [post_process_query_results(item) for item in data]
- elif isinstance(data, datetime):
+ if isinstance(data, datetime):
return data.isoformat()
return data
@@ -92,7 +92,7 @@ def post_data_to_api(api_url, report_data, token_generation_url, current_host):
try:
response = requests.post(api_url, json=payload, headers=headers, timeout=10)
response.raise_for_status()
- except requests.Timeout:
- raise requests.Timeout("The request to Shipyard API timed out.")
+ except requests.Timeout as exc:
+ raise requests.Timeout("The request to Shipyard API timed out.") from exc
except requests.RequestException as e:
raise requests.RequestException(f"Failed to post data to Shipyard API: {e}")
From f4b7063551eb0becbc209b00681ba1733ccf72af Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Wed, 26 Feb 2025 18:41:13 -0500
Subject: [PATCH 31/60] chore: pylint fixes
---
eox_core/api/data/data_collector/tasks.py | 2 ++
eox_core/api/data/data_collector/utils.py | 2 ++
eox_core/api/data/data_collector/v1/permissions.py | 5 +++--
eox_core/api/data/data_collector/v1/urls.py | 1 +
eox_core/api/data/data_collector/v1/views.py | 8 +++++---
eox_core/utils.py | 2 +-
6 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/eox_core/api/data/data_collector/tasks.py b/eox_core/api/data/data_collector/tasks.py
index 5abc49f34..a6b2f8c35 100644
--- a/eox_core/api/data/data_collector/tasks.py
+++ b/eox_core/api/data/data_collector/tasks.py
@@ -4,8 +4,10 @@
"""
import logging
+
from celery import shared_task
from django.db.utils import DatabaseError, OperationalError
+
from eox_core.api.data.data_collector.queries import PREDEFINED_QUERIES
from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, post_process_query_results
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/data_collector/utils.py
index 1331670c8..0170a43d3 100644
--- a/eox_core/api/data/data_collector/utils.py
+++ b/eox_core/api/data/data_collector/utils.py
@@ -5,9 +5,11 @@
import logging
from datetime import datetime
+
import requests
from django.conf import settings
from django.db import connection
+
from eox_core.utils import get_access_token
logger = logging.getLogger(__name__)
diff --git a/eox_core/api/data/data_collector/v1/permissions.py b/eox_core/api/data/data_collector/v1/permissions.py
index 89331b109..eacf48a02 100644
--- a/eox_core/api/data/data_collector/v1/permissions.py
+++ b/eox_core/api/data/data_collector/v1/permissions.py
@@ -3,9 +3,10 @@
"""
import logging
-from rest_framework.permissions import BasePermission
-from rest_framework.authentication import get_authorization_header
+
from django.conf import settings
+from rest_framework.authentication import get_authorization_header
+from rest_framework.permissions import BasePermission
logger = logging.getLogger(__name__)
diff --git a/eox_core/api/data/data_collector/v1/urls.py b/eox_core/api/data/data_collector/v1/urls.py
index 1cf78f1e7..4a518fdbb 100644
--- a/eox_core/api/data/data_collector/v1/urls.py
+++ b/eox_core/api/data/data_collector/v1/urls.py
@@ -1,5 +1,6 @@
"""_"""
from django.urls import path
+
from eox_core.api.data.data_collector.v1.views import DataCollectorView
# pylint: disable=invalid-name
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/data_collector/v1/views.py
index f1d6e542a..788dd6bb9 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/data_collector/v1/views.py
@@ -5,10 +5,12 @@
"""
import logging
-from rest_framework.views import APIView
-from rest_framework.response import Response
-from rest_framework import status
+
from django.conf import settings
+from rest_framework import status
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
from eox_core.api.data.data_collector.tasks import generate_report
from eox_core.api.data.data_collector.v1.permissions import DatacollectorPermission
diff --git a/eox_core/utils.py b/eox_core/utils.py
index cf9261b4c..25f586ff5 100644
--- a/eox_core/utils.py
+++ b/eox_core/utils.py
@@ -5,8 +5,8 @@
import datetime
import hashlib
import re
-import requests
+import requests
from django.conf import settings
from django.contrib.sites.models import Site
from django.core import cache
From 33dd1b5bdd399c97789fe2d09a9f54227920c843 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Wed, 26 Feb 2025 19:18:03 -0500
Subject: [PATCH 32/60] refactor: package rename data_collector to
aggregated_collector
---
.../__init__.py | 0
.../queries.py | 0
.../tasks.py | 4 ++--
.../urls.py | 4 ++--
.../utils.py | 0
.../v1/__init__.py | 0
.../v1/permissions.py | 6 +++---
eox_core/api/data/aggregated_collector/v1/urls.py | 11 +++++++++++
.../v1/views.py | 14 +++++++-------
eox_core/api/data/data_collector/v1/urls.py | 11 -----------
eox_core/api/data/v1/urls.py | 2 +-
11 files changed, 26 insertions(+), 26 deletions(-)
rename eox_core/api/data/{data_collector => aggregated_collector}/__init__.py (100%)
rename eox_core/api/data/{data_collector => aggregated_collector}/queries.py (100%)
rename eox_core/api/data/{data_collector => aggregated_collector}/tasks.py (89%)
rename eox_core/api/data/{data_collector => aggregated_collector}/urls.py (59%)
rename eox_core/api/data/{data_collector => aggregated_collector}/utils.py (100%)
rename eox_core/api/data/{data_collector => aggregated_collector}/v1/__init__.py (100%)
rename eox_core/api/data/{data_collector => aggregated_collector}/v1/permissions.py (75%)
create mode 100644 eox_core/api/data/aggregated_collector/v1/urls.py
rename eox_core/api/data/{data_collector => aggregated_collector}/v1/views.py (77%)
delete mode 100644 eox_core/api/data/data_collector/v1/urls.py
diff --git a/eox_core/api/data/data_collector/__init__.py b/eox_core/api/data/aggregated_collector/__init__.py
similarity index 100%
rename from eox_core/api/data/data_collector/__init__.py
rename to eox_core/api/data/aggregated_collector/__init__.py
diff --git a/eox_core/api/data/data_collector/queries.py b/eox_core/api/data/aggregated_collector/queries.py
similarity index 100%
rename from eox_core/api/data/data_collector/queries.py
rename to eox_core/api/data/aggregated_collector/queries.py
diff --git a/eox_core/api/data/data_collector/tasks.py b/eox_core/api/data/aggregated_collector/tasks.py
similarity index 89%
rename from eox_core/api/data/data_collector/tasks.py
rename to eox_core/api/data/aggregated_collector/tasks.py
index a6b2f8c35..c35b8d460 100644
--- a/eox_core/api/data/data_collector/tasks.py
+++ b/eox_core/api/data/aggregated_collector/tasks.py
@@ -8,8 +8,8 @@
from celery import shared_task
from django.db.utils import DatabaseError, OperationalError
-from eox_core.api.data.data_collector.queries import PREDEFINED_QUERIES
-from eox_core.api.data.data_collector.utils import execute_query, post_data_to_api, post_process_query_results
+from eox_core.api.data.aggregated_collector.queries import PREDEFINED_QUERIES
+from eox_core.api.data.aggregated_collector.utils import execute_query, post_data_to_api, post_process_query_results
logger = logging.getLogger(__name__)
diff --git a/eox_core/api/data/data_collector/urls.py b/eox_core/api/data/aggregated_collector/urls.py
similarity index 59%
rename from eox_core/api/data/data_collector/urls.py
rename to eox_core/api/data/aggregated_collector/urls.py
index 77b523045..cb2ef2c76 100644
--- a/eox_core/api/data/data_collector/urls.py
+++ b/eox_core/api/data/aggregated_collector/urls.py
@@ -2,7 +2,7 @@
URL configuration for the Microsite API.
This module defines the URL patterns for the Microsite API,
-including versioned endpoints for data_collector.
+including versioned endpoints for aggregated_collector.
"""
from django.urls import include, re_path
@@ -10,5 +10,5 @@
urlpatterns = [ # pylint: disable=invalid-name
- re_path(r'^v1/', include('eox_core.api.data.data_collector.v1.urls', namespace='eox-data-api-collector-v1')),
+ re_path(r'^v1/', include('eox_core.api.data.aggregated_collector.v1.urls', namespace='eox-data-api-collector-v1')),
]
diff --git a/eox_core/api/data/data_collector/utils.py b/eox_core/api/data/aggregated_collector/utils.py
similarity index 100%
rename from eox_core/api/data/data_collector/utils.py
rename to eox_core/api/data/aggregated_collector/utils.py
diff --git a/eox_core/api/data/data_collector/v1/__init__.py b/eox_core/api/data/aggregated_collector/v1/__init__.py
similarity index 100%
rename from eox_core/api/data/data_collector/v1/__init__.py
rename to eox_core/api/data/aggregated_collector/v1/__init__.py
diff --git a/eox_core/api/data/data_collector/v1/permissions.py b/eox_core/api/data/aggregated_collector/v1/permissions.py
similarity index 75%
rename from eox_core/api/data/data_collector/v1/permissions.py
rename to eox_core/api/data/aggregated_collector/v1/permissions.py
index eacf48a02..cb25002f2 100644
--- a/eox_core/api/data/data_collector/v1/permissions.py
+++ b/eox_core/api/data/aggregated_collector/v1/permissions.py
@@ -1,5 +1,5 @@
"""
-Custom permission classes for the Data Collector API.
+Custom permission classes for the Aggregated Collector API.
"""
import logging
@@ -11,14 +11,14 @@
logger = logging.getLogger(__name__)
-class DatacollectorPermission(BasePermission):
+class AggregatedCollectorPermission(BasePermission):
"""
Permission class to allow access only if the request contains a valid GitHub Action token.
"""
def has_permission(self, request, view):
auth_header = get_authorization_header(request).decode('utf-8')
- auth_token = settings.EOX_CORE_DATA_COLLECT_AUTH_TOKEN
+ auth_token = settings.EOX_CORE_AGGREGATED_COLLECT_AUTH_TOKEN
if auth_header and auth_header == f"Bearer {auth_token}":
return True
return False
diff --git a/eox_core/api/data/aggregated_collector/v1/urls.py b/eox_core/api/data/aggregated_collector/v1/urls.py
new file mode 100644
index 000000000..15d69e396
--- /dev/null
+++ b/eox_core/api/data/aggregated_collector/v1/urls.py
@@ -0,0 +1,11 @@
+"""_"""
+from django.urls import path
+
+from eox_core.api.data.aggregated_collector.v1.views import AggregatedCollectorView
+
+# pylint: disable=invalid-name
+app_name = "aggregated_collector"
+
+urlpatterns = [
+ path("aggregated-collector/", AggregatedCollectorView.as_view(), name="aggregated_collector"),
+]
diff --git a/eox_core/api/data/data_collector/v1/views.py b/eox_core/api/data/aggregated_collector/v1/views.py
similarity index 77%
rename from eox_core/api/data/data_collector/v1/views.py
rename to eox_core/api/data/aggregated_collector/v1/views.py
index 788dd6bb9..f2a94ed5e 100644
--- a/eox_core/api/data/data_collector/v1/views.py
+++ b/eox_core/api/data/aggregated_collector/v1/views.py
@@ -1,5 +1,5 @@
"""
-Views for the Data Collector API (v1).
+Views for the Aggregated Collector API (v1).
This module defines the API views for collecting and processing data.
"""
@@ -11,13 +11,13 @@
from rest_framework.response import Response
from rest_framework.views import APIView
-from eox_core.api.data.data_collector.tasks import generate_report
-from eox_core.api.data.data_collector.v1.permissions import DatacollectorPermission
+from eox_core.api.data.aggregated_collector.tasks import generate_report
+from eox_core.api.data.aggregated_collector.v1.permissions import AggregatedCollectorPermission
logger = logging.getLogger(__name__)
-class DataCollectorView(APIView):
+class AggregatedCollectorView(APIView):
"""
API view to handle data collection requests.
@@ -27,7 +27,7 @@ class DataCollectorView(APIView):
This view:
- Triggers an async task to execute queries and send results to a specified destination.
"""
- permission_classes = [DatacollectorPermission]
+ permission_classes = [AggregatedCollectorPermission]
def post(self, request):
"""
@@ -45,8 +45,8 @@ def post(self, request):
status=status.HTTP_403_FORBIDDEN
)
- destination_url = getattr(settings, "EOX_CORE_DATA_COLLECT_DESTINATION_URL", None)
- token_generation_url = getattr(settings, "EOX_CORE_DATA_COLLECT_TOKEN_URL", None)
+ destination_url = getattr(settings, "EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL", None)
+ token_generation_url = getattr(settings, "EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL", None)
if not destination_url or not token_generation_url:
logger.error("Data collection settings are missing.")
diff --git a/eox_core/api/data/data_collector/v1/urls.py b/eox_core/api/data/data_collector/v1/urls.py
deleted file mode 100644
index 4a518fdbb..000000000
--- a/eox_core/api/data/data_collector/v1/urls.py
+++ /dev/null
@@ -1,11 +0,0 @@
-"""_"""
-from django.urls import path
-
-from eox_core.api.data.data_collector.v1.views import DataCollectorView
-
-# pylint: disable=invalid-name
-app_name = "data_collector"
-
-urlpatterns = [
- path("collect-data/", DataCollectorView.as_view(), name="collect_data"),
-]
diff --git a/eox_core/api/data/v1/urls.py b/eox_core/api/data/v1/urls.py
index 292589f20..11cb207ba 100644
--- a/eox_core/api/data/v1/urls.py
+++ b/eox_core/api/data/v1/urls.py
@@ -11,5 +11,5 @@
urlpatterns = [ # pylint: disable=invalid-name
re_path(r'^v1/', include((ROUTER.urls, 'eox_core'), namespace='eox-data-api-v1')),
re_path(r'^v1/tasks/(?P<task_id>.*)$', CeleryTasksStatus.as_view(), name="celery-data-api-tasks"),
- re_path(r'^', include('eox_core.api.data.data_collector.urls', namespace='eox-data-api-collector')),
+ re_path(r'^', include('eox_core.api.data.aggregated_collector.urls', namespace='eox-data-api-collector')),
]
From eece34523bd510efc7c964b1e6013842d5c25edd Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Thu, 27 Feb 2025 00:08:03 -0500
Subject: [PATCH 33/60] feat: add aggregated collecor test
---
.../aggregated_collector/v1/tests/__init__.py | 2 +
.../v1/tests/test_views.py | 66 +++++++++++++++++++
2 files changed, 68 insertions(+)
create mode 100644 eox_core/api/data/aggregated_collector/v1/tests/__init__.py
create mode 100644 eox_core/api/data/aggregated_collector/v1/tests/test_views.py
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/__init__.py b/eox_core/api/data/aggregated_collector/v1/tests/__init__.py
new file mode 100644
index 000000000..faa18be5b
--- /dev/null
+++ b/eox_core/api/data/aggregated_collector/v1/tests/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
new file mode 100644
index 000000000..66f127d36
--- /dev/null
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -0,0 +1,66 @@
+import pytest
+from unittest.mock import patch
+from django.conf import settings
+from rest_framework.test import APIClient
+from rest_framework import status
+from eox_core.api.data.aggregated_collector.tasks import generate_report
+
+@pytest.fixture
+def api_client():
+ return APIClient()
+
+@pytest.fixture
+def api_url():
+ return "/eox-core/data-api/v1/aggregated-collector/"
+
+@pytest.mark.django_db
+class TestAggregatedCollectorView:
+
+ @pytest.fixture(autouse=True)
+ def setup(self):
+ """Set default values for settings before each test."""
+ self.default_settings = {
+ "AGGREGATED_DATA_COLLECTOR_API_ENABLED": True,
+ "EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL": "https://example.com/destination",
+ "EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL": "https://example.com/token",
+ "EOX_CORE_DATA_COLLECT_AUTH_TOKEN": "valid_token"
+ }
+ settings.configure(**self.default_settings)
+
+ def test_endpoint_disabled(self, api_client, api_url):
+ """Should return 403 if the API is disabled in settings."""
+ settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = False
+
+ response = api_client.post(api_url)
+
+ assert response.status_code == status.HTTP_403_FORBIDDEN
+ assert response.data["error"] == "This endpoint is currently disabled."
+
+ def test_missing_settings(self, api_client, api_url):
+ """Should return 500 if required settings are missing."""
+ settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = None
+ settings.EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL = None
+
+ response = api_client.post(api_url)
+
+ assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
+ assert response.data["error"] == "Data collection settings are not properly configured."
+
+ @patch("eox_core.api.data.aggregated_collector.tasks.generate_report.delay")
+ def test_successful_request(self, mock_generate_report, api_client, api_url):
+ """Should return 202 and trigger the async task correctly."""
+ response = api_client.post(api_url)
+
+ assert response.status_code == status.HTTP_202_ACCEPTED
+ assert response.data["message"] == "Data collection task has been initiated successfully."
+ mock_generate_report.assert_called_once_with(
+ "https://example.com/destination",
+ "https://example.com/token",
+ "testserver"
+ )
+
+ def test_unauthorized_request(self, api_client, api_url):
+ """Should return 403 if no authentication token is provided."""
+ response = api_client.post(api_url, HTTP_AUTHORIZATION="")
+
+ assert response.status_code == status.HTTP_403_FORBIDDEN
From 340bb60f362436ed64c2916781d49eeded3195de Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Thu, 27 Feb 2025 00:13:45 -0500
Subject: [PATCH 34/60] feat: add aggregated collecor test
---
.../aggregated_collector/v1/tests/test_views.py | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 66f127d36..a49870a34 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -17,15 +17,12 @@ def api_url():
class TestAggregatedCollectorView:
@pytest.fixture(autouse=True)
- def setup(self):
+ def setup(settings):
"""Set default values for settings before each test."""
- self.default_settings = {
- "AGGREGATED_DATA_COLLECTOR_API_ENABLED": True,
- "EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL": "https://example.com/destination",
- "EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL": "https://example.com/token",
- "EOX_CORE_DATA_COLLECT_AUTH_TOKEN": "valid_token"
- }
- settings.configure(**self.default_settings)
+ settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
+ settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = "https://example.com/destination"
+ settings.EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL = "https://example.com/token"
+ settings.EOX_CORE_DATA_COLLECT_AUTH_TOKEN = "valid_token"
def test_endpoint_disabled(self, api_client, api_url):
"""Should return 403 if the API is disabled in settings."""
From 77ca34cbdc17c565c8fcbcaee5163739f30ff8f8 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Thu, 27 Feb 2025 11:16:31 -0500
Subject: [PATCH 35/60] refactor: add type hints
---
eox_core/api/data/aggregated_collector/tasks.py | 2 +-
eox_core/api/data/aggregated_collector/utils.py | 6 +++---
eox_core/api/data/aggregated_collector/v1/views.py | 3 ++-
3 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/eox_core/api/data/aggregated_collector/tasks.py b/eox_core/api/data/aggregated_collector/tasks.py
index c35b8d460..ddc314ba5 100644
--- a/eox_core/api/data/aggregated_collector/tasks.py
+++ b/eox_core/api/data/aggregated_collector/tasks.py
@@ -18,7 +18,7 @@
@shared_task(bind=True)
-def generate_report(self, destination_url, token_generation_url, current_host):
+def generate_report(self, destination_url: str, token_generation_url: str, current_host: str):
"""
Async task to generate a report:
1. Executes all predefined queries.
diff --git a/eox_core/api/data/aggregated_collector/utils.py b/eox_core/api/data/aggregated_collector/utils.py
index 0170a43d3..c3a51289d 100644
--- a/eox_core/api/data/aggregated_collector/utils.py
+++ b/eox_core/api/data/aggregated_collector/utils.py
@@ -15,7 +15,7 @@
logger = logging.getLogger(__name__)
-def execute_query(sql_query):
+def execute_query(sql_query: str):
"""
Execute a raw SQL query and return the results in a structured format.
@@ -44,7 +44,7 @@ def execute_query(sql_query):
return rows
-def post_process_query_results(data):
+def post_process_query_results(data: any):
"""
Cleans and processes query results by:
- Serializing datetime objects into strings.
@@ -69,7 +69,7 @@ def post_process_query_results(data):
return data
-def post_data_to_api(api_url, report_data, token_generation_url, current_host):
+def post_data_to_api(api_url: str, report_data: dict, token_generation_url: str, current_host: str):
"""
Sends the generated report data to the Shipyard API.
diff --git a/eox_core/api/data/aggregated_collector/v1/views.py b/eox_core/api/data/aggregated_collector/v1/views.py
index f2a94ed5e..a680a1b12 100644
--- a/eox_core/api/data/aggregated_collector/v1/views.py
+++ b/eox_core/api/data/aggregated_collector/v1/views.py
@@ -7,6 +7,7 @@
import logging
from django.conf import settings
+from django.http import HttpRequest
from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import APIView
@@ -29,7 +30,7 @@ class AggregatedCollectorView(APIView):
"""
permission_classes = [AggregatedCollectorPermission]
- def post(self, request):
+ def post(self, request: HttpRequest) -> Response:
"""
Handles POST requests to collect data.
From 45fcc361d27d8d58ad8dc73d5b4bb5ebe1dcc992 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Thu, 27 Feb 2025 11:47:45 -0500
Subject: [PATCH 36/60] feat: add aggregated collecor test
---
.../api/data/aggregated_collector/v1/tests/test_views.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index a49870a34..388fc51a3 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -1,6 +1,7 @@
import pytest
from unittest.mock import patch
from django.conf import settings
+from django.urls import reverse
from rest_framework.test import APIClient
from rest_framework import status
from eox_core.api.data.aggregated_collector.tasks import generate_report
@@ -11,13 +12,13 @@ def api_client():
@pytest.fixture
def api_url():
- return "/eox-core/data-api/v1/aggregated-collector/"
+ return reverse('eox-data-api:eox-data-api-collector-v1:aggregated_collector')
@pytest.mark.django_db
class TestAggregatedCollectorView:
@pytest.fixture(autouse=True)
- def setup(settings):
+ def setup(self, settings):
"""Set default values for settings before each test."""
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = "https://example.com/destination"
From 9415b1d76f86941608bac2e39b51a6a55c29bb60 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Thu, 27 Feb 2025 14:53:49 -0500
Subject: [PATCH 37/60] feat: add aggregated collecor test
---
.../v1/tests/test_views.py | 88 +++++++------------
1 file changed, 31 insertions(+), 57 deletions(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 388fc51a3..5e8901acf 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -1,64 +1,38 @@
-import pytest
-from unittest.mock import patch
+from unittest.mock import patch, MagicMock
+from django.test import TestCase
from django.conf import settings
-from django.urls import reverse
from rest_framework.test import APIClient
from rest_framework import status
+from eox_core.api.data.aggregated_collector.utils import execute_query
from eox_core.api.data.aggregated_collector.tasks import generate_report
+from eox_core.api.data.aggregated_collector.v1.views import AggregatedCollectorView
-@pytest.fixture
-def api_client():
- return APIClient()
-
-@pytest.fixture
-def api_url():
- return reverse('eox-data-api:eox-data-api-collector-v1:aggregated_collector')
-
-@pytest.mark.django_db
-class TestAggregatedCollectorView:
-
- @pytest.fixture(autouse=True)
- def setup(self, settings):
- """Set default values for settings before each test."""
+class AggregatedCollectorTests(TestCase):
+ def setUp(self):
+ self.client = APIClient()
+ self.url = "/v1/aggregated-collector/"
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
- settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = "https://example.com/destination"
- settings.EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL = "https://example.com/token"
- settings.EOX_CORE_DATA_COLLECT_AUTH_TOKEN = "valid_token"
-
- def test_endpoint_disabled(self, api_client, api_url):
- """Should return 403 if the API is disabled in settings."""
- settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = False
-
- response = api_client.post(api_url)
+ settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = "http://mock-api.com"
+ settings.EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL = "http://mock-token.com"
+ settings.EOX_CORE_AGGREGATED_COLLECT_AUTH_TOKEN = "test-token"
+
+ @patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
+ def test_execute_query(self, mock_cursor):
+ mock_cursor.return_value.__enter__.return_value.fetchall.return_value = [(1, "test_user")]
+ mock_cursor.return_value.__enter__.return_value.description = [("id",), ("username",)]
- assert response.status_code == status.HTTP_403_FORBIDDEN
- assert response.data["error"] == "This endpoint is currently disabled."
-
- def test_missing_settings(self, api_client, api_url):
- """Should return 500 if required settings are missing."""
- settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = None
- settings.EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL = None
-
- response = api_client.post(api_url)
-
- assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
- assert response.data["error"] == "Data collection settings are not properly configured."
-
- @patch("eox_core.api.data.aggregated_collector.tasks.generate_report.delay")
- def test_successful_request(self, mock_generate_report, api_client, api_url):
- """Should return 202 and trigger the async task correctly."""
- response = api_client.post(api_url)
-
- assert response.status_code == status.HTTP_202_ACCEPTED
- assert response.data["message"] == "Data collection task has been initiated successfully."
- mock_generate_report.assert_called_once_with(
- "https://example.com/destination",
- "https://example.com/token",
- "testserver"
- )
-
- def test_unauthorized_request(self, api_client, api_url):
- """Should return 403 if no authentication token is provided."""
- response = api_client.post(api_url, HTTP_AUTHORIZATION="")
-
- assert response.status_code == status.HTTP_403_FORBIDDEN
+ result = execute_query("SELECT id, username FROM auth_user;")
+ self.assertEqual(result, [{"id": 1, "username": "test_user"}])
+
+ @patch("eox_core.api.data.aggregated_collector.tasks.execute_query")
+ @patch("eox_core.api.data.aggregated_collector.tasks.post_data_to_api")
+ def test_generate_report(self, mock_post, mock_execute):
+ mock_execute.return_value = [{"id": 1, "data": "sample"}]
+ generate_report("http://mock-api.com", "http://mock-token.com", "localhost")
+ mock_post.assert_called_once()
+
+ @patch("eox_core.api.data.aggregated_collector.v1.views.generate_report.delay")
+ def test_aggregated_collector_view(self, mock_task):
+ response = self.client.post(self.url, HTTP_AUTHORIZATION=f"Bearer {settings.EOX_CORE_AGGREGATED_COLLECT_AUTH_TOKEN}")
+ self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
+ mock_task.assert_called_once()
From 67144c1ea3becf18df53a4a6013dc3ebf37df11e Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Thu, 27 Feb 2025 15:03:00 -0500
Subject: [PATCH 38/60] feat: add aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 5e8901acf..89dc056dc 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -10,7 +10,7 @@
class AggregatedCollectorTests(TestCase):
def setUp(self):
self.client = APIClient()
- self.url = "/v1/aggregated-collector/"
+ self.url = "/eox-core/data-api/v1/aggregated-collector/"
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = "http://mock-api.com"
settings.EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL = "http://mock-token.com"
From 562353ea5216d6bd72fe8b8b18df95891b283f0f Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Thu, 27 Feb 2025 15:19:02 -0500
Subject: [PATCH 39/60] feat: fix aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 89dc056dc..dfe91dbb4 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -1,6 +1,7 @@
from unittest.mock import patch, MagicMock
from django.test import TestCase
from django.conf import settings
+from django.urls import reverse
from rest_framework.test import APIClient
from rest_framework import status
from eox_core.api.data.aggregated_collector.utils import execute_query
@@ -10,7 +11,7 @@
class AggregatedCollectorTests(TestCase):
def setUp(self):
self.client = APIClient()
- self.url = "/eox-core/data-api/v1/aggregated-collector/"
+ self.url = reverse("eox_core.api.data.v1.urls:eox-data-api-collector-v1:aggregated_collector")
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = "http://mock-api.com"
settings.EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL = "http://mock-token.com"
From 543c3c62481ca0e45404a5f728ff5bf898c9791d Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Thu, 27 Feb 2025 15:25:22 -0500
Subject: [PATCH 40/60] feat: fix aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index dfe91dbb4..a0d1d87d2 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -11,7 +11,7 @@
class AggregatedCollectorTests(TestCase):
def setUp(self):
self.client = APIClient()
- self.url = reverse("eox_core.api.data.v1.urls:eox-data-api-collector-v1:aggregated_collector")
+ self.url = reverse("eox-data-api:eox-data-api-collector-v1:aggregated_collector")
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = "http://mock-api.com"
settings.EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL = "http://mock-token.com"
From d3068db110c882eca625e6f7f4015fab49ad80a8 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Thu, 27 Feb 2025 15:40:33 -0500
Subject: [PATCH 41/60] feat: fix aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index a0d1d87d2..51ace0ed0 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -11,7 +11,7 @@
class AggregatedCollectorTests(TestCase):
def setUp(self):
self.client = APIClient()
- self.url = reverse("eox-data-api:eox-data-api-collector-v1:aggregated_collector")
+ self.url = reverse("eox-data-api-collector-v1:aggregated_collector")
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = "http://mock-api.com"
settings.EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL = "http://mock-token.com"
From 855d4407f8fcef4aae2bf5f04989f3764db87516 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Fri, 28 Feb 2025 09:07:25 -0500
Subject: [PATCH 42/60] chore: add improvements
---
CHANGELOG.md | 2 +-
.../api/data/aggregated_collector/queries.py | 11 +++-
.../api/data/aggregated_collector/utils.py | 11 ++--
.../aggregated_collector/v1/permissions.py | 11 +++-
.../aggregated_collector/v1/tests/__init__.py | 2 -
.../v1/tests/test_utils.py | 56 +++++++++++++++++++
.../v1/tests/test_views.py | 27 +++++----
.../api/data/aggregated_collector/v1/views.py | 16 ++----
8 files changed, 103 insertions(+), 33 deletions(-)
create mode 100644 eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 81d134526..e2dff07bc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,7 +15,7 @@ Please do not update the unreleased notes.
### Added
-- API to collect data and generate reports.
+- New API endpoint to support data collection and report generation (if the feature is enabled).
## [v11.1.0](https://github.com/eduNEXT/eox-core/compare/v11.0.0...v11.1.0) - (2024-11-21)
diff --git a/eox_core/api/data/aggregated_collector/queries.py b/eox_core/api/data/aggregated_collector/queries.py
index 91e6f7497..21a263dc0 100644
--- a/eox_core/api/data/aggregated_collector/queries.py
+++ b/eox_core/api/data/aggregated_collector/queries.py
@@ -14,6 +14,7 @@
FROM
auth_user as au;
"""
+
# This query counts the number of unique active users in the last month.
# A user is considered active if they have modified a student module within the last month.
ACTIVE_USERS_LAST_MONTH_QUERY = """
@@ -29,6 +30,7 @@
GROUP BY
YEAR(cs.modified), MONTH(cs.modified);
"""
+
# This query counts the number of unique active users in the last 7 days.
# A user is considered active if they have modified a student module within the last 7 days.
ACTIVE_USERS_LAST_7_DAYS_QUERY = """
@@ -39,6 +41,7 @@
WHERE
cs.modified >= DATE_SUB(CURDATE(), INTERVAL 7 DAY);
"""
+
# This query counts the total number of courses created on the platform.
TOTAL_COURSES_CREATED_QUERY = """
SELECT
@@ -46,6 +49,7 @@
FROM
course_overviews_courseoverview as coc;
"""
+
# This query counts the number of CourseOverviews objects that started before
# now and have not yet ended
ACTIVE_COURSES_COUNT_QUERY = """
@@ -60,6 +64,7 @@
OR coc.end IS NULL
);
"""
+
# This query counts the number of courses that have at least one issued certificate.
COURSES_WITH_ACTIVE_CERTIFICATES_QUERY = """
SELECT
@@ -71,6 +76,7 @@
ON
coc.id = cg.course_id;
"""
+
# This query counts the number of new enrollments in the last month.
ENROLLMENTS_LAST_MONTH_QUERY = """
SELECT
@@ -85,6 +91,7 @@
GROUP BY
YEAR(sc.created), MONTH(sc.created);
"""
+
# This query counts the number of new enrollments in the last 7 days.
ENROLLMENTS_LAST_7_DAYS_QUERY = """
SELECT
@@ -94,13 +101,15 @@
WHERE
sc.created >= DATE_SUB(CURDATE(), INTERVAL 7 DAY);
"""
+
+# This query counts the total number of certificates issued on the platform.
CERTIFICATES_ISSUED_QUERY = """
SELECT
COUNT(*)
FROM
certificates_generatedcertificate as cg;
"""
-# This query counts the total number of certificates issued on the platform.
+
PREDEFINED_QUERIES = {
"Total Users": TOTAL_USERS_QUERY,
"Active Users Last Month": ACTIVE_USERS_LAST_MONTH_QUERY,
diff --git a/eox_core/api/data/aggregated_collector/utils.py b/eox_core/api/data/aggregated_collector/utils.py
index c3a51289d..8a2915e72 100644
--- a/eox_core/api/data/aggregated_collector/utils.py
+++ b/eox_core/api/data/aggregated_collector/utils.py
@@ -1,6 +1,5 @@
"""
-Utility functions for report generation, including query execution
-and integration with the Shipyard API.
+Utility functions for report generation, including query execution and data posting.
"""
import logging
@@ -81,8 +80,8 @@ def post_data_to_api(api_url: str, report_data: dict, token_generation_url: str,
"""
token = get_access_token(
token_generation_url,
- settings.EOX_CORE_AGGREGATED_DATA_API_CLIENT_ID,
- settings.EOX_CORE_AGGREGATED_DATA_API_CLIENT_SECRET,
+ settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_CLIENT_ID,
+ settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_CLIENT_SECRET,
)
headers = {
"Authorization": f"Bearer {token}",
@@ -95,6 +94,6 @@ def post_data_to_api(api_url: str, report_data: dict, token_generation_url: str,
response = requests.post(api_url, json=payload, headers=headers, timeout=10)
response.raise_for_status()
except requests.Timeout as exc:
- raise requests.Timeout("The request to Shipyard API timed out.") from exc
+ raise requests.Timeout("The request to API timed out.") from exc
except requests.RequestException as e:
- raise requests.RequestException(f"Failed to post data to Shipyard API: {e}")
+ raise requests.RequestException(f"Failed to post data to API: {e}")
diff --git a/eox_core/api/data/aggregated_collector/v1/permissions.py b/eox_core/api/data/aggregated_collector/v1/permissions.py
index cb25002f2..be3fb82e8 100644
--- a/eox_core/api/data/aggregated_collector/v1/permissions.py
+++ b/eox_core/api/data/aggregated_collector/v1/permissions.py
@@ -13,12 +13,19 @@
class AggregatedCollectorPermission(BasePermission):
"""
- Permission class to allow access only if the request contains a valid GitHub Action token.
+ Permission class to allow access only if:
+ - The AGGREGATED_DATA_COLLECTOR_API_ENABLED setting is True.
+ - The request contains a valid GitHub Action token.
"""
def has_permission(self, request, view):
+ # Check if the API is enabled
+ if not getattr(settings, "AGGREGATED_DATA_COLLECTOR_API_ENABLED", False):
+ return False
+
+ # Check if the request contains a valid token
auth_header = get_authorization_header(request).decode('utf-8')
- auth_token = settings.EOX_CORE_AGGREGATED_COLLECT_AUTH_TOKEN
+ auth_token = settings.EOX_CORE_AGGREGATED_COLLECTOR_AUTH_TOKEN
if auth_header and auth_header == f"Bearer {auth_token}":
return True
return False
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/__init__.py b/eox_core/api/data/aggregated_collector/v1/tests/__init__.py
index faa18be5b..e69de29bb 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/__init__.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/__init__.py
@@ -1,2 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
new file mode 100644
index 000000000..a2db714d3
--- /dev/null
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
@@ -0,0 +1,56 @@
+"""
+Test suite for Aggregated Data Collector API.
+"""
+from unittest.mock import patch, MagicMock
+from django.test import TestCase
+from eox_core.api.data.aggregated_collector.utils import execute_query, fetch_access_token
+
+class UtilsTests(TestCase):
+ @patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
+ def test_execute_query_success(self, mock_cursor):
+ """
+ Test that execute_query returns the expected result when the query executes successfully.
+ """
+ mock_cursor.return_value.__enter__.return_value.fetchall.return_value = [(1, "test_user")]
+ mock_cursor.return_value.__enter__.return_value.description = [("id",), ("username",)]
+
+ result = execute_query("SELECT id, username FROM auth_user;")
+ expected_result = [{"id": 1, "username": "test_user"}]
+
+ self.assertEqual(result, expected_result)
+
+ @patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
+ def test_execute_query_failure(self, mock_cursor):
+ """
+ Test that execute_query handles exceptions gracefully.
+ """
+ mock_cursor.return_value.__enter__.side_effect = Exception("Database error")
+
+ result = execute_query("SELECT id, username FROM auth_user;")
+ self.assertEqual(result, []) # Expected to return an empty list on failure
+
+ @patch("eox_core.api.data.aggregated_collector.utils.requests.post")
+ def test_fetch_access_token_success(self, mock_post):
+ """
+ Test that fetch_access_token correctly retrieves an access token.
+ """
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {"access_token": "mock_token"}
+ mock_post.return_value = mock_response
+
+ token = fetch_access_token("http://mock-token.com", "client_id", "client_secret")
+ self.assertEqual(token, "mock_token")
+
+ @patch("eox_core.api.data.aggregated_collector.utils.requests.post")
+ def test_fetch_access_token_failure(self, mock_post):
+ """
+ Test that fetch_access_token returns None if the request fails.
+ """
+ mock_response = MagicMock()
+ mock_response.status_code = 400
+ mock_response.json.return_value = {"error": "invalid_request"}
+ mock_post.return_value = mock_response
+
+ token = fetch_access_token("http://mock-token.com", "client_id", "client_secret")
+ self.assertIsNone(token)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 51ace0ed0..d68c8431f 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -1,3 +1,6 @@
+"""
+Test suite for Aggregated Data Collector API.
+"""
from unittest.mock import patch, MagicMock
from django.test import TestCase
from django.conf import settings
@@ -8,32 +11,34 @@
from eox_core.api.data.aggregated_collector.tasks import generate_report
from eox_core.api.data.aggregated_collector.v1.views import AggregatedCollectorView
-class AggregatedCollectorTests(TestCase):
+class AggregatedCollectorViewTests(TestCase):
def setUp(self):
self.client = APIClient()
- self.url = reverse("eox-data-api-collector-v1:aggregated_collector")
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
- settings.EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL = "http://mock-api.com"
- settings.EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL = "http://mock-token.com"
- settings.EOX_CORE_AGGREGATED_COLLECT_AUTH_TOKEN = "test-token"
+ settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_URL = "http://mock-api.com"
+ settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_TOKEN_URL = "http://mock-token.com"
+ settings.EOX_CORE_AGGREGATED_COLLECTOR_AUTH_TOKEN = "test-token"
@patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
def test_execute_query(self, mock_cursor):
+ """
+ Test execute_query function to ensure it correctly fetches and formats SQL query results.
+ """
mock_cursor.return_value.__enter__.return_value.fetchall.return_value = [(1, "test_user")]
mock_cursor.return_value.__enter__.return_value.description = [("id",), ("username",)]
result = execute_query("SELECT id, username FROM auth_user;")
+
self.assertEqual(result, [{"id": 1, "username": "test_user"}])
@patch("eox_core.api.data.aggregated_collector.tasks.execute_query")
@patch("eox_core.api.data.aggregated_collector.tasks.post_data_to_api")
def test_generate_report(self, mock_post, mock_execute):
+ """
+ Test generate_report function to ensure data is retrieved and posted correctly.
+ """
mock_execute.return_value = [{"id": 1, "data": "sample"}]
+
generate_report("http://mock-api.com", "http://mock-token.com", "localhost")
- mock_post.assert_called_once()
- @patch("eox_core.api.data.aggregated_collector.v1.views.generate_report.delay")
- def test_aggregated_collector_view(self, mock_task):
- response = self.client.post(self.url, HTTP_AUTHORIZATION=f"Bearer {settings.EOX_CORE_AGGREGATED_COLLECT_AUTH_TOKEN}")
- self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
- mock_task.assert_called_once()
+ mock_post.assert_called_once()
diff --git a/eox_core/api/data/aggregated_collector/v1/views.py b/eox_core/api/data/aggregated_collector/v1/views.py
index a680a1b12..34fb313c7 100644
--- a/eox_core/api/data/aggregated_collector/v1/views.py
+++ b/eox_core/api/data/aggregated_collector/v1/views.py
@@ -40,17 +40,13 @@ def post(self, request: HttpRequest) -> Response:
Returns:
Response: A success or error message.
"""
- if not getattr(settings, "AGGREGATED_DATA_COLLECTOR_API_ENABLED", False):
- return Response(
- {"error": "This endpoint is currently disabled."},
- status=status.HTTP_403_FORBIDDEN
- )
-
- destination_url = getattr(settings, "EOX_CORE_AGGREGATED_COLLECT_DESTINATION_URL", None)
- token_generation_url = getattr(settings, "EOX_CORE_AGGREGATED_COLLECT_TOKEN_URL", None)
+ destination_url = getattr(settings, "EOX_CORE_AGGREGATED_COLLECTOR_TARGET_URL", None)
+ token_generation_url = getattr(settings, "EOX_CORE_AGGREGATED_COLLECTOR_TARGET_TOKEN_URL", None)
+ client_id = getattr(settings, "EOX_CORE_AGGREGATED_COLLECTOR_TARGET_CLIENT_ID", None)
+ client_secret = getattr(settings, "EOX_CORE_AGGREGATED_COLLECTOR_TARGET_CLIENT_SECRET", None)
- if not destination_url or not token_generation_url:
- logger.error("Data collection settings are missing.")
+ if not all([destination_url, token_generation_url, client_id, client_secret]):
+ logger.error("Missing required Aggregated Data Collector settings.")
return Response(
{"error": "Data collection settings are not properly configured."},
status=status.HTTP_500_INTERNAL_SERVER_ERROR
From 8bc6d13a56759445e767ac75b075e25d78607b98 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Fri, 28 Feb 2025 09:14:15 -0500
Subject: [PATCH 43/60] chore: add improvements
---
.../v1/tests/test_utils.py | 28 +------------------
1 file changed, 1 insertion(+), 27 deletions(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
index a2db714d3..e7c114447 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
@@ -3,7 +3,7 @@
"""
from unittest.mock import patch, MagicMock
from django.test import TestCase
-from eox_core.api.data.aggregated_collector.utils import execute_query, fetch_access_token
+from eox_core.api.data.aggregated_collector.utils import execute_query
class UtilsTests(TestCase):
@patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
@@ -28,29 +28,3 @@ def test_execute_query_failure(self, mock_cursor):
result = execute_query("SELECT id, username FROM auth_user;")
self.assertEqual(result, []) # Expected to return an empty list on failure
-
- @patch("eox_core.api.data.aggregated_collector.utils.requests.post")
- def test_fetch_access_token_success(self, mock_post):
- """
- Test that fetch_access_token correctly retrieves an access token.
- """
- mock_response = MagicMock()
- mock_response.status_code = 200
- mock_response.json.return_value = {"access_token": "mock_token"}
- mock_post.return_value = mock_response
-
- token = fetch_access_token("http://mock-token.com", "client_id", "client_secret")
- self.assertEqual(token, "mock_token")
-
- @patch("eox_core.api.data.aggregated_collector.utils.requests.post")
- def test_fetch_access_token_failure(self, mock_post):
- """
- Test that fetch_access_token returns None if the request fails.
- """
- mock_response = MagicMock()
- mock_response.status_code = 400
- mock_response.json.return_value = {"error": "invalid_request"}
- mock_post.return_value = mock_response
-
- token = fetch_access_token("http://mock-token.com", "client_id", "client_secret")
- self.assertIsNone(token)
From b619c7a79dacf6d0e1744dd35e853f480872c54f Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Fri, 28 Feb 2025 09:22:55 -0500
Subject: [PATCH 44/60] chore: add improvements
---
.../aggregated_collector/v1/tests/test_utils.py | 10 ----------
.../aggregated_collector/v1/tests/test_views.py | 13 -------------
2 files changed, 23 deletions(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
index e7c114447..06bdc212d 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
@@ -18,13 +18,3 @@ def test_execute_query_success(self, mock_cursor):
expected_result = [{"id": 1, "username": "test_user"}]
self.assertEqual(result, expected_result)
-
- @patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
- def test_execute_query_failure(self, mock_cursor):
- """
- Test that execute_query handles exceptions gracefully.
- """
- mock_cursor.return_value.__enter__.side_effect = Exception("Database error")
-
- result = execute_query("SELECT id, username FROM auth_user;")
- self.assertEqual(result, []) # Expected to return an empty list on failure
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index d68c8431f..fbad281f6 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -7,7 +7,6 @@
from django.urls import reverse
from rest_framework.test import APIClient
from rest_framework import status
-from eox_core.api.data.aggregated_collector.utils import execute_query
from eox_core.api.data.aggregated_collector.tasks import generate_report
from eox_core.api.data.aggregated_collector.v1.views import AggregatedCollectorView
@@ -19,18 +18,6 @@ def setUp(self):
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_TOKEN_URL = "http://mock-token.com"
settings.EOX_CORE_AGGREGATED_COLLECTOR_AUTH_TOKEN = "test-token"
- @patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
- def test_execute_query(self, mock_cursor):
- """
- Test execute_query function to ensure it correctly fetches and formats SQL query results.
- """
- mock_cursor.return_value.__enter__.return_value.fetchall.return_value = [(1, "test_user")]
- mock_cursor.return_value.__enter__.return_value.description = [("id",), ("username",)]
-
- result = execute_query("SELECT id, username FROM auth_user;")
-
- self.assertEqual(result, [{"id": 1, "username": "test_user"}])
-
@patch("eox_core.api.data.aggregated_collector.tasks.execute_query")
@patch("eox_core.api.data.aggregated_collector.tasks.post_data_to_api")
def test_generate_report(self, mock_post, mock_execute):
From d7dbe305d23f3c9b5260b788ac5fbf83c5878adf Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Fri, 28 Feb 2025 09:25:09 -0500
Subject: [PATCH 45/60] chore: add improvements
---
eox_core/api/data/aggregated_collector/v1/tests/test_utils.py | 3 ++-
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
index 06bdc212d..bc99f9856 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
@@ -5,6 +5,7 @@
from django.test import TestCase
from eox_core.api.data.aggregated_collector.utils import execute_query
+
class UtilsTests(TestCase):
@patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
def test_execute_query_success(self, mock_cursor):
@@ -13,7 +14,7 @@ def test_execute_query_success(self, mock_cursor):
"""
mock_cursor.return_value.__enter__.return_value.fetchall.return_value = [(1, "test_user")]
mock_cursor.return_value.__enter__.return_value.description = [("id",), ("username",)]
-
+
result = execute_query("SELECT id, username FROM auth_user;")
expected_result = [{"id": 1, "username": "test_user"}]
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index fbad281f6..948e1d134 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -10,6 +10,7 @@
from eox_core.api.data.aggregated_collector.tasks import generate_report
from eox_core.api.data.aggregated_collector.v1.views import AggregatedCollectorView
+
class AggregatedCollectorViewTests(TestCase):
def setUp(self):
self.client = APIClient()
From b7e1c98dfddaeb5e02bb9e11dc19837ffa4600cf Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Fri, 28 Feb 2025 09:39:38 -0500
Subject: [PATCH 46/60] chore: add improvements
---
.../aggregated_collector/v1/tests/test_utils.py | 5 ++++-
.../aggregated_collector/v1/tests/test_views.py | 16 ++++++++++++++--
2 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
index bc99f9856..f131a0a3e 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
@@ -1,12 +1,15 @@
"""
Test suite for Aggregated Data Collector API.
"""
-from unittest.mock import patch, MagicMock
+from unittest.mock import patch
from django.test import TestCase
from eox_core.api.data.aggregated_collector.utils import execute_query
class UtilsTests(TestCase):
+ """
+ Test suite for utility functions used in the Aggregated Data Collector API.
+ """
@patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
def test_execute_query_success(self, mock_cursor):
"""
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 948e1d134..201662383 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -7,13 +7,14 @@
from django.urls import reverse
from rest_framework.test import APIClient
from rest_framework import status
+from eox_core.api.data.aggregated_collector.utils import execute_query
from eox_core.api.data.aggregated_collector.tasks import generate_report
from eox_core.api.data.aggregated_collector.v1.views import AggregatedCollectorView
-
class AggregatedCollectorViewTests(TestCase):
def setUp(self):
self.client = APIClient()
+ self.url = reverse("eox-data-api-collector-v1:aggregated_collector")
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_URL = "http://mock-api.com"
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_TOKEN_URL = "http://mock-token.com"
@@ -27,6 +28,17 @@ def test_generate_report(self, mock_post, mock_execute):
"""
mock_execute.return_value = [{"id": 1, "data": "sample"}]
- generate_report("http://mock-api.com", "http://mock-token.com", "localhost")
+ generate_report(self, "http://mock-api.com", "http://mock-token.com", "localhost")
mock_post.assert_called_once()
+
+ @patch("eox_core.api.data.aggregated_collector.v1.views.generate_report.delay")
+ def test_aggregated_collector_view(self, mock_task):
+ """
+ Test AggregatedCollectorView to ensure it correctly triggers the report generation task.
+ """
+ response = self.client.post(self.url, HTTP_AUTHORIZATION=f"Bearer {settings.EOX_CORE_AGGREGATED_COLLECTOR_AUTH_TOKEN}")
+
+ self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
+
+ mock_task.assert_called_once()
From de697b9cab3fcc59175667ddc423ab05f07abc6b Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Fri, 28 Feb 2025 16:24:41 -0500
Subject: [PATCH 47/60] chore: delete duplicate code
---
eox_core/api/data/aggregated_collector/utils.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/utils.py b/eox_core/api/data/aggregated_collector/utils.py
index 8a2915e72..2965c0c40 100644
--- a/eox_core/api/data/aggregated_collector/utils.py
+++ b/eox_core/api/data/aggregated_collector/utils.py
@@ -88,7 +88,6 @@ def post_data_to_api(api_url: str, report_data: dict, token_generation_url: str,
"Content-Type": "application/json",
}
payload = {"instance_domain": current_host, "data": report_data}
- response = requests.post(api_url, json=payload, headers=headers, timeout=10)
try:
response = requests.post(api_url, json=payload, headers=headers, timeout=10)
From d911b946ef1a713deb6947d95d8111db77e99c1c Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Sun, 2 Mar 2025 23:49:28 -0500
Subject: [PATCH 48/60] feat: fix aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 201662383..293603a55 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -14,11 +14,13 @@
class AggregatedCollectorViewTests(TestCase):
def setUp(self):
self.client = APIClient()
- self.url = reverse("eox-data-api-collector-v1:aggregated_collector")
+ self.url = "/eox-core/data-api/v1/aggregated-collector/"
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_URL = "http://mock-api.com"
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_TOKEN_URL = "http://mock-token.com"
settings.EOX_CORE_AGGREGATED_COLLECTOR_AUTH_TOKEN = "test-token"
+ settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_CLIENT_ID = "test-client-id"
+ settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_CLIENT_SECRET = "test-client-secret"
@patch("eox_core.api.data.aggregated_collector.tasks.execute_query")
@patch("eox_core.api.data.aggregated_collector.tasks.post_data_to_api")
From be5623c19bdafbf467a7ca354c67fad44d0fda81 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Sun, 2 Mar 2025 23:58:17 -0500
Subject: [PATCH 49/60] feat: fix aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 293603a55..8802abfcb 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -14,7 +14,7 @@
class AggregatedCollectorViewTests(TestCase):
def setUp(self):
self.client = APIClient()
- self.url = "/eox-core/data-api/v1/aggregated-collector/"
+ self.url = reverse("aggregated_collector")
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_URL = "http://mock-api.com"
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_TOKEN_URL = "http://mock-token.com"
From 6a3395b2b4dd111d7060a4cb6e2be99986bf0da3 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 00:08:25 -0500
Subject: [PATCH 50/60] feat: fix aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 8802abfcb..65d33b8cf 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -14,7 +14,7 @@
class AggregatedCollectorViewTests(TestCase):
def setUp(self):
self.client = APIClient()
- self.url = reverse("aggregated_collector")
+ self.url = reverse("eox-core:eox-data-api:eox-data-api-collector-v1:aggregated_collector")
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_URL = "http://mock-api.com"
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_TOKEN_URL = "http://mock-token.com"
From 7bab3bcf69ad0d4796b6d571deb20a45ea92cd34 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 00:43:06 -0500
Subject: [PATCH 51/60] feat: fix aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 65d33b8cf..65bf6cea2 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -14,7 +14,7 @@
class AggregatedCollectorViewTests(TestCase):
def setUp(self):
self.client = APIClient()
- self.url = reverse("eox-core:eox-data-api:eox-data-api-collector-v1:aggregated_collector")
+ self.url = reverse("eox-data-api:eox-data-api-collector:eox-data-api-collector-v1:aggregated_collector")
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = True
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_URL = "http://mock-api.com"
settings.EOX_CORE_AGGREGATED_COLLECTOR_TARGET_TOKEN_URL = "http://mock-token.com"
From 79f33e130950dd541137f6278670a8aa1a486fbf Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 00:46:30 -0500
Subject: [PATCH 52/60] feat: fix aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 65bf6cea2..b5e2491b0 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -30,7 +30,7 @@ def test_generate_report(self, mock_post, mock_execute):
"""
mock_execute.return_value = [{"id": 1, "data": "sample"}]
- generate_report(self, "http://mock-api.com", "http://mock-token.com", "localhost")
+ generate_report("http://mock-api.com", "http://mock-token.com", "localhost")
mock_post.assert_called_once()
From c444d0065aeb24206948466c6a5ea5daaf7758a0 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 00:50:23 -0500
Subject: [PATCH 53/60] feat: fix aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index b5e2491b0..11850e3a0 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -11,6 +11,7 @@
from eox_core.api.data.aggregated_collector.tasks import generate_report
from eox_core.api.data.aggregated_collector.v1.views import AggregatedCollectorView
+
class AggregatedCollectorViewTests(TestCase):
def setUp(self):
self.client = APIClient()
@@ -42,5 +43,5 @@ def test_aggregated_collector_view(self, mock_task):
response = self.client.post(self.url, HTTP_AUTHORIZATION=f"Bearer {settings.EOX_CORE_AGGREGATED_COLLECTOR_AUTH_TOKEN}")
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
-
+
mock_task.assert_called_once()
From 274ab80626c4d477466e5257e8309f01c12f293e Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 01:01:31 -0500
Subject: [PATCH 54/60] feat: fix aggregated collecor test
---
.../api/data/aggregated_collector/v1/tests/test_views.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 11850e3a0..0da15bcc0 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -1,18 +1,19 @@
"""
Test suite for Aggregated Data Collector API.
"""
-from unittest.mock import patch, MagicMock
+from unittest.mock import patch
from django.test import TestCase
from django.conf import settings
from django.urls import reverse
from rest_framework.test import APIClient
from rest_framework import status
-from eox_core.api.data.aggregated_collector.utils import execute_query
from eox_core.api.data.aggregated_collector.tasks import generate_report
-from eox_core.api.data.aggregated_collector.v1.views import AggregatedCollectorView
class AggregatedCollectorViewTests(TestCase):
+ """
+ Test cases for the Aggregated Data Collector API.
+ """
def setUp(self):
self.client = APIClient()
self.url = reverse("eox-data-api:eox-data-api-collector:eox-data-api-collector-v1:aggregated_collector")
@@ -31,7 +32,7 @@ def test_generate_report(self, mock_post, mock_execute):
"""
mock_execute.return_value = [{"id": 1, "data": "sample"}]
- generate_report("http://mock-api.com", "http://mock-token.com", "localhost")
+ generate_report(None, "http://mock-api.com", "http://mock-token.com", "localhost")
mock_post.assert_called_once()
From 8ca5f18e028194c865802f5928b74510ba03678e Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 01:07:37 -0500
Subject: [PATCH 55/60] feat: fix aggregated collecor test
---
eox_core/api/data/aggregated_collector/v1/tests/test_views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 0da15bcc0..b6a5a34b2 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -32,7 +32,7 @@ def test_generate_report(self, mock_post, mock_execute):
"""
mock_execute.return_value = [{"id": 1, "data": "sample"}]
- generate_report(None, "http://mock-api.com", "http://mock-token.com", "localhost")
+ generate_report.run("http://mock-api.com", "http://mock-token.com", "localhost")
mock_post.assert_called_once()
From c8648770e848c634b2b03f348eb4fc9f76883cf6 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 01:13:31 -0500
Subject: [PATCH 56/60] feat: fix aggregated collecor test
---
.../api/data/aggregated_collector/v1/tests/test_utils.py | 2 ++
.../api/data/aggregated_collector/v1/tests/test_views.py | 6 ++++--
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
index f131a0a3e..3b5966e00 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
@@ -2,7 +2,9 @@
Test suite for Aggregated Data Collector API.
"""
from unittest.mock import patch
+
from django.test import TestCase
+
from eox_core.api.data.aggregated_collector.utils import execute_query
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index b6a5a34b2..f1cfa4e8c 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -2,11 +2,13 @@
Test suite for Aggregated Data Collector API.
"""
from unittest.mock import patch
-from django.test import TestCase
+
from django.conf import settings
+from django.test import TestCase
from django.urls import reverse
-from rest_framework.test import APIClient
from rest_framework import status
+from rest_framework.test import APIClient
+
from eox_core.api.data.aggregated_collector.tasks import generate_report
From 72dc4e835a1d5fe459bda27d7c0a81eb3887d52a Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 10:49:23 -0500
Subject: [PATCH 57/60] chore: add test for utils
---
.../v1/tests/test_utils.py | 31 +++++++++++++++++++
1 file changed, 31 insertions(+)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
index 3b5966e00..3fdc9138a 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_utils.py
@@ -24,3 +24,34 @@ def test_execute_query_success(self, mock_cursor):
expected_result = [{"id": 1, "username": "test_user"}]
self.assertEqual(result, expected_result)
+
+ @patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
+ def test_execute_query_empty_result(self, mock_cursor):
+ """
+ Test that execute_query returns an empty list when there are no results.
+ """
+ mock_cursor.return_value.__enter__.return_value.fetchall.return_value = []
+ mock_cursor.return_value.__enter__.return_value.description = [("id",), ("username",)]
+
+ result = execute_query("SELECT id, username FROM auth_user;")
+
+ self.assertEqual(result, [])
+
+ @patch("eox_core.api.data.aggregated_collector.utils.connection.cursor")
+ def test_execute_query_multiple_rows(self, mock_cursor):
+ """
+ Test that execute_query correctly handles multiple rows in the result.
+ """
+ mock_cursor.return_value.__enter__.return_value.fetchall.return_value = [
+ (1, "test_user1"),
+ (2, "test_user2"),
+ ]
+ mock_cursor.return_value.__enter__.return_value.description = [("id",), ("username",)]
+
+ result = execute_query("SELECT id, username FROM auth_user;")
+ expected_result = [
+ {"id": 1, "username": "test_user1"},
+ {"id": 2, "username": "test_user2"},
+ ]
+
+ self.assertEqual(result, expected_result)
From 647113093dc1650d1e0d808ee4f757441d198272 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 10:53:54 -0500
Subject: [PATCH 58/60] chore: add test for views
---
.../v1/tests/test_views.py | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index f1cfa4e8c..64f4e9504 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -48,3 +48,21 @@ def test_aggregated_collector_view(self, mock_task):
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
mock_task.assert_called_once()
+
+ def test_aggregated_collector_view_disabled(self):
+ """
+ Test that the view returns 503 if the API is disabled.
+ """
+ settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = False
+
+ response = self.client.post(self.url, HTTP_AUTHORIZATION=f"Bearer {settings.EOX_CORE_AGGREGATED_COLLECTOR_AUTH_TOKEN}")
+
+ self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE)
+
+ def test_aggregated_collector_view_no_auth(self):
+ """
+ Test that the view returns 401 if no authentication token is provided.
+ """
+ response = self.client.post(self.url)
+
+ self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
From 54f49141422d7be7601e9bcec3fd8defdd24ac47 Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 11:06:56 -0500
Subject: [PATCH 59/60] chore: add test for views
---
.../data/aggregated_collector/v1/tests/test_views.py | 12 ++----------
1 file changed, 2 insertions(+), 10 deletions(-)
diff --git a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
index 64f4e9504..2eab43aa5 100644
--- a/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
+++ b/eox_core/api/data/aggregated_collector/v1/tests/test_views.py
@@ -51,18 +51,10 @@ def test_aggregated_collector_view(self, mock_task):
def test_aggregated_collector_view_disabled(self):
"""
- Test that the view returns 503 if the API is disabled.
+ Test that the view returns 403 if the API is disabled.
"""
settings.AGGREGATED_DATA_COLLECTOR_API_ENABLED = False
response = self.client.post(self.url, HTTP_AUTHORIZATION=f"Bearer {settings.EOX_CORE_AGGREGATED_COLLECTOR_AUTH_TOKEN}")
- self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE)
-
- def test_aggregated_collector_view_no_auth(self):
- """
- Test that the view returns 401 if no authentication token is provided.
- """
- response = self.client.post(self.url)
-
- self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
+ self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
From 0bc33489c61d692d79b53cb386c66b5bebbb536a Mon Sep 17 00:00:00 2001
From: Luis Felipe Castano <felipe.castano@edunext.co>
Date: Mon, 3 Mar 2025 11:13:09 -0500
Subject: [PATCH 60/60] chore: add missing docstring
---
eox_core/api/data/aggregated_collector/v1/permissions.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/eox_core/api/data/aggregated_collector/v1/permissions.py b/eox_core/api/data/aggregated_collector/v1/permissions.py
index be3fb82e8..df9a9295f 100644
--- a/eox_core/api/data/aggregated_collector/v1/permissions.py
+++ b/eox_core/api/data/aggregated_collector/v1/permissions.py
@@ -19,6 +19,9 @@ class AggregatedCollectorPermission(BasePermission):
"""
def has_permission(self, request, view):
+ """"
+ Determines if the request has permission to access the Aggregated Collector API.
+ """
# Check if the API is enabled
if not getattr(settings, "AGGREGATED_DATA_COLLECTOR_API_ENABLED", False):
return False