Merge pull request #176 from matematikk-mooc/staging

Staging

statistics_api/clients/kpas_client.py

@@ -32,7 +32,7 @@ def get_schools_by_county_id(self, county_id: int) -> Tuple[Dict]:
             print("Received invalid json")
             return None
         return tuple(response_json.get("result"))
-            
+
 
     def get_municipalities_by_county_id(self, county_id):
         web_response = self.web_session.get(f"{KPAS_NSR_API_URL}/counties/{county_id}/communities/")
@@ -43,9 +43,6 @@ def get_municipalities_by_county_id(self, county_id):
             print("Received invalid json")
             return None
 
-    def post_trigger_to_activate_schedule_of_job(self) -> None:
-        web_response = self.web_session.post(f"{KPAS_API_URL}/run_scheduler")
-        assert web_response.status_code == 200
 
     def get_county(self, county_id: int) -> Union[Dict, None]:
         web_response = self.web_session.get(f"{KPAS_NSR_API_URL}/counties/{county_id}")

statistics_api/enrollment_activity/views.py

@@ -1,5 +1,6 @@
 from rest_framework import viewsets, serializers
 from rest_framework.response import Response
+from django.core.exceptions import ValidationError
 
 # Create your views here.
 from statistics_api.enrollment_activity.models import EnrollmentActivity
@@ -13,33 +14,41 @@ class EnrollmentActivityViewSet(viewsets.ViewSet):
     def list(self, request):
         from_date = request.GET.get("from")
         to_date = request.GET.get("to")
-        if from_date and to_date:
-            queryset = EnrollmentActivity.objects.filter(activity_date__gte=from_date, activity_date__lte=to_date)
-        elif from_date:
-            queryset = EnrollmentActivity.objects.filter(activity_date__gte=from_date)
-        elif to_date:
-            queryset = EnrollmentActivity.objects.filter(activity_date__lte=to_date)
-        else:
-            queryset = EnrollmentActivity.objects.all()
-        queryset = queryset.order_by('-activity_date')
-        serializer = EnrollmentActivitySerializer(queryset, many=True)
-        return Response(serializer.data)
+        try:
+            queryset = self.filter_query(from_date, to_date)
+            serializer = EnrollmentActivitySerializer(queryset, many=True)
+            return Response(serializer.data)
+        except ValidationError as e:
+            return Response({"Error": str(e)}, status=400)
+        except Exception as e:
+            return Response({"Error": str(e)}, status=500)
 
     def retrieve(self, request, pk=None):
         from_date = request.GET.get("from")
         to_date = request.GET.get("to")
+        try:
+            queryset = self.filter_query(from_date, to_date, course_id=pk)
+            serializer = EnrollmentActivitySerializer(queryset, many=True)
+            return Response(serializer.data)
+        except ValidationError as e:
+            return Response({"Error": str(e)}, status=400)
+        except Exception as e:
+            return Response({"Error": str(e)}, status=500)
+
+    def filter_query(self, from_date, to_date, course_id=None):
+        queryset = EnrollmentActivity.objects.all()
+        if course_id:
+            queryset = queryset.filter(course_id=course_id)
+
         if from_date and to_date:
-            queryset = EnrollmentActivity.objects.filter(course_id=pk, activity_date__gte=from_date, activity_date__lte=to_date)
+            queryset = queryset.filter(activity_date__gte=from_date, activity_date__lte=to_date)
         elif from_date:
-            queryset = EnrollmentActivity.objects.filter(course_id=pk, activity_date__gte=from_date)
+            queryset = queryset.filter(activity_date__gte=from_date)
         elif to_date:
-            queryset = EnrollmentActivity.objects.filter(course_id=pk, activity_date__lte=to_date)
-        else:
-            queryset = EnrollmentActivity.objects.filter(course_id=pk)
-        queryset = queryset.order_by('-activity_date')
-        serializer = EnrollmentActivitySerializer(queryset, many=True)
-        return Response(serializer.data)
+            queryset = queryset.filter(activity_date__lte=to_date)
 
+        queryset = queryset.order_by('-activity_date')
+        return queryset
 
 class EnrollmentActivitySerializer(serializers.ModelSerializer):
     class Meta:

statistics_api/history/migrations/0005_alter_history_visited_url.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.18 on 2024-02-21 08:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('history', '0004_alter_history_context_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='history',
+            name='visited_url',
+            field=models.URLField(blank=True, max_length=1024, null=True),
+        ),
+    ]

statistics_api/history/migrations/0006_alter_history_asset_name.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.18 on 2024-02-22 11:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('history', '0005_alter_history_visited_url'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='history',
+            name='asset_name',
+            field=models.CharField(blank=True, max_length=512, null=True),
+        ),
+    ]

statistics_api/history/models.py

@@ -9,9 +9,9 @@ class History(models.Model):
     context_id = models.IntegerField(null=True, blank=True)
     context_type = models.CharField(max_length=255, null=True, blank=True)
     visited_at = models.DateTimeField(auto_now_add= False, auto_now=False)
-    visited_url = models.URLField(null=True, blank=True)
+    visited_url = models.URLField(max_length=1024, null=True, blank=True)
     interaction_seconds = models.IntegerField(blank=True, null=True)
     asset_icon = models.CharField(max_length=255, null=True, blank=True)
     asset_readable_category = models.CharField(max_length=255, null=True, blank=True)
-    asset_name = models.CharField(max_length=255, null=True, blank=True)
+    asset_name = models.CharField(max_length=512, null=True, blank=True)
     context_name = models.CharField(max_length=255, null=True, blank=True)

statistics_api/history/views.py

@@ -1,34 +1,54 @@
 from rest_framework import serializers
 from rest_framework.response import Response
 from rest_framework.decorators import api_view
+from django.core.exceptions import ObjectDoesNotExist
 
 from statistics_api.history.models import History
 # Create your views here.
 
 @api_view(('GET',))
 def user_history(self, user_id: int):
-    query = History.objects.all().filter(canvas_userid = user_id)
-    result = HistorySerializer(query, many=True)
-    return Response(result.data)
+    try:
+        query = History.objects.all().filter(canvas_userid = user_id)
+        result = HistorySerializer(query, many=True)
+        return Response(result.data)
+    except ObjectDoesNotExist:
+        return Response({"Error": f"User with ID {user_id} not found"}, status=404)
+    except Exception as e:
+        return Response({"Error": str(e)}, status=500)
 
 @api_view(('GET',))
 def user_history_on_context(self, user_id: int, context_id: int):
-    history_events = History.objects.all().filter(canvas_userid = user_id, context_id = context_id)
-    statistics = activity_history(history_events)
-    return Response({"Result": statistics})
+    try:
+        history_events = History.objects.all().filter(canvas_userid = user_id, context_id = context_id)
+        statistics = activity_history(history_events)
+        return Response({"Result": statistics})
+    except ObjectDoesNotExist:
+        return Response({"Error": f"No history found for user {user_id} in context {context_id}"}, status=404)
+    except Exception as e:
+        return Response({"Error": str(e)}, status=500)
 
 @api_view(('GET',))
 def context_history(self, context_id: int):
-    history_events = History.objects.all().filter(context_id = context_id)
-    statistics = activity_history(history_events)
-    return Response({"Result": statistics})
+    try:
+        history_events = History.objects.all().filter(context_id = context_id)
+        statistics = activity_history(history_events)
+        return Response({"Result": statistics})
+    except ObjectDoesNotExist:
+        return Response({"Error": f"No history found for context {context_id}"}, status=404)
+    except Exception as e:
+        return Response({"Error": str(e)}, status=500)
 
 @api_view(('GET',))
 def user_aggregated_history(self, user_id: int):
-    history_events = History.objects.all().filter(canvas_userid = user_id)
-    statistics = activity_history(history_events)
-    return Response({"Result": statistics})
-
+    try:
+        history_events = History.objects.all().filter(canvas_userid = user_id)
+        statistics = activity_history(history_events)
+        return Response({"Result": statistics})
+    except ObjectDoesNotExist:
+        return Response({"Error": f"No history found for user {user_id}"}, status=404)
+    except Exception as e:
+        return Response({"Error": str(e)}, status=500)
 
 def activity_history(history_events) -> list:
     all_visited_pages = history_events.values('asset_code', 'asset_name', 'context_name').distinct()
@@ -37,18 +57,19 @@ def activity_history(history_events) -> list:
         code = page.get('asset_code')
         filtered_list = [obj for obj in history_events if obj.asset_code==code]
         sorted_by_visit = sorted(filtered_list, key=lambda h: h.visited_at, reverse=True)
-        last_visited = sorted_by_visit[0].visited_at
-        seconds_sum = sum(filter(None, (obj.interaction_seconds for obj in filtered_list)))
-        count = len(filtered_list)
-        statistics = {
-            "asset_code" : code,
-            "visits" : count,
-            "time_spent_seconds" : seconds_sum,
-            "last_visted" : last_visited,
-            "asset_name" : page.get('asset_name'),
-            "context_name" : page.get('context_name')
-        }
-        all_statistics.append(statistics)
+        if sorted_by_visit:
+            last_visited = sorted_by_visit[0].visited_at
+            seconds_sum = sum(filter(None, (obj.interaction_seconds for obj in filtered_list)))
+            count = len(filtered_list)
+            statistics = {
+                "asset_code" : code,
+                "visits" : count,
+                "time_spent_seconds" : seconds_sum,
+                "last_visted" : last_visited,
+                "asset_name" : page.get('asset_name'),
+                "context_name" : page.get('context_name')
+            }
+            all_statistics.append(statistics)
     return all_statistics
 
 class HistorySerializer(serializers.ModelSerializer):

statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py

@@ -14,11 +14,12 @@ def handle(self, *args, **options):
         logger = logging.getLogger()
         commands = (
             "pull_total_students_counts_from_courses",
-            "trigger_scheduling_of_kpas_job",
             "pull_course_member_counts_from_canvas",
-            "fetch_course_enrollment_and_post_to_kpas",
+            "fetch_course_enrollment_activity",
             "pull_data_from_matomo",
-            "pull_finnish_marks_canvas")
+            "pull_finnish_marks_canvas",
+            "pull_history_from_canvas_and_update_db",
+            )
 
         for command in commands:
             try:
@@ -27,4 +28,4 @@ def handle(self, *args, **options):
                 management.call_command(command)
                 logger.info(f"Command {command} finished")
             except (JSONDecodeError, AssertionError) as e:
-                logger.critical(e)
+                logger.error(e)

statistics_api/management/commands/fetch_course_enrollment_and_post_to_kpas.py → statistics_api/management/commands/fetch_course_enrollment_activity.py

@@ -1,4 +1,3 @@
-import asyncio
 import datetime
 import logging
 import sys
@@ -10,35 +9,36 @@
 from python_graphql_client import GraphqlClient
 
 from statistics_api.clients.canvas_api_client import CanvasApiClient
-from statistics_api.clients.kpas_client import KpasClient
 from statistics_api.definitions import CANVAS_DOMAIN, CANVAS_ACCESS_KEY, CA_FILE_PATH, CANVAS_ACCOUNT_ID
 from statistics_api.enrollment_activity.models import EnrollmentActivity as EnrollmentActivityModel
 
+logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+logger = logging.getLogger()
 
 class Command(BaseCommand):
     help = """Retrieves per-course enrollment activity for all courses administrated by the Canvas account ID
             set in environment settings."""
 
     def handle(self, *args, **options):
-        logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
-        logger = logging.getLogger()
         logger.info("Starting fetching course enrollment activity from Canvas")
-        api_client = CanvasApiClient()
-        canvas_account_id: int = CANVAS_ACCOUNT_ID if CANVAS_ACCOUNT_ID else api_client.get_canvas_account_id_of_current_user()
-        courses = api_client.get_courses(canvas_account_id=canvas_account_id)
-        for course in courses:
-            course_enrollment = EnrollmentActivity(graphql_api_url="https://{}/api/graphql".format(CANVAS_DOMAIN),
-                                                   course_id=int(course['id']),
-                                                   access_token=CANVAS_ACCESS_KEY, logger=logger)
-            course_enrollment.fetch_enrollment_activity()
-        logger.info("Finished fetching course enrollment activity from Canvas")
-
+        try:
+            api_client = CanvasApiClient()
+            canvas_account_id: int = CANVAS_ACCOUNT_ID if CANVAS_ACCOUNT_ID else api_client.get_canvas_account_id_of_current_user()
+            courses = api_client.get_courses(canvas_account_id=canvas_account_id)
+            for course in courses:
+                try:
+                    course_enrollment = EnrollmentActivity(graphql_api_url="https://{}/api/graphql".format(CANVAS_DOMAIN),
+						course_id=int(course['id']), access_token=CANVAS_ACCESS_KEY)
+                    course_enrollment.fetch_enrollment_activity()
+                except Exception as e:
+                    logger.error("Error processing course ID %s: %s", course['id'], str(e))
+            logger.info("Finished fetching course enrollment activity from Canvas")
+        except Exception as e:
+            logger.error("Error fetching courses from Canvas: %s", str(e))
 
 class EnrollmentActivity(object):
-    def __init__(self, access_token: str, graphql_api_url: str, course_id: int, logger: Logger) -> None:
-        self.logger = logger
+    def __init__(self, access_token: str, graphql_api_url: str, course_id: int) -> None:
         self.access_token = access_token
-        self.kpas_client = KpasClient()
         self.course_id = course_id
         self.headers = {'Authorization': 'Bearer ' + self.access_token,
                         "Content-Type": "application/json"}
@@ -81,7 +81,7 @@ def fetch_enrollment_activity(self):
         try:
             result = self.client.execute(query=self.query, variables=self.variables)
         except Exception as err:
-            print("EnrollmentActivity error : {0}".format(err))
+            logger.error("EnrollmentActivity error : {0}".format(err))
             raise
         active_users_count += filter_enrollment_activity_by_date(result)
         second_query = """
@@ -109,31 +109,28 @@ def fetch_enrollment_activity(self):
 
         while result['data']['course']['enrollmentsConnection']['pageInfo']['hasNextPage']:
           checked_nodes += len(result['data']['course']['enrollmentsConnection']['edges'])
-          # self.logger.info(f"Checked activity for {checked_nodes} out of {self.total_nr_of_students_for_course}")
           after_cursor = result['data']['course']['enrollmentsConnection']['pageInfo']['endCursor']
           self.variables["after"] = after_cursor
           try:
             result = self.client.execute(query=second_query, variables=self.variables)
             active_users_count += filter_enrollment_activity_by_date(result)
           except Exception as err:
-            print("EnrollmentActivity error: {0}".format(err))
+            logger.error("EnrollmentActivity error: {0}".format(err))
             raise
 
         yesterday = datetime.datetime.now().replace(tzinfo=datetime.timezone.utc) - datetime.timedelta(days=1)
         enrollment_activity['activity_date'] = yesterday
         enrollment_activity['active_users_count'] = active_users_count
         enrollment_activity['course_id'] = self.course_id
         enrollment_activity['course_name'] = result['data']['course']['name'].strip()
-        self.logger.info(f"saving {enrollment_activity} to DB")
 
         created_enrollment_object = EnrollmentActivityModel(
-            course_id=enrollment_activity['course_id'],
-            course_name=enrollment_activity['course_name'],
-            active_users_count=enrollment_activity['active_users_count'],
-            activity_date=enrollment_activity['activity_date']
+          course_id=enrollment_activity['course_id'],
+          course_name=enrollment_activity['course_name'],
+          active_users_count=enrollment_activity['active_users_count'],
+          activity_date=enrollment_activity['activity_date']
         )
         created_enrollment_object.save()
-        self.logger.info(f"{created_enrollment_object} created in DB")
 
 
 def filter_enrollment_activity_by_date(data):
@@ -142,8 +139,8 @@ def filter_enrollment_activity_by_date(data):
     :param data: Dict
     :return:
     """
-    edges = data["data"]["course"]["enrollmentsConnection"]["edges"]
-    active_users_yesterday = list(filter(compare_date, edges))
+    edges = data.get("data", {}).get("course", {}).get("enrollmentsConnection", {}).get("edges", [])
+    active_users_yesterday = [edge for edge in edges if compare_date(edge.get("node", {}))]
     return len(active_users_yesterday)
 
 
@@ -153,8 +150,9 @@ def compare_date(node):
     :param node: Enrollment object
     :return: boolean
     """
-    if not node["node"]['lastActivityAt']:
+    last_activity_at = node.get('lastActivityAt')
+    if not last_activity_at:
         return False
     yesterday = arrow.utcnow().shift(days=-1)
-    last_activity_at = arrow.get(node["node"]['lastActivityAt'])
+    last_activity_at = arrow.get(last_activity_at)
     return last_activity_at >= yesterday