From 3c4c7e7f67fdc294784abb1164f55a7fdbd1bd1c Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Thu, 15 Feb 2024 15:16:59 +0100 Subject: [PATCH 01/13] KURSP-1099: improve errorhandling in matomo command --- .../commands/pull_data_from_matomo.py | 54 ++++++++++------- statistics_api/matomo/views.py | 58 +++++++++++-------- 2 files changed, 65 insertions(+), 47 deletions(-) diff --git a/statistics_api/management/commands/pull_data_from_matomo.py b/statistics_api/management/commands/pull_data_from_matomo.py index f121be2..88c33b6 100644 --- a/statistics_api/management/commands/pull_data_from_matomo.py +++ b/statistics_api/management/commands/pull_data_from_matomo.py @@ -9,22 +9,25 @@ from statistics_api.clients.matomo_api_client import MatomoApiClient from statistics_api.matomo.models import Visits, PageStatistics - +logging.basicConfig(stream=sys.stderr, level=logging.INFO) +logger = logging.getLogger() class Command(BaseCommand): def handle(self, *args, **options): - logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - logger = logging.getLogger() logger.info("Starting pulling data from Matomo") matomo_api_client = MatomoApiClient() yesterday = date.today() - timedelta(1) - self.fetch_visits_and_visitors(matomo_api_client, yesterday) - self.fetch_page_statistics(matomo_api_client, yesterday) - logger.info("Finished pulling data from Matomo") + try: + self.fetch_visits_and_visitors(matomo_api_client, yesterday) + self.fetch_page_statistics(matomo_api_client, yesterday) + logger.info("Finished pulling data from Matomo") + except Exception as e: + logger.error("Error pulling data from Matomo: " + str(e)) @transaction.atomic def fetch_visits_and_visitors(self, matomo_api_client, date): '''fetch daily visits and visitors whole domain count''' + visits = matomo_api_client.get_matomo_visits() unique_visitors = matomo_api_client.get_matomo_unique_visitors() frequency = matomo_api_client.get_matomo_visit_frequency() @@ -51,12 +54,14 @@ def fetch_visits_and_visitors(self, matomo_api_client, date): 'actions_per_visit_returning': frequency.get('nb_actions_per_visit_returning'), }) + @transaction.atomic def fetch_page_statistics(self, matomo_api_client, date): '''fetch daily statistics for each url in domain''' pages = matomo_api_client.get_matomo_page_statistics() self.page_statistics(date, pages, None) + def page_statistics(self, date, pages, canvas_course_id): '''update db with page statistics''' for page in pages: @@ -80,24 +85,29 @@ def main_course_page(self, date, pages): self.page_statistics(date, page['subtable'], canvas_course_id) def update_db(self, date, page, canvas_course_id): + # Ignore login pages, this should be done in a better way + # when we have more information about how matomo statistics will be used if(page.get('url') != None and 'login/saml' in page.get('url') or 'login_hint' in page.get('label') or page.get('segment') != None and 'login_hint' in page.get('segment')): return - print(page.get('segment')) - PageStatistics.objects.create( - date=date, - label=page.get('label'), - url=page.get('url'), - segment=page.get('segment'), - visits=page.get('nb_visits'), - sum_time_spent=page.get('sum_time_spent'), - average_time_spent=page.get('avg_time_on_page'), - unique_visitors=page.get('nb_uniq_visitors'), - bounce_rate=page.get('bounce_rate'), - exit_rate=page.get('exit_rate'), - exit_visits=page.get('exit_nb_visits'), - entry_visits=page.get('entry_nb_visits'), - canvas_course_id=canvas_course_id - ) + try: + PageStatistics.objects.create( + date=date, + label=page.get('label'), + url=page.get('url'), + segment=page.get('segment'), + visits=page.get('nb_visits'), + sum_time_spent=page.get('sum_time_spent'), + average_time_spent=page.get('avg_time_on_page'), + unique_visitors=page.get('nb_uniq_visitors'), + bounce_rate=page.get('bounce_rate'), + exit_rate=page.get('exit_rate'), + exit_visits=page.get('exit_nb_visits'), + entry_visits=page.get('entry_nb_visits'), + canvas_course_id=canvas_course_id + ) + except Exception as e: + logger.error("Error updating page: " + page.get('url') + ", statistics from Matomo: " + str(e)) + raise e \ No newline at end of file diff --git a/statistics_api/matomo/views.py b/statistics_api/matomo/views.py index ecf1048..15959f8 100644 --- a/statistics_api/matomo/views.py +++ b/statistics_api/matomo/views.py @@ -1,5 +1,6 @@ from rest_framework import serializers from rest_framework.response import Response +from django.core.exceptions import ValidationError from rest_framework.decorators import api_view from statistics_api.matomo.models import Visits, PageStatistics @@ -7,41 +8,48 @@ # Create your views here. -@api_view(('GET',)) -def visits_statistics(request): - queryset = Visits.objects.all() - from_date = request.GET.get('from', None) - to_date = request.GET.get('to', None) +def filterTimeFrame(queryset, from_date, to_date): if from_date: queryset = queryset.filter(date__gte=from_date) if to_date: queryset = queryset.filter(date__lte=to_date) - result = VisitsSerializer(queryset, many=True) - return Response(result.data) + return queryset + +@api_view(('GET',)) +def visits_statistics(request): + try: + queryset = Visits.objects.all() + queryset = filterTimeFrame(queryset, request.GET.get('from', None), request.GET.get('to', None)) + result = VisitsSerializer(queryset, many=True) + return Response(result.data) + except ValidationError as e: + return Response({'error': str(e)}, status=400) + except Exception as e: + return Response({'error': 'An unexpected error occurred'}, status=500) @api_view(('GET',)) def page_statistics(request): - queryset = PageStatistics.objects.all() - from_date = request.GET.get('from', None) - to_date = request.GET.get('to', None) - if from_date: - queryset = queryset.filter(date__gte=from_date) - if to_date: - queryset = queryset.filter(date__lte=to_date) - result = PageStatisticsSerializer(queryset, many=True) - return Response(result.data) + try: + queryset = PageStatistics.objects.all() + queryset = filterTimeFrame(queryset, request.GET.get('from', None), request.GET.get('to', None)) + result = PageStatisticsSerializer(queryset, many=True) + return Response(result.data) + except ValidationError as e: + return Response({'error': str(e)}, status=400) + except Exception as e: + return Response({'error': 'An unexpected error occurred'}, status=500) @api_view(('GET',)) def course_pages_statistics(request, canvas_course_id: int): - queryset = PageStatistics.objects.all().filter(canvas_course_id = canvas_course_id) - from_date = request.GET.get('from', None) - to_date = request.GET.get('to', None) - if from_date: - queryset = queryset.filter(date__gte=from_date) - if to_date: - queryset = queryset.filter(date__lte=to_date) - result = PageStatisticsSerializer(queryset, many=True) - return Response(result.data) + try: + queryset = PageStatistics.objects.all().filter(canvas_course_id = canvas_course_id) + queryset = filterTimeFrame(queryset, request.GET.get('from', None), request.GET.get('to', None)) + result = PageStatisticsSerializer(queryset, many=True) + return Response(result.data) + except ValidationError as e: + return Response({'error': str(e)}, status=400) + except Exception as e: + return Response({'error': 'An unexpected error occurred'}, status=500) class VisitsSerializer(serializers.ModelSerializer): From 0f777ce4158dddc57e97ff05de15f8b801c5b3da Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Mon, 19 Feb 2024 09:23:22 +0100 Subject: [PATCH 02/13] KURSP-1101: Error handling history --- statistics_api/clients/kpas_client.py | 5 +- statistics_api/history/views.py | 37 ++++++++---- .../do_all_scheduled_maintenance_jobs.py | 7 ++- ...etch_course_enrollment_and_post_to_kpas.py | 19 +++--- .../commands/pull_finnish_marks_canvas.py | 5 +- .../pull_history_from_canvas_and_update_db.py | 59 ++++++++++--------- .../trigger_scheduling_of_kpas_job.py | 17 ------ 7 files changed, 72 insertions(+), 77 deletions(-) delete mode 100644 statistics_api/management/commands/trigger_scheduling_of_kpas_job.py diff --git a/statistics_api/clients/kpas_client.py b/statistics_api/clients/kpas_client.py index dff87db..d596deb 100644 --- a/statistics_api/clients/kpas_client.py +++ b/statistics_api/clients/kpas_client.py @@ -32,7 +32,7 @@ def get_schools_by_county_id(self, county_id: int) -> Tuple[Dict]: print("Received invalid json") return None return tuple(response_json.get("result")) - + def get_municipalities_by_county_id(self, county_id): web_response = self.web_session.get(f"{KPAS_NSR_API_URL}/counties/{county_id}/communities/") @@ -43,9 +43,6 @@ def get_municipalities_by_county_id(self, county_id): print("Received invalid json") return None - def post_trigger_to_activate_schedule_of_job(self) -> None: - web_response = self.web_session.post(f"{KPAS_API_URL}/run_scheduler") - assert web_response.status_code == 200 def get_county(self, county_id: int) -> Union[Dict, None]: web_response = self.web_session.get(f"{KPAS_NSR_API_URL}/counties/{county_id}") diff --git a/statistics_api/history/views.py b/statistics_api/history/views.py index 6df9763..91c535f 100644 --- a/statistics_api/history/views.py +++ b/statistics_api/history/views.py @@ -7,28 +7,39 @@ @api_view(('GET',)) def user_history(self, user_id: int): - query = History.objects.all().filter(canvas_userid = user_id) - result = HistorySerializer(query, many=True) - return Response(result.data) + try: + query = History.objects.all().filter(canvas_userid = user_id) + result = HistorySerializer(query, many=True) + return Response(result.data) + except Exception as e: + return Response({"Error": str(e)}, status=500) @api_view(('GET',)) def user_history_on_context(self, user_id: int, context_id: int): - history_events = History.objects.all().filter(canvas_userid = user_id, context_id = context_id) - statistics = activity_history(history_events) - return Response({"Result": statistics}) + try: + history_events = History.objects.all().filter(canvas_userid = user_id, context_id = context_id) + statistics = activity_history(history_events) + return Response({"Result": statistics}) + except Exception as e: + return Response({"Error": str(e)}, status=500) @api_view(('GET',)) def context_history(self, context_id: int): - history_events = History.objects.all().filter(context_id = context_id) - statistics = activity_history(history_events) - return Response({"Result": statistics}) + try: + history_events = History.objects.all().filter(context_id = context_id) + statistics = activity_history(history_events) + return Response({"Result": statistics}) + except Exception as e: + return Response({"Error": str(e)}, status=500) @api_view(('GET',)) def user_aggregated_history(self, user_id: int): - history_events = History.objects.all().filter(canvas_userid = user_id) - statistics = activity_history(history_events) - return Response({"Result": statistics}) - + try: + history_events = History.objects.all().filter(canvas_userid = user_id) + statistics = activity_history(history_events) + return Response({"Result": statistics}) + except Exception as e: + return Response({"Error": str(e)}, status=500) def activity_history(history_events) -> list: all_visited_pages = history_events.values('asset_code', 'asset_name', 'context_name').distinct() diff --git a/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py b/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py index be10b2a..ad01a89 100644 --- a/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py +++ b/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py @@ -14,11 +14,12 @@ def handle(self, *args, **options): logger = logging.getLogger() commands = ( "pull_total_students_counts_from_courses", - "trigger_scheduling_of_kpas_job", "pull_course_member_counts_from_canvas", "fetch_course_enrollment_and_post_to_kpas", "pull_data_from_matomo", - "pull_finnish_marks_canvas") + "pull_finnish_marks_canvas" + "pull_history_from_canvas_and_update_db", + ) for command in commands: try: @@ -27,4 +28,4 @@ def handle(self, *args, **options): management.call_command(command) logger.info(f"Command {command} finished") except (JSONDecodeError, AssertionError) as e: - logger.critical(e) + logger.error(e) diff --git a/statistics_api/management/commands/fetch_course_enrollment_and_post_to_kpas.py b/statistics_api/management/commands/fetch_course_enrollment_and_post_to_kpas.py index 3937a99..3a4b0ca 100644 --- a/statistics_api/management/commands/fetch_course_enrollment_and_post_to_kpas.py +++ b/statistics_api/management/commands/fetch_course_enrollment_and_post_to_kpas.py @@ -14,14 +14,14 @@ from statistics_api.definitions import CANVAS_DOMAIN, CANVAS_ACCESS_KEY, CA_FILE_PATH, CANVAS_ACCOUNT_ID from statistics_api.enrollment_activity.models import EnrollmentActivity as EnrollmentActivityModel +logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) +logger = logging.getLogger() class Command(BaseCommand): help = """Retrieves per-course enrollment activity for all courses administrated by the Canvas account ID set in environment settings.""" def handle(self, *args, **options): - logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - logger = logging.getLogger() logger.info("Starting fetching course enrollment activity from Canvas") api_client = CanvasApiClient() canvas_account_id: int = CANVAS_ACCOUNT_ID if CANVAS_ACCOUNT_ID else api_client.get_canvas_account_id_of_current_user() @@ -81,7 +81,7 @@ def fetch_enrollment_activity(self): try: result = self.client.execute(query=self.query, variables=self.variables) except Exception as err: - print("EnrollmentActivity error : {0}".format(err)) + logger("EnrollmentActivity error : {0}".format(err)) raise active_users_count += filter_enrollment_activity_by_date(result) second_query = """ @@ -109,14 +109,13 @@ def fetch_enrollment_activity(self): while result['data']['course']['enrollmentsConnection']['pageInfo']['hasNextPage']: checked_nodes += len(result['data']['course']['enrollmentsConnection']['edges']) - # self.logger.info(f"Checked activity for {checked_nodes} out of {self.total_nr_of_students_for_course}") after_cursor = result['data']['course']['enrollmentsConnection']['pageInfo']['endCursor'] self.variables["after"] = after_cursor try: result = self.client.execute(query=second_query, variables=self.variables) active_users_count += filter_enrollment_activity_by_date(result) except Exception as err: - print("EnrollmentActivity error: {0}".format(err)) + logger("EnrollmentActivity error: {0}".format(err)) raise yesterday = datetime.datetime.now().replace(tzinfo=datetime.timezone.utc) - datetime.timedelta(days=1) @@ -124,16 +123,14 @@ def fetch_enrollment_activity(self): enrollment_activity['active_users_count'] = active_users_count enrollment_activity['course_id'] = self.course_id enrollment_activity['course_name'] = result['data']['course']['name'].strip() - self.logger.info(f"saving {enrollment_activity} to DB") created_enrollment_object = EnrollmentActivityModel( - course_id=enrollment_activity['course_id'], - course_name=enrollment_activity['course_name'], - active_users_count=enrollment_activity['active_users_count'], - activity_date=enrollment_activity['activity_date'] + course_id=enrollment_activity['course_id'], + course_name=enrollment_activity['course_name'], + active_users_count=enrollment_activity['active_users_count'], + activity_date=enrollment_activity['activity_date'] ) created_enrollment_object.save() - self.logger.info(f"{created_enrollment_object} created in DB") def filter_enrollment_activity_by_date(data): diff --git a/statistics_api/management/commands/pull_finnish_marks_canvas.py b/statistics_api/management/commands/pull_finnish_marks_canvas.py index 790e571..56bcabe 100644 --- a/statistics_api/management/commands/pull_finnish_marks_canvas.py +++ b/statistics_api/management/commands/pull_finnish_marks_canvas.py @@ -10,12 +10,12 @@ from statistics_api.clients.canvas_api_client import CanvasApiClient +logging.basicConfig(stream=sys.stderr, level=logging.INFO) +logger = logging.getLogger() class Command(BaseCommand): def handle(self, *args, **options): - logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - logger = logging.getLogger() logger.info("Starting pulling finnish marks from Canvas") api_client = CanvasApiClient() canvas_account_id: int = CANVAS_ACCOUNT_ID if CANVAS_ACCOUNT_ID else api_client.get_canvas_account_id_of_current_user() @@ -34,6 +34,7 @@ def handle(self, *args, **options): def parse_courses(self, api_client, courses): for course in courses: course_id = course.get('id') + # Skip the course with id 360, because of the size of the course if course_id == 360: continue self.course_modules(api_client, course) diff --git a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py index 96121f8..2a623c2 100644 --- a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py +++ b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py @@ -6,22 +6,23 @@ import logging import sys +logging.basicConfig(stream=sys.stderr, level=logging.INFO) +logger = logging.getLogger() class Command(BaseCommand): - help = """Retrieves per-course enrollment activity for all courses administrated by the Canvas account ID - set in environment settings.""" def handle(self, *args, **options): - logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - logger = logging.getLogger() - logger.info("Starting fetching course enrollment activity from Canvas") + logger.info("Starting fetching recent activity history from Canvas") api_client = CanvasApiClient() yesterday = date.today() - timedelta(1) - accounts = api_client.get_canvas_accounts() - for account in accounts: - users = api_client.get_account_users(account.get("id")) - for user in users: - self.fetch_user_history(api_client, user.get("id"), yesterday) - logger.info("Finished fetching course enrollment activity from Canvas") + try: + accounts = api_client.get_canvas_accounts() + for account in accounts: + users = api_client.get_account_users(account.get("id")) + for user in users: + self.fetch_user_history(api_client, user.get("id"), yesterday) + logger.info("Finished fetching recent activity history from Canvas") + except Exception as e: + logger.error("Error while fetching recent activity history from Canvas: " + str(e)) def fetch_user_history(self, api_client, canvas_userid, date): history_response = api_client.get_user_history(canvas_userid) @@ -29,23 +30,27 @@ def fetch_user_history(self, api_client, canvas_userid, date): filter( lambda x: datetime.strptime(x['visited_at'], '%Y-%m-%d' + 'T' + '%H:%M:%S' + 'Z') >= datetime.combine(date, - datetime.min.time()), + datetime.min.time()), history_response ) ) for event in history: - History.objects.get_or_create( - canvas_userid=canvas_userid, - visited_at=event.get('visited_at'), - defaults={ - 'asset_code': event.get('asset_code'), - 'context_id': event.get('context_id'), - 'context_type': event.get('context_type'), - 'visited_url': event.get('visited_url'), - 'interaction_seconds': event.get('interaction_seconds'), - 'asset_icon': event.get('asset_icon'), - 'asset_readable_category': event.get('asset_readable_category'), - 'asset_name': event.get('asset_name'), - 'context_name': event.get('context_name') - } - ) + try: + History.objects.get_or_create( + canvas_userid=canvas_userid, + visited_at=event.get('visited_at'), + defaults={ + 'asset_code': event.get('asset_code'), + 'context_id': event.get('context_id'), + 'context_type': event.get('context_type'), + 'visited_url': event.get('visited_url'), + 'interaction_seconds': event.get('interaction_seconds'), + 'asset_icon': event.get('asset_icon'), + 'asset_readable_category': event.get('asset_readable_category'), + 'asset_name': event.get('asset_name'), + 'context_name': event.get('context_name') + } + ) + except Exception as e: + logger.error("Error while saving history: " + event.get('context_name') + ", " + str(e)) + continue \ No newline at end of file diff --git a/statistics_api/management/commands/trigger_scheduling_of_kpas_job.py b/statistics_api/management/commands/trigger_scheduling_of_kpas_job.py deleted file mode 100644 index b624271..0000000 --- a/statistics_api/management/commands/trigger_scheduling_of_kpas_job.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.core.management import BaseCommand - -from statistics_api.clients.kpas_client import KpasClient -import logging -import sys - -class Command(BaseCommand): - help = """This command does NOT trigger a scheduled job at the KPAS LTI module, but it ensures that an already scheduled - job remains on schedule.""" - - def handle(self, *args, **options): - logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - logger = logging.getLogger() - kpas_client = KpasClient() - logger.info("Starting trigger scheduling of KPAS job") - kpas_client.post_trigger_to_activate_schedule_of_job() - logger.info("Finished trigger scheduling of KPAS job") From 67cc58ecbe28e00a402c7bcacb22c45c4b13416d Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Mon, 19 Feb 2024 13:25:05 +0100 Subject: [PATCH 03/13] new lines --- statistics_api/management/commands/pull_data_from_matomo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statistics_api/management/commands/pull_data_from_matomo.py b/statistics_api/management/commands/pull_data_from_matomo.py index 88c33b6..4519116 100644 --- a/statistics_api/management/commands/pull_data_from_matomo.py +++ b/statistics_api/management/commands/pull_data_from_matomo.py @@ -110,4 +110,4 @@ def update_db(self, date, page, canvas_course_id): ) except Exception as e: logger.error("Error updating page: " + page.get('url') + ", statistics from Matomo: " + str(e)) - raise e \ No newline at end of file + raise e From e4d0672f1e91677ca60b9637a730c5f60383b9e3 Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Mon, 19 Feb 2024 14:54:58 +0100 Subject: [PATCH 04/13] KURSP-1100 and KURSP-1101: Improve errorhandling for history and enrollment activity commands --- statistics_api/enrollment_activity/views.py | 47 +++++++++------- statistics_api/history/views.py | 34 +++++++---- .../do_all_scheduled_maintenance_jobs.py | 2 +- ...py => fetch_course_enrollment_activity.py} | 43 +++++++------- .../pull_history_from_canvas_and_update_db.py | 56 +++++++++---------- 5 files changed, 100 insertions(+), 82 deletions(-) rename statistics_api/management/commands/{fetch_course_enrollment_and_post_to_kpas.py => fetch_course_enrollment_activity.py} (78%) diff --git a/statistics_api/enrollment_activity/views.py b/statistics_api/enrollment_activity/views.py index a18a54e..b7144e4 100644 --- a/statistics_api/enrollment_activity/views.py +++ b/statistics_api/enrollment_activity/views.py @@ -1,5 +1,6 @@ from rest_framework import viewsets, serializers from rest_framework.response import Response +from django.core.exceptions import ValidationError # Create your views here. from statistics_api.enrollment_activity.models import EnrollmentActivity @@ -13,33 +14,41 @@ class EnrollmentActivityViewSet(viewsets.ViewSet): def list(self, request): from_date = request.GET.get("from") to_date = request.GET.get("to") - if from_date and to_date: - queryset = EnrollmentActivity.objects.filter(activity_date__gte=from_date, activity_date__lte=to_date) - elif from_date: - queryset = EnrollmentActivity.objects.filter(activity_date__gte=from_date) - elif to_date: - queryset = EnrollmentActivity.objects.filter(activity_date__lte=to_date) - else: - queryset = EnrollmentActivity.objects.all() - queryset = queryset.order_by('-activity_date') - serializer = EnrollmentActivitySerializer(queryset, many=True) - return Response(serializer.data) + try: + queryset = self.filter_query(from_date, to_date) + serializer = EnrollmentActivitySerializer(queryset, many=True) + return Response(serializer.data) + except ValidationError as e: + return Response({"Error": str(e)}, status=400) + except Exception as e: + return Response({"Error": str(e)}, status=500) def retrieve(self, request, pk=None): from_date = request.GET.get("from") to_date = request.GET.get("to") + try: + queryset = self.filter_query(from_date, to_date, course_id=pk) + serializer = EnrollmentActivitySerializer(queryset, many=True) + return Response(serializer.data) + except ValidationError as e: + return Response({"Error": str(e)}, status=400) + except Exception as e: + return Response({"Error": str(e)}, status=500) + + def filter_query(self, from_date, to_date, course_id=None): + queryset = EnrollmentActivity.objects.all() + if course_id: + queryset = queryset.filter(course_id=course_id) + if from_date and to_date: - queryset = EnrollmentActivity.objects.filter(course_id=pk, activity_date__gte=from_date, activity_date__lte=to_date) + queryset = queryset.filter(activity_date__gte=from_date, activity_date__lte=to_date) elif from_date: - queryset = EnrollmentActivity.objects.filter(course_id=pk, activity_date__gte=from_date) + queryset = queryset.filter(activity_date__gte=from_date) elif to_date: - queryset = EnrollmentActivity.objects.filter(course_id=pk, activity_date__lte=to_date) - else: - queryset = EnrollmentActivity.objects.filter(course_id=pk) - queryset = queryset.order_by('-activity_date') - serializer = EnrollmentActivitySerializer(queryset, many=True) - return Response(serializer.data) + queryset = queryset.filter(activity_date__lte=to_date) + queryset = queryset.order_by('-activity_date') + return queryset class EnrollmentActivitySerializer(serializers.ModelSerializer): class Meta: diff --git a/statistics_api/history/views.py b/statistics_api/history/views.py index 91c535f..cfecc3c 100644 --- a/statistics_api/history/views.py +++ b/statistics_api/history/views.py @@ -1,6 +1,7 @@ from rest_framework import serializers from rest_framework.response import Response from rest_framework.decorators import api_view +from django.core.exceptions import ObjectDoesNotExist from statistics_api.history.models import History # Create your views here. @@ -11,6 +12,8 @@ def user_history(self, user_id: int): query = History.objects.all().filter(canvas_userid = user_id) result = HistorySerializer(query, many=True) return Response(result.data) + except ObjectDoesNotExist: + return Response({"Error": f"User with ID {user_id} not found"}, status=404) except Exception as e: return Response({"Error": str(e)}, status=500) @@ -20,6 +23,8 @@ def user_history_on_context(self, user_id: int, context_id: int): history_events = History.objects.all().filter(canvas_userid = user_id, context_id = context_id) statistics = activity_history(history_events) return Response({"Result": statistics}) + except ObjectDoesNotExist: + return Response({"Error": f"No history found for user {user_id} in context {context_id}"}, status=404) except Exception as e: return Response({"Error": str(e)}, status=500) @@ -29,6 +34,8 @@ def context_history(self, context_id: int): history_events = History.objects.all().filter(context_id = context_id) statistics = activity_history(history_events) return Response({"Result": statistics}) + except ObjectDoesNotExist: + return Response({"Error": f"No history found for context {context_id}"}, status=404) except Exception as e: return Response({"Error": str(e)}, status=500) @@ -38,6 +45,8 @@ def user_aggregated_history(self, user_id: int): history_events = History.objects.all().filter(canvas_userid = user_id) statistics = activity_history(history_events) return Response({"Result": statistics}) + except ObjectDoesNotExist: + return Response({"Error": f"No history found for user {user_id}"}, status=404) except Exception as e: return Response({"Error": str(e)}, status=500) @@ -48,18 +57,19 @@ def activity_history(history_events) -> list: code = page.get('asset_code') filtered_list = [obj for obj in history_events if obj.asset_code==code] sorted_by_visit = sorted(filtered_list, key=lambda h: h.visited_at, reverse=True) - last_visited = sorted_by_visit[0].visited_at - seconds_sum = sum(filter(None, (obj.interaction_seconds for obj in filtered_list))) - count = len(filtered_list) - statistics = { - "asset_code" : code, - "visits" : count, - "time_spent_seconds" : seconds_sum, - "last_visted" : last_visited, - "asset_name" : page.get('asset_name'), - "context_name" : page.get('context_name') - } - all_statistics.append(statistics) + if sorted_by_visit: + last_visited = sorted_by_visit[0].visited_at + seconds_sum = sum(filter(None, (obj.interaction_seconds for obj in filtered_list))) + count = len(filtered_list) + statistics = { + "asset_code" : code, + "visits" : count, + "time_spent_seconds" : seconds_sum, + "last_visted" : last_visited, + "asset_name" : page.get('asset_name'), + "context_name" : page.get('context_name') + } + all_statistics.append(statistics) return all_statistics class HistorySerializer(serializers.ModelSerializer): diff --git a/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py b/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py index ad01a89..9710305 100644 --- a/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py +++ b/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py @@ -15,7 +15,7 @@ def handle(self, *args, **options): commands = ( "pull_total_students_counts_from_courses", "pull_course_member_counts_from_canvas", - "fetch_course_enrollment_and_post_to_kpas", + "fetch_course_enrollment_activity", "pull_data_from_matomo", "pull_finnish_marks_canvas" "pull_history_from_canvas_and_update_db", diff --git a/statistics_api/management/commands/fetch_course_enrollment_and_post_to_kpas.py b/statistics_api/management/commands/fetch_course_enrollment_activity.py similarity index 78% rename from statistics_api/management/commands/fetch_course_enrollment_and_post_to_kpas.py rename to statistics_api/management/commands/fetch_course_enrollment_activity.py index 3a4b0ca..51bdd1d 100644 --- a/statistics_api/management/commands/fetch_course_enrollment_and_post_to_kpas.py +++ b/statistics_api/management/commands/fetch_course_enrollment_activity.py @@ -1,4 +1,3 @@ -import asyncio import datetime import logging import sys @@ -10,7 +9,6 @@ from python_graphql_client import GraphqlClient from statistics_api.clients.canvas_api_client import CanvasApiClient -from statistics_api.clients.kpas_client import KpasClient from statistics_api.definitions import CANVAS_DOMAIN, CANVAS_ACCESS_KEY, CA_FILE_PATH, CANVAS_ACCOUNT_ID from statistics_api.enrollment_activity.models import EnrollmentActivity as EnrollmentActivityModel @@ -23,22 +21,24 @@ class Command(BaseCommand): def handle(self, *args, **options): logger.info("Starting fetching course enrollment activity from Canvas") - api_client = CanvasApiClient() - canvas_account_id: int = CANVAS_ACCOUNT_ID if CANVAS_ACCOUNT_ID else api_client.get_canvas_account_id_of_current_user() - courses = api_client.get_courses(canvas_account_id=canvas_account_id) - for course in courses: - course_enrollment = EnrollmentActivity(graphql_api_url="https://{}/api/graphql".format(CANVAS_DOMAIN), - course_id=int(course['id']), - access_token=CANVAS_ACCESS_KEY, logger=logger) - course_enrollment.fetch_enrollment_activity() - logger.info("Finished fetching course enrollment activity from Canvas") - + try: + api_client = CanvasApiClient() + canvas_account_id: int = CANVAS_ACCOUNT_ID if CANVAS_ACCOUNT_ID else api_client.get_canvas_account_id_of_current_user() + courses = api_client.get_courses(canvas_account_id=canvas_account_id) + for course in courses: + try: + course_enrollment = EnrollmentActivity(graphql_api_url="https://{}/api/graphql".format(CANVAS_DOMAIN), + course_id=int(course['id']), access_token=CANVAS_ACCESS_KEY) + course_enrollment.fetch_enrollment_activity() + except Exception as e: + logger.error("Error processing course ID %s: %s", course['id'], str(e)) + logger.info("Finished fetching course enrollment activity from Canvas") + except Exception as e: + logger.error("Error fetching courses from Canvas: %s", str(e)) class EnrollmentActivity(object): - def __init__(self, access_token: str, graphql_api_url: str, course_id: int, logger: Logger) -> None: - self.logger = logger + def __init__(self, access_token: str, graphql_api_url: str, course_id: int) -> None: self.access_token = access_token - self.kpas_client = KpasClient() self.course_id = course_id self.headers = {'Authorization': 'Bearer ' + self.access_token, "Content-Type": "application/json"} @@ -81,7 +81,7 @@ def fetch_enrollment_activity(self): try: result = self.client.execute(query=self.query, variables=self.variables) except Exception as err: - logger("EnrollmentActivity error : {0}".format(err)) + logger.error("EnrollmentActivity error : {0}".format(err)) raise active_users_count += filter_enrollment_activity_by_date(result) second_query = """ @@ -115,7 +115,7 @@ def fetch_enrollment_activity(self): result = self.client.execute(query=second_query, variables=self.variables) active_users_count += filter_enrollment_activity_by_date(result) except Exception as err: - logger("EnrollmentActivity error: {0}".format(err)) + logger.error("EnrollmentActivity error: {0}".format(err)) raise yesterday = datetime.datetime.now().replace(tzinfo=datetime.timezone.utc) - datetime.timedelta(days=1) @@ -139,8 +139,8 @@ def filter_enrollment_activity_by_date(data): :param data: Dict :return: """ - edges = data["data"]["course"]["enrollmentsConnection"]["edges"] - active_users_yesterday = list(filter(compare_date, edges)) + edges = data.get("data", {}).get("course", {}).get("enrollmentsConnection", {}).get("edges", []) + active_users_yesterday = [edge for edge in edges if compare_date(edge.get("node", {}))] return len(active_users_yesterday) @@ -150,8 +150,9 @@ def compare_date(node): :param node: Enrollment object :return: boolean """ - if not node["node"]['lastActivityAt']: + last_activity_at = node.get('lastActivityAt') + if not last_activity_at: return False yesterday = arrow.utcnow().shift(days=-1) - last_activity_at = arrow.get(node["node"]['lastActivityAt']) + last_activity_at = arrow.get(last_activity_at) return last_activity_at >= yesterday diff --git a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py index 2a623c2..183afed 100644 --- a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py +++ b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py @@ -25,32 +25,30 @@ def handle(self, *args, **options): logger.error("Error while fetching recent activity history from Canvas: " + str(e)) def fetch_user_history(self, api_client, canvas_userid, date): - history_response = api_client.get_user_history(canvas_userid) - history = list( - filter( - lambda x: - datetime.strptime(x['visited_at'], '%Y-%m-%d' + 'T' + '%H:%M:%S' + 'Z') >= datetime.combine(date, - datetime.min.time()), - history_response - ) - ) - for event in history: - try: - History.objects.get_or_create( - canvas_userid=canvas_userid, - visited_at=event.get('visited_at'), - defaults={ - 'asset_code': event.get('asset_code'), - 'context_id': event.get('context_id'), - 'context_type': event.get('context_type'), - 'visited_url': event.get('visited_url'), - 'interaction_seconds': event.get('interaction_seconds'), - 'asset_icon': event.get('asset_icon'), - 'asset_readable_category': event.get('asset_readable_category'), - 'asset_name': event.get('asset_name'), - 'context_name': event.get('context_name') - } - ) - except Exception as e: - logger.error("Error while saving history: " + event.get('context_name') + ", " + str(e)) - continue \ No newline at end of file + try: + history_response = api_client.get_user_history(canvas_userid) + history = [ + event for event in history_response + if datetime.strptime(event['visited_at'], '%Y-%m-%dT%H:%M:%SZ') >= datetime.combine(date, datetime.min.time()) + ] + for event in history: + try: + History.objects.get_or_create( + canvas_userid=canvas_userid, + visited_at=event.get('visited_at'), + defaults={ + 'asset_code': event.get('asset_code'), + 'context_id': event.get('context_id'), + 'context_type': event.get('context_type'), + 'visited_url': event.get('visited_url'), + 'interaction_seconds': event.get('interaction_seconds'), + 'asset_icon': event.get('asset_icon'), + 'asset_readable_category': event.get('asset_readable_category'), + 'asset_name': event.get('asset_name'), + 'context_name': event.get('context_name') + } + ) + except Exception as e: + logger.error("Error while saving history: " + event.get('context_name') + ", " + str(e)) + except Exception as e: + logger.error("Error while fetching history for user " + canvas_userid + ": " + str(e)) \ No newline at end of file From f601ce0d3c6645bf7f307958e540143ba641afb4 Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Mon, 19 Feb 2024 14:57:12 +0100 Subject: [PATCH 05/13] new lines --- .../commands/pull_history_from_canvas_and_update_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py index 183afed..dacd3d6 100644 --- a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py +++ b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py @@ -51,4 +51,4 @@ def fetch_user_history(self, api_client, canvas_userid, date): except Exception as e: logger.error("Error while saving history: " + event.get('context_name') + ", " + str(e)) except Exception as e: - logger.error("Error while fetching history for user " + canvas_userid + ": " + str(e)) \ No newline at end of file + logger.error("Error while fetching history for user " + canvas_userid + ": " + str(e)) From 52cc344b7d1deee882e3cc7bf32d3f8a64b013ce Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Tue, 20 Feb 2024 12:43:00 +0100 Subject: [PATCH 06/13] KURSP-1101: missing comma in scheduler --- .../management/commands/do_all_scheduled_maintenance_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py b/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py index 9710305..81f1d3e 100644 --- a/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py +++ b/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py @@ -17,7 +17,7 @@ def handle(self, *args, **options): "pull_course_member_counts_from_canvas", "fetch_course_enrollment_activity", "pull_data_from_matomo", - "pull_finnish_marks_canvas" + "pull_finnish_marks_canvas", "pull_history_from_canvas_and_update_db", ) From f0ef4e835487744c8a39e2ab0c458d5954cb0463 Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Wed, 21 Feb 2024 09:48:26 +0100 Subject: [PATCH 07/13] KURSP-1101: Maxlength visited_url in history table --- .../0005_alter_history_visited_url.py | 18 ++++++++++++++++++ statistics_api/history/models.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 statistics_api/history/migrations/0005_alter_history_visited_url.py diff --git a/statistics_api/history/migrations/0005_alter_history_visited_url.py b/statistics_api/history/migrations/0005_alter_history_visited_url.py new file mode 100644 index 0000000..bc050c5 --- /dev/null +++ b/statistics_api/history/migrations/0005_alter_history_visited_url.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.18 on 2024-02-21 08:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('history', '0004_alter_history_context_id'), + ] + + operations = [ + migrations.AlterField( + model_name='history', + name='visited_url', + field=models.URLField(blank=True, max_length=1024, null=True), + ), + ] diff --git a/statistics_api/history/models.py b/statistics_api/history/models.py index 4149464..ceba49a 100644 --- a/statistics_api/history/models.py +++ b/statistics_api/history/models.py @@ -9,7 +9,7 @@ class History(models.Model): context_id = models.IntegerField(null=True, blank=True) context_type = models.CharField(max_length=255, null=True, blank=True) visited_at = models.DateTimeField(auto_now_add= False, auto_now=False) - visited_url = models.URLField(null=True, blank=True) + visited_url = models.URLField(max_length=1024, null=True, blank=True) interaction_seconds = models.IntegerField(blank=True, null=True) asset_icon = models.CharField(max_length=255, null=True, blank=True) asset_readable_category = models.CharField(max_length=255, null=True, blank=True) From 419b46964c6891681eee17f098051b93dff54be7 Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Thu, 22 Feb 2024 10:16:25 +0100 Subject: [PATCH 08/13] KURSP-1101: temp change to debug history --- .../commands/do_all_scheduled_maintenance_jobs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py b/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py index 81f1d3e..5ea4606 100644 --- a/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py +++ b/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py @@ -13,11 +13,11 @@ def handle(self, *args, **options): logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) logger = logging.getLogger() commands = ( - "pull_total_students_counts_from_courses", - "pull_course_member_counts_from_canvas", - "fetch_course_enrollment_activity", - "pull_data_from_matomo", - "pull_finnish_marks_canvas", + # "pull_total_students_counts_from_courses", + # "pull_course_member_counts_from_canvas", + # "fetch_course_enrollment_activity", + # "pull_data_from_matomo", + # "pull_finnish_marks_canvas", "pull_history_from_canvas_and_update_db", ) From 800a6e064ed8502661a8369e191c460ab2cfb212 Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Thu, 22 Feb 2024 11:36:53 +0100 Subject: [PATCH 09/13] KURSP-1101: temp change to debug history --- .../commands/pull_history_from_canvas_and_update_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py index dacd3d6..ebd177d 100644 --- a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py +++ b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py @@ -50,5 +50,6 @@ def fetch_user_history(self, api_client, canvas_userid, date): ) except Exception as e: logger.error("Error while saving history: " + event.get('context_name') + ", " + str(e)) + logger.info("Asset code: " + event.get('asset_code')) except Exception as e: logger.error("Error while fetching history for user " + canvas_userid + ": " + str(e)) From 5febd9e4b53163b6bfcb6d4b0c950c1b8085a00f Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Thu, 22 Feb 2024 11:49:37 +0100 Subject: [PATCH 10/13] KURSP-1101: temp change to debug history --- .../commands/pull_history_from_canvas_and_update_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py index ebd177d..10af9de 100644 --- a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py +++ b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py @@ -50,6 +50,6 @@ def fetch_user_history(self, api_client, canvas_userid, date): ) except Exception as e: logger.error("Error while saving history: " + event.get('context_name') + ", " + str(e)) - logger.info("Asset code: " + event.get('asset_code')) + logger.info("Asset name: " + event.get('asset_name')) except Exception as e: logger.error("Error while fetching history for user " + canvas_userid + ": " + str(e)) From c7fbbd8de72dd9d72f8d9a2ab3574240a830d64c Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Thu, 22 Feb 2024 12:15:52 +0100 Subject: [PATCH 11/13] KURSP-1101: increase size of asset_name --- .../0006_alter_history_asset_name.py | 18 ++++++++++++++++++ statistics_api/history/models.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 statistics_api/history/migrations/0006_alter_history_asset_name.py diff --git a/statistics_api/history/migrations/0006_alter_history_asset_name.py b/statistics_api/history/migrations/0006_alter_history_asset_name.py new file mode 100644 index 0000000..ec93ddf --- /dev/null +++ b/statistics_api/history/migrations/0006_alter_history_asset_name.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.18 on 2024-02-22 11:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('history', '0005_alter_history_visited_url'), + ] + + operations = [ + migrations.AlterField( + model_name='history', + name='asset_name', + field=models.CharField(blank=True, max_length=512, null=True), + ), + ] diff --git a/statistics_api/history/models.py b/statistics_api/history/models.py index ceba49a..8708a2d 100644 --- a/statistics_api/history/models.py +++ b/statistics_api/history/models.py @@ -13,5 +13,5 @@ class History(models.Model): interaction_seconds = models.IntegerField(blank=True, null=True) asset_icon = models.CharField(max_length=255, null=True, blank=True) asset_readable_category = models.CharField(max_length=255, null=True, blank=True) - asset_name = models.CharField(max_length=255, null=True, blank=True) + asset_name = models.CharField(max_length=512, null=True, blank=True) context_name = models.CharField(max_length=255, null=True, blank=True) From a6a255025c669e3e1312a38fe8aba0a061047459 Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Thu, 22 Feb 2024 12:46:55 +0100 Subject: [PATCH 12/13] KURSP-1101: Set scheduler to run all commands --- .../commands/do_all_scheduled_maintenance_jobs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py b/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py index 5ea4606..81f1d3e 100644 --- a/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py +++ b/statistics_api/management/commands/do_all_scheduled_maintenance_jobs.py @@ -13,11 +13,11 @@ def handle(self, *args, **options): logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) logger = logging.getLogger() commands = ( - # "pull_total_students_counts_from_courses", - # "pull_course_member_counts_from_canvas", - # "fetch_course_enrollment_activity", - # "pull_data_from_matomo", - # "pull_finnish_marks_canvas", + "pull_total_students_counts_from_courses", + "pull_course_member_counts_from_canvas", + "fetch_course_enrollment_activity", + "pull_data_from_matomo", + "pull_finnish_marks_canvas", "pull_history_from_canvas_and_update_db", ) From c4eac1e776374db23a29c76f0aebed5dbbc3966f Mon Sep 17 00:00:00 2001 From: Thea Hvalen Thodesen Date: Thu, 22 Feb 2024 12:49:43 +0100 Subject: [PATCH 13/13] KURSP-1101: remove loggers used during debugging --- .../commands/pull_history_from_canvas_and_update_db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py index 10af9de..dacd3d6 100644 --- a/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py +++ b/statistics_api/management/commands/pull_history_from_canvas_and_update_db.py @@ -50,6 +50,5 @@ def fetch_user_history(self, api_client, canvas_userid, date): ) except Exception as e: logger.error("Error while saving history: " + event.get('context_name') + ", " + str(e)) - logger.info("Asset name: " + event.get('asset_name')) except Exception as e: logger.error("Error while fetching history for user " + canvas_userid + ": " + str(e))