From 096ddc671a8e71d5e8ddce92a7546e9963d49d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Vit=C3=B3ria=20Silva?= <8648976+joaovitoriasilva@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:09:20 +0100 Subject: [PATCH 1/9] Added bulk import functionality [backend] added new route for handling bulk import requests [backend] added new folder for bulk import actions [backend] adjusted code for better re usability for bulk import and single file upload [docker-compose] added new volume mapping in docker compose example file [README] added new volumes information to backend section of README file --- .gitignore | 6 + README.md | 10 +- backend/app/activities/router.py | 144 +++++++++--------- backend/app/activities/utils.py | 115 ++++++++++++++ docker-compose.yml.example | 3 +- .../Settings/SettingsIntegrationsZone.vue | 25 +++ .../settingsIntegrationsZoneComponent.json | 7 +- .../app/src/services/activitiesService.js | 7 +- 8 files changed, 237 insertions(+), 80 deletions(-) diff --git a/.gitignore b/.gitignore index 484d4bd8..1cf36035 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,12 @@ backend/app/user_images/*.jpeg backend/app/user_images/*.png backend/app/user_images/*.jpg +# gpx and fit files +backend/app/uploads/*.gpx +backend/app/uploads/*.fit +backend/app/bulk_import/*.gpx +backend/app/bulk_import/*.fit + # Frontend frontend/app/img/users_img/*.* # Logs diff --git a/README.md b/README.md index 3cf81aa8..6432a145 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Currently the service supports: - Create/edit/delete users - Basic admin and regular user profiles that adapt the interface - Import activities using .gpx files + - Bulk import for activity files - Connect with Strava and retrieve activities and gear from Strava - Feed with user activities, current user week stats and month stats - Feed with followers activities @@ -47,7 +48,6 @@ Currently the service supports: To do features (not by order): - Support import of .fit files - - Bulk import for .gpx and .fit files - Default gear for activity type - Gear components logic for component usage tracking - Comments and likes logic for activities @@ -112,6 +112,14 @@ MYSQL_DATABASE | endurain | `No` | N/A MYSQL_USER | endurain | `No` | N/A MYSQL_PASSWORD | changeme | `No` | N/A +Table bellow shows the volumes available to be configured. Not obligatory, but recommended: + +Volume | Path | Notes +--- | --- | --- +/app | /endurain/backend/app:/app | Configure volume if you want to edit the code locally by cloning the repo and comment next line +/app/user_images | /endurain/backend/app/user_images:/app/user_images | Necessary for user image persistence on container image updates +/app/bulk_import | /endurain/backend/bulkimport | Necessary to enable bulk import of activities. Place here your activities files + To check Python backend dependencies used, use poetry file (pyproject.toml) --- diff --git a/backend/app/activities/router.py b/backend/app/activities/router.py index f4fe5884..bbc34803 100644 --- a/backend/app/activities/router.py +++ b/backend/app/activities/router.py @@ -4,7 +4,15 @@ from typing import Annotated, Callable -from fastapi import APIRouter, Depends, HTTPException, status, UploadFile, Security +from fastapi import ( + APIRouter, + Depends, + HTTPException, + status, + UploadFile, + Security, + BackgroundTasks, +) from sqlalchemy.orm import Session from datetime import datetime, timedelta, timezone @@ -20,12 +28,6 @@ import users.dependencies as users_dependencies -import activity_streams.crud as activity_streams_crud - -import gpx.utils as gpx_utils - -import fit.utils as fit_utils - import database import dependencies_global @@ -105,9 +107,7 @@ async def read_activities_useractivities_thisweek_distances( ): # Calculate the start of the current week today = datetime.now(timezone.utc) - start_of_week = today - timedelta( - days=today.weekday() - ) + start_of_week = today - timedelta(days=today.weekday()) end_of_week = start_of_week + timedelta(days=6) if user_id == token_user_id: @@ -122,8 +122,8 @@ async def read_activities_useractivities_thisweek_distances( ) # Check if activities is None - #if activities is None: - # Return None if activities is None + # if activities is None: + # Return None if activities is None # return None # Return the activities distances for this week @@ -167,8 +167,8 @@ async def read_activities_useractivities_thismonth_distances( user_id, start_of_month, end_of_month, db ) - #if activities is None: - # Return None if activities is None + # if activities is None: + # Return None if activities is None # return None # Return the activities distances for this month @@ -187,9 +187,7 @@ async def read_activities_useractivities_thismonth_number( ], token_user_id: Annotated[ Callable, - Depends( - session_security.get_user_id_from_access_token - ), + Depends(session_security.get_user_id_from_access_token), ], db: Annotated[ Session, @@ -234,9 +232,7 @@ async def read_activities_gearactivities( ], token_user_id: Annotated[ Callable, - Depends( - session_security.get_user_id_from_access_token - ), + Depends(session_security.get_user_id_from_access_token), ], db: Annotated[ Session, @@ -433,51 +429,8 @@ async def create_activity_with_uploaded_file( upload_dir = "uploads" os.makedirs(upload_dir, exist_ok=True) - # Get file extension - _, file_extension = os.path.splitext(file.filename) - - # Save the uploaded file in the 'uploads' directory - with open(file.filename, "wb") as save_file: - save_file.write(file.file.read()) - - # Choose the appropriate parser based on file extension - if file_extension.lower() == ".gpx": - # Parse the GPX file - parsed_info = gpx_utils.parse_gpx_file(file.filename, token_user_id) - elif file_extension.lower() == ".fit": - # Parse the FIT file - parsed_info = fit_utils.parse_fit_file(file.filename, token_user_id) - else: - # file extension not supported raise an HTTPException with a 406 Not Acceptable status code - raise HTTPException( - status_code=status.HTTP_406_NOT_ACCEPTABLE, - detail="File extension not supported. Supported file extensions are .gpx and .fit", - ) - - # create the activity in the database - created_activity = activities_crud.create_activity(parsed_info["activity"], db) - - # Check if created_activity is None - if created_activity is None: - # raise an HTTPException with a 500 Internal Server Error status code - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error creating activity", - ) - - # Parse the activity streams from the parsed info - activity_streams = gpx_utils.parse_activity_streams_from_gpx_file( - parsed_info, created_activity.id - ) - - # Create activity streams in the database - activity_streams_crud.create_activity_streams(activity_streams, db) - - # Remove the file after processing - os.remove(file.filename) - - # Return activity ID - return created_activity + # Return activity + return activies_utils.parse_and_store_activity_from_uploaded_file(token_user_id, file, db) except Exception as err: # Log the exception logger.error( @@ -490,8 +443,55 @@ async def create_activity_with_uploaded_file( ) from err +@router.post( + "/create/bulkimport", +) +async def create_activity_with_bulk_import( + token_user_id: Annotated[ + int, + Depends(session_security.get_user_id_from_access_token), + ], + check_scopes: Annotated[ + Callable, Security(session_security.check_scopes, scopes=["activities:write"]) + ], + db: Annotated[ + Session, + Depends(database.get_db), + ], + background_tasks: BackgroundTasks, +): + try: + # Ensure the 'bulk_import' directory exists + bulk_import_dir = "bulk_import" + os.makedirs(bulk_import_dir, exist_ok=True) + + # Iterate over each file in the 'bulk_import' directory + for filename in os.listdir(bulk_import_dir): + file_path = os.path.join(bulk_import_dir, filename) + + if os.path.isfile(file_path): + # Parse and store the activity + background_tasks.add_task( + activies_utils.parse_and_store_activity_from_file, + token_user_id, + file_path, + db, + ) + + # Return a success message + return {"Bulk import initiated. Processing files in the background."} + except Exception as err: + # Log the exception + logger.error(f"Error in create_activity_with_bulk_import: {err}", exc_info=True) + # Raise an HTTPException with a 500 Internal Server Error status code + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal Server Error", + ) from err + + @router.put( - "/edit", + "/edit", ) async def edit_activity( token_user_id: Annotated[ @@ -529,9 +529,7 @@ async def activity_add_gear( ], token_user_id: Annotated[ int, - Depends( - session_security.get_user_id_from_access_token - ), + Depends(session_security.get_user_id_from_access_token), ], db: Annotated[ Session, @@ -580,9 +578,7 @@ async def delete_activity_gear( ], token_user_id: Annotated[ int, - Depends( - session_security.get_user_id_from_access_token - ), + Depends(session_security.get_user_id_from_access_token), ], db: Annotated[ Session, @@ -621,9 +617,7 @@ async def delete_activity( ], token_user_id: Annotated[ int, - Depends( - session_security.get_user_id_from_access_token - ), + Depends(session_security.get_user_id_from_access_token), ], db: Annotated[ Session, diff --git a/backend/app/activities/utils.py b/backend/app/activities/utils.py index 4591f348..46cc79f4 100644 --- a/backend/app/activities/utils.py +++ b/backend/app/activities/utils.py @@ -3,16 +3,131 @@ import requests import math +from fastapi import HTTPException, status, UploadFile + +from typing import Union from datetime import datetime from urllib.parse import urlencode from statistics import mean +from sqlalchemy.orm import Session + import activities.schema as activities_schema +import activities.crud as activities_crud + +import activity_streams.crud as activity_streams_crud + +import gpx.utils as gpx_utils +import fit.utils as fit_utils # Define a loggger created on main.py logger = logging.getLogger("myLogger") +async def parse_and_store_activity_from_file( + token_user_id: int, file_path: str, db: Session +): + try: + # Get file extension + _, file_extension = os.path.splitext(file_path) + + # Open the file and process it + with open(file_path, "rb") as file: + # Parse the file + parsed_info = parse_file(token_user_id, file_extension, file_path) + + # Store the activity in the database + created_activity = store_activity(parsed_info, db) + + # Return the created activity + return created_activity + except Exception as err: + # Log the exception + logger.error( + f"Error in parse_and_store_activity_from_file: {err}", exc_info=True + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal Server Error", + ) from err + + +def parse_and_store_activity_from_uploaded_file( + token_user_id: int, file: UploadFile, db: Session +): + + # Get file extension + _, file_extension = os.path.splitext(file.filename) + + # Save the uploaded file in the 'uploads' directory + with open(file.filename, "wb") as save_file: + save_file.write(file.file.read()) + + # Parse the file + parsed_info = parse_file(token_user_id, file_extension, file.filename) + + # Store the activity in the database + created_activity = store_activity(parsed_info, db) + + # Return the created activity + return created_activity + + +def parse_file(token_user_id: int, file_extension: str, filename: str) -> dict: + try: + # Choose the appropriate parser based on file extension + if file_extension.lower() == ".gpx": + # Parse the GPX file + parsed_info = gpx_utils.parse_gpx_file(filename, token_user_id) + elif file_extension.lower() == ".fit": + # Parse the FIT file + parsed_info = fit_utils.parse_fit_file(filename, token_user_id) + else: + # file extension not supported raise an HTTPException with a 406 Not Acceptable status code + raise HTTPException( + status_code=status.HTTP_406_NOT_ACCEPTABLE, + detail="File extension not supported. Supported file extensions are .gpx and .fit", + ) + + # Return the parsed information + return parsed_info + except Exception as err: + # Log the exception + logger.error(f"Error in parse_file: {err}", exc_info=True) + # Raise an HTTPException with a 500 Internal Server Error status code + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal Server Error", + ) from err + finally: + # Remove the file after processing + os.remove(filename) + + +def store_activity(parsed_info: dict, db: Session): + # create the activity in the database + created_activity = activities_crud.create_activity(parsed_info["activity"], db) + + # Check if created_activity is None + if created_activity is None: + # raise an HTTPException with a 500 Internal Server Error status code + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Error creating activity", + ) + + # Parse the activity streams from the parsed info + activity_streams = gpx_utils.parse_activity_streams_from_gpx_file( + parsed_info, created_activity.id + ) + + # Create activity streams in the database + activity_streams_crud.create_activity_streams(activity_streams, db) + + # Return the created activity + return created_activity + + def calculate_activity_distances(activities: list[activities_schema.Activity]): """Calculate the distances of the activities for each type of activity (run, bike, swim)""" # Initialize the distances diff --git a/docker-compose.yml.example b/docker-compose.yml.example index 4a568992..e0afd457 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -30,8 +30,9 @@ services: ports: - "98:80" # API port, change per your needs volumes: - # - /endurain/backend/app:/app # Configure volume if you want to edit the code locally by cloning the repo and comment next line + # - /endurain/backend/app:/app # Configure volume if you want to edit the code locally by cloning the repo - /endurain/backend/user_images:/app/user_images # necessary for user image persistence on container image updates + - /endurain/backend/bulkimport:/app/bulk_import # necessary to enable bulk import of activities. Place here your activities files depends_on: - mariadb - jaeger # optional diff --git a/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue b/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue index 76cf2bbd..ba84056b 100644 --- a/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue +++ b/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue @@ -1,6 +1,7 @@ @@ -28,6 +39,7 @@ import { addToast } from '@/utils/toastUtils'; import { useAuthStore } from '@/stores/authStore'; // Importing the services import { strava } from '@/services/stravaService'; +import { activities } from '@/services/activitiesService'; export default { components: { @@ -76,12 +88,25 @@ export default { } } + async function submitBulkImport() { + try { + await activities.bulkImportActivities(); + + // Set the loading message and show the loading alert. + addToast(t('settingsIntegrationsZone.loadingMessageBulkImport'), 'loading', true); + } catch(error) { + // If there is an error, set the error message and show the error alert. + addToast(t('settingsIntegrationsZone.errorMessageUnableToImportActivities') + " - " + error, 'danger', true); + } + } + return { authStore, t, submitConnectStrava, submitRetrieveStravaLastWeekActivities, submitRetrieveStravaGear, + submitBulkImport, }; }, }; diff --git a/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json b/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json index c0bc8085..22f0d293 100644 --- a/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json +++ b/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json @@ -10,5 +10,10 @@ "errorMessageUnableToGetStravaActivities": "Unable to get Strava activities", "errorMessageUnableToGetStravaGear": "Unable to get Strava gear", "loadingMessageRetrievingStravaActivities": "Retrieving Strava activities", - "loadingMessageRetrievingStravaGear": "Retrieving Strava gear" + "loadingMessageRetrievingStravaGear": "Retrieving Strava gear", + "bulkImportIntegrationTitle": "Bulk import", + "bulkImportIntegrationBody": "Bulk import activities from files stored in the bulk_import folder", + "buttonBulkImport": "Import", + "errorMessageUnableToImportActivities": "An error occurred while importing activities", + "loadingMessageBulkImport": "Importing activities" } \ No newline at end of file diff --git a/frontend/app/src/services/activitiesService.js b/frontend/app/src/services/activitiesService.js index 5cb16262..975b0638 100644 --- a/frontend/app/src/services/activitiesService.js +++ b/frontend/app/src/services/activitiesService.js @@ -1,4 +1,4 @@ -import { fetchGetRequest, fetchPostFileRequest, fetchDeleteRequest, fetchPutRequest } from '@/utils/serviceUtils'; +import { fetchGetRequest, fetchPostFileRequest, fetchDeleteRequest, fetchPutRequest, fetchPostRequest } from '@/utils/serviceUtils'; export const activities = { getUserWeekActivities(user_id, week_number) { @@ -34,6 +34,9 @@ export const activities = { uploadActivityFile(formData) { return fetchPostFileRequest('activities/create/upload', formData); }, + bulkImportActivities() { + return fetchPostRequest('activities/create/bulkimport'); + }, addGearToActivity(activityId, gearId) { return fetchPutRequest(`activities/${activityId}/addgear/${gearId}`); }, @@ -41,7 +44,7 @@ export const activities = { return fetchPutRequest(`activities/${activityId}/deletegear`); }, editActivity(activityId, data) { - return fetchPutRequest(`activities/edit`, data); + return fetchPutRequest('activities/edit', data); }, deleteActivity(activityId) { return fetchDeleteRequest(`activities/${activityId}/delete`); From f418d79238845d18e45cf81356628cfe4bf278c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Vit=C3=B3ria=20Silva?= <8648976+joaovitoriasilva@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:09:20 +0100 Subject: [PATCH 2/9] Added bulk import functionality [backend] added new route for handling bulk import requests [backend] added new folder for bulk import actions [backend] adjusted code for better re usability for bulk import and single file upload [docker-compose] added new volume mapping in docker compose example file [README] added new volumes information to backend section of README file --- .gitignore | 6 + README.md | 10 +- backend/app/activities/router.py | 144 +++++++++--------- backend/app/activities/utils.py | 115 ++++++++++++++ docker-compose.yml.example | 3 +- .../Settings/SettingsIntegrationsZone.vue | 25 +++ .../settingsIntegrationsZoneComponent.json | 7 +- .../app/src/services/activitiesService.js | 7 +- 8 files changed, 237 insertions(+), 80 deletions(-) diff --git a/.gitignore b/.gitignore index 484d4bd8..1cf36035 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,12 @@ backend/app/user_images/*.jpeg backend/app/user_images/*.png backend/app/user_images/*.jpg +# gpx and fit files +backend/app/uploads/*.gpx +backend/app/uploads/*.fit +backend/app/bulk_import/*.gpx +backend/app/bulk_import/*.fit + # Frontend frontend/app/img/users_img/*.* # Logs diff --git a/README.md b/README.md index ce6481e7..21037939 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Currently the service supports: - Create/edit/delete users - Basic admin and regular user profiles that adapt the interface - Import activities using .gpx files + - Bulk import for activity files - Connect with Strava and retrieve activities and gear from Strava - Feed with user activities, current user week stats and month stats - Feed with followers activities @@ -47,7 +48,6 @@ Currently the service supports: To do features (not by order): - Support import of .fit files - - Bulk import for .gpx and .fit files - Default gear for activity type - Gear components logic for component usage tracking - Comments and likes logic for activities @@ -109,6 +109,14 @@ MYSQL_DATABASE | endurain | `No` | N/A MYSQL_USER | endurain | `No` | N/A MYSQL_PASSWORD | changeme | `No` | N/A +Table bellow shows the volumes available to be configured. Not obligatory, but recommended: + +Volume | Path | Notes +--- | --- | --- +/app | /endurain/backend/app:/app | Configure volume if you want to edit the code locally by cloning the repo and comment next line +/app/user_images | /endurain/backend/app/user_images:/app/user_images | Necessary for user image persistence on container image updates +/app/bulk_import | /endurain/backend/bulkimport | Necessary to enable bulk import of activities. Place here your activities files + To check Python backend dependencies used, use poetry file (pyproject.toml) # Strava integration diff --git a/backend/app/activities/router.py b/backend/app/activities/router.py index f4fe5884..bbc34803 100644 --- a/backend/app/activities/router.py +++ b/backend/app/activities/router.py @@ -4,7 +4,15 @@ from typing import Annotated, Callable -from fastapi import APIRouter, Depends, HTTPException, status, UploadFile, Security +from fastapi import ( + APIRouter, + Depends, + HTTPException, + status, + UploadFile, + Security, + BackgroundTasks, +) from sqlalchemy.orm import Session from datetime import datetime, timedelta, timezone @@ -20,12 +28,6 @@ import users.dependencies as users_dependencies -import activity_streams.crud as activity_streams_crud - -import gpx.utils as gpx_utils - -import fit.utils as fit_utils - import database import dependencies_global @@ -105,9 +107,7 @@ async def read_activities_useractivities_thisweek_distances( ): # Calculate the start of the current week today = datetime.now(timezone.utc) - start_of_week = today - timedelta( - days=today.weekday() - ) + start_of_week = today - timedelta(days=today.weekday()) end_of_week = start_of_week + timedelta(days=6) if user_id == token_user_id: @@ -122,8 +122,8 @@ async def read_activities_useractivities_thisweek_distances( ) # Check if activities is None - #if activities is None: - # Return None if activities is None + # if activities is None: + # Return None if activities is None # return None # Return the activities distances for this week @@ -167,8 +167,8 @@ async def read_activities_useractivities_thismonth_distances( user_id, start_of_month, end_of_month, db ) - #if activities is None: - # Return None if activities is None + # if activities is None: + # Return None if activities is None # return None # Return the activities distances for this month @@ -187,9 +187,7 @@ async def read_activities_useractivities_thismonth_number( ], token_user_id: Annotated[ Callable, - Depends( - session_security.get_user_id_from_access_token - ), + Depends(session_security.get_user_id_from_access_token), ], db: Annotated[ Session, @@ -234,9 +232,7 @@ async def read_activities_gearactivities( ], token_user_id: Annotated[ Callable, - Depends( - session_security.get_user_id_from_access_token - ), + Depends(session_security.get_user_id_from_access_token), ], db: Annotated[ Session, @@ -433,51 +429,8 @@ async def create_activity_with_uploaded_file( upload_dir = "uploads" os.makedirs(upload_dir, exist_ok=True) - # Get file extension - _, file_extension = os.path.splitext(file.filename) - - # Save the uploaded file in the 'uploads' directory - with open(file.filename, "wb") as save_file: - save_file.write(file.file.read()) - - # Choose the appropriate parser based on file extension - if file_extension.lower() == ".gpx": - # Parse the GPX file - parsed_info = gpx_utils.parse_gpx_file(file.filename, token_user_id) - elif file_extension.lower() == ".fit": - # Parse the FIT file - parsed_info = fit_utils.parse_fit_file(file.filename, token_user_id) - else: - # file extension not supported raise an HTTPException with a 406 Not Acceptable status code - raise HTTPException( - status_code=status.HTTP_406_NOT_ACCEPTABLE, - detail="File extension not supported. Supported file extensions are .gpx and .fit", - ) - - # create the activity in the database - created_activity = activities_crud.create_activity(parsed_info["activity"], db) - - # Check if created_activity is None - if created_activity is None: - # raise an HTTPException with a 500 Internal Server Error status code - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error creating activity", - ) - - # Parse the activity streams from the parsed info - activity_streams = gpx_utils.parse_activity_streams_from_gpx_file( - parsed_info, created_activity.id - ) - - # Create activity streams in the database - activity_streams_crud.create_activity_streams(activity_streams, db) - - # Remove the file after processing - os.remove(file.filename) - - # Return activity ID - return created_activity + # Return activity + return activies_utils.parse_and_store_activity_from_uploaded_file(token_user_id, file, db) except Exception as err: # Log the exception logger.error( @@ -490,8 +443,55 @@ async def create_activity_with_uploaded_file( ) from err +@router.post( + "/create/bulkimport", +) +async def create_activity_with_bulk_import( + token_user_id: Annotated[ + int, + Depends(session_security.get_user_id_from_access_token), + ], + check_scopes: Annotated[ + Callable, Security(session_security.check_scopes, scopes=["activities:write"]) + ], + db: Annotated[ + Session, + Depends(database.get_db), + ], + background_tasks: BackgroundTasks, +): + try: + # Ensure the 'bulk_import' directory exists + bulk_import_dir = "bulk_import" + os.makedirs(bulk_import_dir, exist_ok=True) + + # Iterate over each file in the 'bulk_import' directory + for filename in os.listdir(bulk_import_dir): + file_path = os.path.join(bulk_import_dir, filename) + + if os.path.isfile(file_path): + # Parse and store the activity + background_tasks.add_task( + activies_utils.parse_and_store_activity_from_file, + token_user_id, + file_path, + db, + ) + + # Return a success message + return {"Bulk import initiated. Processing files in the background."} + except Exception as err: + # Log the exception + logger.error(f"Error in create_activity_with_bulk_import: {err}", exc_info=True) + # Raise an HTTPException with a 500 Internal Server Error status code + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal Server Error", + ) from err + + @router.put( - "/edit", + "/edit", ) async def edit_activity( token_user_id: Annotated[ @@ -529,9 +529,7 @@ async def activity_add_gear( ], token_user_id: Annotated[ int, - Depends( - session_security.get_user_id_from_access_token - ), + Depends(session_security.get_user_id_from_access_token), ], db: Annotated[ Session, @@ -580,9 +578,7 @@ async def delete_activity_gear( ], token_user_id: Annotated[ int, - Depends( - session_security.get_user_id_from_access_token - ), + Depends(session_security.get_user_id_from_access_token), ], db: Annotated[ Session, @@ -621,9 +617,7 @@ async def delete_activity( ], token_user_id: Annotated[ int, - Depends( - session_security.get_user_id_from_access_token - ), + Depends(session_security.get_user_id_from_access_token), ], db: Annotated[ Session, diff --git a/backend/app/activities/utils.py b/backend/app/activities/utils.py index 4591f348..46cc79f4 100644 --- a/backend/app/activities/utils.py +++ b/backend/app/activities/utils.py @@ -3,16 +3,131 @@ import requests import math +from fastapi import HTTPException, status, UploadFile + +from typing import Union from datetime import datetime from urllib.parse import urlencode from statistics import mean +from sqlalchemy.orm import Session + import activities.schema as activities_schema +import activities.crud as activities_crud + +import activity_streams.crud as activity_streams_crud + +import gpx.utils as gpx_utils +import fit.utils as fit_utils # Define a loggger created on main.py logger = logging.getLogger("myLogger") +async def parse_and_store_activity_from_file( + token_user_id: int, file_path: str, db: Session +): + try: + # Get file extension + _, file_extension = os.path.splitext(file_path) + + # Open the file and process it + with open(file_path, "rb") as file: + # Parse the file + parsed_info = parse_file(token_user_id, file_extension, file_path) + + # Store the activity in the database + created_activity = store_activity(parsed_info, db) + + # Return the created activity + return created_activity + except Exception as err: + # Log the exception + logger.error( + f"Error in parse_and_store_activity_from_file: {err}", exc_info=True + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal Server Error", + ) from err + + +def parse_and_store_activity_from_uploaded_file( + token_user_id: int, file: UploadFile, db: Session +): + + # Get file extension + _, file_extension = os.path.splitext(file.filename) + + # Save the uploaded file in the 'uploads' directory + with open(file.filename, "wb") as save_file: + save_file.write(file.file.read()) + + # Parse the file + parsed_info = parse_file(token_user_id, file_extension, file.filename) + + # Store the activity in the database + created_activity = store_activity(parsed_info, db) + + # Return the created activity + return created_activity + + +def parse_file(token_user_id: int, file_extension: str, filename: str) -> dict: + try: + # Choose the appropriate parser based on file extension + if file_extension.lower() == ".gpx": + # Parse the GPX file + parsed_info = gpx_utils.parse_gpx_file(filename, token_user_id) + elif file_extension.lower() == ".fit": + # Parse the FIT file + parsed_info = fit_utils.parse_fit_file(filename, token_user_id) + else: + # file extension not supported raise an HTTPException with a 406 Not Acceptable status code + raise HTTPException( + status_code=status.HTTP_406_NOT_ACCEPTABLE, + detail="File extension not supported. Supported file extensions are .gpx and .fit", + ) + + # Return the parsed information + return parsed_info + except Exception as err: + # Log the exception + logger.error(f"Error in parse_file: {err}", exc_info=True) + # Raise an HTTPException with a 500 Internal Server Error status code + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal Server Error", + ) from err + finally: + # Remove the file after processing + os.remove(filename) + + +def store_activity(parsed_info: dict, db: Session): + # create the activity in the database + created_activity = activities_crud.create_activity(parsed_info["activity"], db) + + # Check if created_activity is None + if created_activity is None: + # raise an HTTPException with a 500 Internal Server Error status code + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Error creating activity", + ) + + # Parse the activity streams from the parsed info + activity_streams = gpx_utils.parse_activity_streams_from_gpx_file( + parsed_info, created_activity.id + ) + + # Create activity streams in the database + activity_streams_crud.create_activity_streams(activity_streams, db) + + # Return the created activity + return created_activity + + def calculate_activity_distances(activities: list[activities_schema.Activity]): """Calculate the distances of the activities for each type of activity (run, bike, swim)""" # Initialize the distances diff --git a/docker-compose.yml.example b/docker-compose.yml.example index 4a568992..e0afd457 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -30,8 +30,9 @@ services: ports: - "98:80" # API port, change per your needs volumes: - # - /endurain/backend/app:/app # Configure volume if you want to edit the code locally by cloning the repo and comment next line + # - /endurain/backend/app:/app # Configure volume if you want to edit the code locally by cloning the repo - /endurain/backend/user_images:/app/user_images # necessary for user image persistence on container image updates + - /endurain/backend/bulkimport:/app/bulk_import # necessary to enable bulk import of activities. Place here your activities files depends_on: - mariadb - jaeger # optional diff --git a/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue b/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue index 76cf2bbd..ba84056b 100644 --- a/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue +++ b/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue @@ -1,6 +1,7 @@ @@ -28,6 +39,7 @@ import { addToast } from '@/utils/toastUtils'; import { useAuthStore } from '@/stores/authStore'; // Importing the services import { strava } from '@/services/stravaService'; +import { activities } from '@/services/activitiesService'; export default { components: { @@ -76,12 +88,25 @@ export default { } } + async function submitBulkImport() { + try { + await activities.bulkImportActivities(); + + // Set the loading message and show the loading alert. + addToast(t('settingsIntegrationsZone.loadingMessageBulkImport'), 'loading', true); + } catch(error) { + // If there is an error, set the error message and show the error alert. + addToast(t('settingsIntegrationsZone.errorMessageUnableToImportActivities') + " - " + error, 'danger', true); + } + } + return { authStore, t, submitConnectStrava, submitRetrieveStravaLastWeekActivities, submitRetrieveStravaGear, + submitBulkImport, }; }, }; diff --git a/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json b/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json index c0bc8085..22f0d293 100644 --- a/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json +++ b/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json @@ -10,5 +10,10 @@ "errorMessageUnableToGetStravaActivities": "Unable to get Strava activities", "errorMessageUnableToGetStravaGear": "Unable to get Strava gear", "loadingMessageRetrievingStravaActivities": "Retrieving Strava activities", - "loadingMessageRetrievingStravaGear": "Retrieving Strava gear" + "loadingMessageRetrievingStravaGear": "Retrieving Strava gear", + "bulkImportIntegrationTitle": "Bulk import", + "bulkImportIntegrationBody": "Bulk import activities from files stored in the bulk_import folder", + "buttonBulkImport": "Import", + "errorMessageUnableToImportActivities": "An error occurred while importing activities", + "loadingMessageBulkImport": "Importing activities" } \ No newline at end of file diff --git a/frontend/app/src/services/activitiesService.js b/frontend/app/src/services/activitiesService.js index 5cb16262..975b0638 100644 --- a/frontend/app/src/services/activitiesService.js +++ b/frontend/app/src/services/activitiesService.js @@ -1,4 +1,4 @@ -import { fetchGetRequest, fetchPostFileRequest, fetchDeleteRequest, fetchPutRequest } from '@/utils/serviceUtils'; +import { fetchGetRequest, fetchPostFileRequest, fetchDeleteRequest, fetchPutRequest, fetchPostRequest } from '@/utils/serviceUtils'; export const activities = { getUserWeekActivities(user_id, week_number) { @@ -34,6 +34,9 @@ export const activities = { uploadActivityFile(formData) { return fetchPostFileRequest('activities/create/upload', formData); }, + bulkImportActivities() { + return fetchPostRequest('activities/create/bulkimport'); + }, addGearToActivity(activityId, gearId) { return fetchPutRequest(`activities/${activityId}/addgear/${gearId}`); }, @@ -41,7 +44,7 @@ export const activities = { return fetchPutRequest(`activities/${activityId}/deletegear`); }, editActivity(activityId, data) { - return fetchPutRequest(`activities/edit`, data); + return fetchPutRequest('activities/edit', data); }, deleteActivity(activityId) { return fetchDeleteRequest(`activities/${activityId}/delete`); From e372b75ef56ba6b7fcb23a52b9e4d481fb8ad6e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Vit=C3=B3ria=20Silva?= <8648976+joaovitoriasilva@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:31:36 +0100 Subject: [PATCH 3/9] Fixed contributing file not showing properly --- CONTRIBUTING.md => CONTRIBUTING | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CONTRIBUTING.md => CONTRIBUTING (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING similarity index 100% rename from CONTRIBUTING.md rename to CONTRIBUTING From c01f7ec76e461e69bd505886039a5e529a66c853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Vit=C3=B3ria=20Silva?= <8648976+joaovitoriasilva@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:41:38 +0100 Subject: [PATCH 4/9] Bump version to 0.4.0 --- backend/app/config.py | 2 +- frontend/app/src/components/FooterComponent.vue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/config.py b/backend/app/config.py index ed803590..62a0d13d 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -1,2 +1,2 @@ # Constant related to version -API_VERSION = "v0.3.3" \ No newline at end of file +API_VERSION = "v0.4.0" \ No newline at end of file diff --git a/frontend/app/src/components/FooterComponent.vue b/frontend/app/src/components/FooterComponent.vue index 2f41bd10..d80c2b10 100644 --- a/frontend/app/src/components/FooterComponent.vue +++ b/frontend/app/src/components/FooterComponent.vue @@ -29,7 +29,7 @@
-

© {{ new Date().getFullYear() === 2023 ? '2023' : '2023 - ' + new Date().getFullYear() }} Endurain • • • v0.3.3

+

© {{ new Date().getFullYear() === 2023 ? '2023' : '2023 - ' + new Date().getFullYear() }} Endurain • • • v0.4.0

Compatible with STRAVA image

From 8fee330c10a8cc9d58c4b9fa8f348487cd44a3f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Vit=C3=B3ria=20Silva?= <8648976+joaovitoriasilva@users.noreply.github.com> Date: Wed, 28 Aug 2024 12:49:55 +0100 Subject: [PATCH 5/9] Settings integration zone from card to list view --- .../Settings/SettingsIntegrationsZone.vue | 66 ++++++++++++------- .../settingsIntegrationsZoneComponent.json | 1 + 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue b/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue index ba84056b..9c213087 100644 --- a/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue +++ b/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue @@ -1,33 +1,55 @@ diff --git a/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json b/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json index 22f0d293..c0c85c0a 100644 --- a/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json +++ b/frontend/app/src/i18n/en/components/settings/settingsIntegrationsZoneComponent.json @@ -2,6 +2,7 @@ "stravaIntegrationTitle": "Strava", "stravaIntegrationBody": "Strava is an American internet service for tracking physical exercise which incorporates social network features.", "buttonConnect": "Connect", + "buttonDropdownOptions": "Options", "buttonStravaRetrieveLastWeekActivities": "Retrieve last week activities", "buttonStravaRetrieveGear": "Retrieve gear", "errorMessageUnableToLinkStrava": "Unable to link Strava account", From a2348da978e8469a1dcb2cdad77a8ba9af7ad3ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Vit=C3=B3ria=20Silva?= <8648976+joaovitoriasilva@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:44:01 +0100 Subject: [PATCH 6/9] User can now specify number of days to retrieve Strava activities [frontend] changed button retrieve last week activities to a modal here the user inputs the number of days to retrieve [frontend] fixed minor issues --- .../Settings/SettingsIntegrationsZone.vue | 44 +++++++++++++++---- .../SettingsUsersZone/UsersListComponent.vue | 10 ++--- .../settingsIntegrationsZoneComponent.json | 5 ++- frontend/app/src/views/HomeView.vue | 10 ++--- 4 files changed, 50 insertions(+), 19 deletions(-) diff --git a/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue b/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue index 9c213087..8f25e64e 100644 --- a/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue +++ b/frontend/app/src/components/Settings/SettingsIntegrationsZone.vue @@ -24,12 +24,37 @@ + + + @@ -54,6 +79,7 @@