Merge pull request #6694 from hotosm/fastapi-refactor

* Team membership and permission fixed for projects. * Author name in recommended projects. * Organisation patch fixed. * Jobs removed from application and separately containerized

backend/api/organisations/resources.py

@@ -12,7 +12,6 @@
 )
 from backend.models.dtos.stats_dto import OrganizationStatsDTO
 from backend.models.dtos.user_dto import AuthUserDTO
-from backend.models.postgis.statuses import OrganisationType
 from backend.models.postgis.user import User
 from backend.services.organisation_service import (
     OrganisationService,
@@ -384,7 +383,7 @@ async def update_organisation(
         user = await User.get_by_id(user.id, db)
         if user.role != 1:
             org = await OrganisationService.get_organisation_by_id(organisation_id, db)
-            organisation_dto.type = OrganisationType(org.type).name
+            organisation_dto.type = org.type
             organisation_dto.subscription_tier = org.subscription_tier
     except Exception as e:
         logger.error(f"error validating request: {str(e)}")

backend/cron_jobs.py

@@ -0,0 +1,157 @@
+import asyncio
+import datetime
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.triggers.interval import IntervalTrigger
+from apscheduler.triggers.cron import CronTrigger
+from loguru import logger
+
+from backend.db import db_connection
+from backend.models.postgis.task import Task
+
+
+async def auto_unlock_tasks():
+    async with db_connection.database.connection() as conn:
+        two_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=120)
+        projects_query = """
+        SELECT DISTINCT project_id
+        FROM task_history
+        WHERE action_date > :two_hours_ago
+        """
+        projects = await conn.fetch_all(
+            query=projects_query, values={"two_hours_ago": two_hours_ago}
+        )
+        for project in projects:
+            project_id = project["project_id"]
+            logger.info(f"Processing project_id: {project_id}")
+            await Task.auto_unlock_tasks(project_id, conn)
+
+
+async def update_all_project_stats():
+    async with db_connection.database.connection() as conn:
+        logger.info("Started updating project stats.")
+        await conn.execute("UPDATE users SET projects_mapped = NULL;")
+        projects_query = "SELECT DISTINCT id FROM projects;"
+        projects = await conn.fetch_all(query=projects_query)
+        for project in projects:
+            project_id = project["id"]
+            logger.info(f"Processing project ID: {project_id}")
+            await conn.execute(
+                """
+                UPDATE projects
+                SET total_tasks = (SELECT COUNT(*) FROM tasks WHERE project_id = :project_id),
+                    tasks_mapped = (SELECT COUNT(*) FROM tasks WHERE project_id = :project_id AND task_status = 2),
+                    tasks_validated = (SELECT COUNT(*) FROM tasks WHERE project_id = :project_id AND task_status = 4),
+                    tasks_bad_imagery = (SELECT COUNT(*) FROM tasks WHERE project_id = :project_id AND task_status = 6)
+                WHERE id = :project_id;
+                """,
+                {"project_id": project_id},
+            )
+            await conn.execute(
+                """
+                UPDATE users
+                SET projects_mapped = array_append(projects_mapped, :project_id)
+                WHERE id IN (
+                    SELECT DISTINCT user_id
+                    FROM task_history
+                    WHERE action = 'STATE_CHANGE' AND project_id = :project_id
+                );
+                """,
+                {"project_id": project_id},
+            )
+        logger.info("Finished updating project stats.")
+
+
+async def update_recent_updated_project_stats():
+    async with db_connection.database.connection() as conn:
+        logger.info("Started updating recently updated projects' project stats.")
+        one_week_ago = datetime.datetime.utcnow() - datetime.timedelta(days=7)
+        projects_query = """
+        SELECT DISTINCT id
+        FROM projects
+        WHERE last_updated > :one_week_ago;
+        """
+        projects = await conn.fetch_all(
+            query=projects_query, values={"one_week_ago": one_week_ago}
+        )
+        for project in projects:
+            project_id = project["id"]
+            logger.info(f"Processing project ID: {project_id}")
+            await conn.execute(
+                """
+                UPDATE projects
+                SET total_tasks = (SELECT COUNT(*) FROM tasks WHERE project_id = :project_id),
+                    tasks_mapped = (SELECT COUNT(*) FROM tasks WHERE project_id = :project_id AND task_status = 2),
+                    tasks_validated = (SELECT COUNT(*) FROM tasks WHERE project_id = :project_id AND task_status = 4),
+                    tasks_bad_imagery = (SELECT COUNT(*) FROM tasks WHERE project_id = :project_id AND task_status = 6)
+                WHERE id = :project_id;
+                """,
+                {"project_id": project_id},
+            )
+            await conn.execute(
+                """
+                UPDATE users
+                SET projects_mapped =
+                    CASE
+                        WHEN :project_id = ANY(projects_mapped) THEN projects_mapped
+                        ELSE array_append(projects_mapped, :project_id)
+                    END
+                WHERE id IN (
+                    SELECT DISTINCT user_id
+                    FROM task_history
+                    WHERE action = 'STATE_CHANGE' AND project_id = :project_id
+                );
+                """,
+                {"project_id": project_id},
+            )
+        logger.info("Finished updating project stats.")
+
+
+async def setup_cron_jobs():
+    scheduler = AsyncIOScheduler()
+    scheduler.add_job(
+        auto_unlock_tasks,
+        IntervalTrigger(minutes=120),
+        id="auto_unlock_tasks",
+        replace_existing=True,
+    )
+    scheduler.add_job(
+        update_all_project_stats,
+        CronTrigger(hour=0, minute=0),
+        id="update_project_stats",
+        replace_existing=True,
+    )
+    scheduler.add_job(
+        update_recent_updated_project_stats,
+        CronTrigger(minute=0),
+        id="update_recent_updated_project_stats",
+        replace_existing=True,
+    )
+    scheduler.start()
+    logger.info("Scheduler initialized and jobs scheduled.")
+    logger.info(f"Scheduled jobs: {scheduler.get_jobs()}")
+
+
+async def main():
+    try:
+        # Initialize the connection pool
+        logger.info("Connecting to the database...")
+        await db_connection.database.connect()
+        logger.info("Database connection established.")
+
+        await setup_cron_jobs()
+
+        # Keeping the process alive.
+        while True:
+            await asyncio.sleep(3600)
+    except (KeyboardInterrupt, SystemExit):
+        logger.info("Shutting down...")
+    finally:
+        # Close the connection pool
+        logger.info("Disconnecting from the database...")
+        await db_connection.database.disconnect()
+        logger.info("Database connection closed.")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

backend/main.py

@@ -10,7 +10,6 @@
 from starlette.middleware.authentication import AuthenticationMiddleware
 
 from backend.config import settings
-from backend.cron import setup_cron_jobs
 from backend.db import db_connection
 from backend.routes import add_api_end_points
 from backend.services.users.authentication_service import TokenAuthBackend
@@ -40,9 +39,6 @@ async def lifespan(app):
         docs_url="/api/docs",
     )
 
-    # Set custom logger
-    # _app.logger = get_logger()
-
     # Custom exception handler for 401 errors
     @_app.exception_handler(HTTPException)
     async def custom_http_exception_handler(request: Request, exc: HTTPException):
@@ -88,7 +84,6 @@ async def pyinstrument_middleware(request, call_next):
     _app.add_middleware(
         AuthenticationMiddleware, backend=TokenAuthBackend(), on_error=None
     )
-    setup_cron_jobs()
     add_api_end_points(_app)
     return _app
 

backend/services/project_search_service.py

@@ -607,7 +607,6 @@ async def _filter_projects(
                     "project_manager_role": TeamRoles.PROJECT_MANAGER.value,
                 },
             )
-
             # Combine and flatten the project IDs from both queries
             project_ids = tuple(
                 set(

backend/services/team_service.py

@@ -8,7 +8,6 @@
 from backend.models.dtos.message_dto import MessageDTO
 from backend.models.dtos.stats_dto import Pagination
 from backend.models.dtos.team_dto import (
-    ListTeamsDTO,
     NewTeamDTO,
     ProjectTeamDTO,
     TeamDetailsDTO,
@@ -496,7 +495,7 @@ async def get_project_teams_as_dto(project_id: int, db: Database) -> TeamsListDT
             query=query, values={"project_id": project_id}
         )
         # Initialize the DTO
-        teams_list_dto = ListTeamsDTO()
+        teams_list_dto = TeamsListDTO()
 
         # Populate the DTO with team data
         for project_team in project_teams:
@@ -773,7 +772,9 @@ async def check_team_membership(
         """Given a project and permitted team roles, check user's membership in the team list"""
         teams_dto = await TeamService.get_project_teams_as_dto(project_id, db)
         teams_allowed = [
-            team_dto for team_dto in teams_dto.teams if team_dto.role in allowed_roles
+            team_dto
+            for team_dto in teams_dto.teams
+            if int(team_dto.role) in allowed_roles
         ]
         user_membership = [
             team_dto.team_id

backend/services/users/user_service.py

@@ -704,9 +704,12 @@ async def get_recommended_projects(
             SELECT DISTINCT
                 p.*,
                 o.name AS organisation_name,
-                o.logo AS organisation_logo
+                o.logo AS organisation_logo,
+                u.name AS author_name,
+                u.username AS author_username
             FROM projects p
             LEFT JOIN organisations o ON p.organisation_id = o.id
+            LEFT JOIN users u ON u.id = p.author_id
             JOIN campaign_projects cp ON p.id = cp.project_id
             JOIN campaigns c ON cp.campaign_id = c.id
             WHERE c.name = ANY(:campaign_tags)
@@ -721,14 +724,14 @@ async def get_recommended_projects(
                 "limit": limit,
             },
         )
-
         # Get only projects matching the user's mapping level if needed
         len_projs = len(recommended_projects)
         if len_projs < limit:
             remaining_projects_query = """
-                SELECT DISTINCT p.*, o.name AS organisation_name, o.logo AS organisation_logo
+                SELECT DISTINCT p.*, o.name AS organisation_name, o.logo AS organisation_logo, u.name AS author_name,u.username AS author_username
                 FROM projects p
                 LEFT JOIN organisations o ON p.organisation_id = o.id
+                LEFT JOIN users u ON u.id = p.author_id
                 WHERE difficulty = :mapping_level
                 LIMIT :remaining_limit
             """