Added cron to scrape github topic repositories

backend/api/src/jobs/github_topics_scraper/contract.rs

@@ -0,0 +1,34 @@
+use error::Result;
+use sea_orm::prelude::Uuid;
+use store::{github_projects, github_repositories};
+
+#[async_trait]
+pub trait DbServiceContract {
+    ///
+    /// Insert new project
+    /// 
+    async fn insert_project(&self, project: github_projects::ActiveModel) -> Result<()>;
+
+    ///
+    /// Insert new repository
+    /// 
+    async fn insert_repository(&self, repository: github_repositories::ActiveModel) -> Result<()>;
+
+    ///
+    /// Run update status of github topic repositories
+    /// 
+    async fn update_projects(&self) -> Result<()>;
+}
+
+#[async_trait]
+pub trait DbRepositoryContract {
+    ///
+    /// Get all projects from github topic repositories that are not scraped
+    /// 
+    async fn get_unscraped_projects(&self) -> Result<Vec<String>>;
+
+    ///
+    /// Get unscraped github topic repositories, return (Project Name, Repository Name, Project ID)
+    /// 
+    async fn get_unscraped_repositories(&self) -> Result<Vec<(String, String, Uuid)>>;
+}

backend/api/src/jobs/github_topics_scraper/domain.rs

@@ -0,0 +1,160 @@
+use super::contract::{DbRepositoryContract, DbServiceContract};
+use cronus::{Job, Schedule};
+use error::Result;
+use sdks::github::{
+    data::{GithubRepository, ProfileInfo},
+    GithubContract,
+};
+use sea_orm::{prelude::Uuid, ActiveValue::NotSet, Set};
+use store::{github_projects, github_repositories};
+
+pub struct GithubTopicsScraper<
+    Repository: DbRepositoryContract,
+    Service: DbServiceContract,
+    Github: GithubContract,
+> {
+    repository: Repository,
+    service: Service,
+    github: Github,
+}
+
+impl<
+        Repository: DbRepositoryContract + Send + Sync + 'static,
+        Service: DbServiceContract + Send + Sync + 'static,
+        Github: GithubContract + Send + Sync + 'static,
+    > GithubTopicsScraper<Repository, Service, Github>
+{
+    ///
+    /// Creates GithubTopicsScraper
+    ///
+    pub fn new(repository: Repository, service: Service, github: Github) -> Self {
+        Self {
+            repository,
+            service,
+            github,
+        }
+    }
+
+    ///
+    /// Cron job that runs once a week
+    ///
+    async fn cron_job(&self) -> Result<()> {
+        self.handle_projects().await?;
+        self.handle_repositories().await?;
+        self.service.update_projects().await?;
+        Ok(())
+    }
+
+    ///
+    /// Download unscraped repositories
+    /// 
+    async fn handle_repositories(&self) -> Result<()> {
+        let repositories = self.repository.get_unscraped_repositories().await?;
+
+        for (project_name, repository_name, project_id) in repositories {
+            if let Err(error) = self
+                .handle_repository(project_name, repository_name, project_id)
+                .await
+            {
+                error!("{}", error);
+            }
+        }
+
+        Ok(())
+    }
+
+    ///
+    /// Download repository and store it in db
+    /// 
+    async fn handle_repository(
+        &self,
+        project_name: String,
+        repository_name: String,
+        project_id: Uuid,
+    ) -> Result<()> {
+        let GithubRepository {
+            name,
+            language,
+            stargazers_count,
+            forks_count,
+            created_at,
+            updated_at,
+            archived,
+            fork,
+        } = self
+            .github
+            .get_repository(&project_name, &repository_name)
+            .await?;
+
+        let repository = github_repositories::ActiveModel {
+            id: NotSet,
+            project: Set(project_id),
+            repository_name: Set(name),
+            language: Set(language),
+            stargazers_count: Set(stargazers_count),
+            forks_count: Set(forks_count),
+            created_at: Set(created_at),
+            updated_at: Set(updated_at),
+            archived: Set(archived),
+            fork: Set(fork),
+        };
+
+        self.service.insert_repository(repository).await?;
+        Ok(())
+    }
+
+    ///
+    /// Download unscraped projects
+    /// 
+    async fn handle_projects(&self) -> Result<()> {
+        let projects = self.repository.get_unscraped_projects().await?;
+        for project in projects {
+            if let Err(error) = self.handle_project(project).await {
+                error!("{}", error);
+            }
+        }
+
+        Ok(())
+    }
+
+    ///
+    /// Download project and store it in db
+    /// 
+    async fn handle_project(&self, project: String) -> Result<()> {
+        let ProfileInfo {
+            profile_type,
+            followers,
+            site,
+        } = self.github.get_profile(&project).await?;
+
+        let project = github_projects::ActiveModel {
+            id: NotSet,
+            name: Set(project),
+            profile_type: Set(Some(profile_type.to_string())),
+            url: Set(match site.is_empty() {
+                true => None,
+                false => Some(site),
+            }),
+            followers: Set(followers),
+        };
+
+        self.service.insert_project(project).await
+    }
+}
+
+#[async_trait]
+impl<Repository, Service, Github> Job for GithubTopicsScraper<Repository, Service, Github>
+where
+    Repository: DbRepositoryContract + Send + Sync + 'static,
+    Service: DbServiceContract + Send + Sync + 'static,
+    Github: GithubContract + Send + Sync + 'static,
+{
+    fn schedule(&self) -> Schedule {
+        "0 0 0 * * Mon".parse().expect("Invalid schedule")
+    }
+    async fn job(&self) {
+        if let Err(error) = self.cron_job().await {
+            error!("{}", error);
+        }
+    }
+}

backend/api/src/jobs/github_topics_scraper/infrastructure/mod.rs

@@ -0,0 +1,5 @@
+mod service;
+mod repository;
+
+pub use service::PgService;
+pub use repository::PgRepository;

backend/api/src/jobs/github_topics_scraper/infrastructure/repository.rs

@@ -0,0 +1,68 @@
+use error::Result;
+use sea_orm::{
+    prelude::Uuid, DatabaseBackend, DatabaseConnection, EntityTrait, FromQueryResult, Statement,
+};
+use std::sync::Arc;
+
+use super::super::contract::DbRepositoryContract;
+use store::topics_repositories;
+
+const UNSCRAPED_PROJECTS_QUERY: &str = include_str!("sql/unscraped_projects.sql");
+const UNSCRAPED_REPOSITORIES_QUERY: &str = include_str!("sql/unscraped_repositories.sql");
+
+pub struct PgRepository {
+    conn: Arc<DatabaseConnection>,
+}
+
+impl PgRepository {
+    pub fn new(conn: Arc<DatabaseConnection>) -> Self {
+        Self { conn }
+    }
+}
+
+#[async_trait]
+impl DbRepositoryContract for PgRepository {
+    async fn get_unscraped_projects(&self) -> Result<Vec<String>> {
+        let projects = topics_repositories::Entity::find()
+            .from_raw_sql(Statement::from_string(
+                DatabaseBackend::Postgres,
+                UNSCRAPED_PROJECTS_QUERY,
+            ))
+            .into_model::<RepositoryOwner>()
+            .all(self.conn.as_ref())
+            .await?
+            .into_iter()
+            .map(|repo| repo.project_name)
+            .collect();
+
+        Ok(projects)
+    }
+
+    async fn get_unscraped_repositories(&self) -> Result<Vec<(String, String, Uuid)>> {
+        let repositories = topics_repositories::Entity::find()
+            .from_raw_sql(Statement::from_string(
+                DatabaseBackend::Postgres,
+                UNSCRAPED_REPOSITORIES_QUERY,
+            ))
+            .into_model::<Repository>()
+            .all(self.conn.as_ref())
+            .await?
+            .into_iter()
+            .map(|repo| (repo.project_name, repo.repository_name, repo.repository_owner))
+            .collect();
+
+        Ok(repositories)
+    }
+}
+
+#[derive(FromQueryResult)]
+pub struct RepositoryOwner {
+    project_name: String,
+}
+
+#[derive(FromQueryResult)]
+pub struct Repository {
+    project_name: String,
+    repository_name: String,
+    repository_owner: Uuid,
+}

backend/api/src/jobs/github_topics_scraper/infrastructure/service.rs

@@ -0,0 +1,48 @@
+use error::Result;
+use sea_orm::{DatabaseConnection, EntityTrait, Statement, ConnectionTrait};
+use std::sync::Arc;
+
+use super::super::contract::DbServiceContract;
+use store::{github_projects, github_repositories};
+
+const UPDATE_STATEMENT: &str = include_str!("sql/update_statement.sql");
+
+pub struct PgService {
+    conn: Arc<DatabaseConnection>,
+}
+
+impl PgService {
+    pub fn new(conn: Arc<DatabaseConnection>) -> Self {
+        Self { conn }
+    }
+}
+
+#[async_trait]
+impl DbServiceContract for PgService {
+    async fn insert_project(&self, project: github_projects::ActiveModel) -> Result<()> {
+        github_projects::Entity::insert(project)
+            .exec(self.conn.as_ref())
+            .await?;
+
+        Ok(())
+    }
+
+    async fn insert_repository(&self, repository: github_repositories::ActiveModel) -> Result<()> {
+        github_repositories::Entity::insert(repository)
+            .exec(self.conn.as_ref())
+            .await?;
+
+        Ok(())
+    }
+
+    async fn update_projects(&self) -> Result<()> {
+        self.conn
+            .as_ref()
+            .execute(Statement::from_string(
+                sea_orm::DatabaseBackend::Postgres,
+                UPDATE_STATEMENT,
+            ))
+            .await?;
+        Ok(())
+    }
+}

backend/api/src/jobs/github_topics_scraper/infrastructure/sql/unscraped_projects.sql

@@ -0,0 +1,5 @@
+select distinct tr.repository_owner as project_name
+from topics_repositories tr
+    full outer join github_projects gp on gp."name" = tr.repository_owner
+where tr.scraped = false
+    and gp.id isnull

backend/api/src/jobs/github_topics_scraper/infrastructure/sql/unscraped_repositories.sql

@@ -0,0 +1,6 @@
+select tr.repository_owner as project_name,
+    tr.repository_name as repository_name,
+    gp.id as repository_owner
+from topics_repositories tr
+    inner join github_projects gp on gp."name" = tr.repository_owner
+where tr.scraped = false

backend/api/src/jobs/github_topics_scraper/infrastructure/sql/update_statement.sql

@@ -0,0 +1,11 @@
+update topics_repositories
+set scraped = true
+from (
+        select tr.id
+        from topics_repositories tr
+            inner join github_projects gp on gp."name" = tr.repository_owner
+            inner join github_repositories gr on gr.repository_name = tr.repository_name
+            and gr.project = gp.id
+        where not tr.scraped
+    ) as subquery
+where topics_repositories.id = subquery.id

backend/api/src/jobs/github_topics_scraper/mod.rs

@@ -0,0 +1,34 @@
+use std::sync::Arc;
+
+use cronus::Cronus;
+use sdks::github::Github;
+use sea_orm::DatabaseConnection;
+
+use self::{
+    domain::GithubTopicsScraper,
+    infrastructure::{PgRepository, PgService},
+};
+
+mod contract;
+mod domain;
+mod infrastructure;
+
+pub fn setup(cron: &Cronus, sea_pool: Arc<DatabaseConnection>) {
+    let job = create_gr(sea_pool);
+    cron.add(job).expect("Error adding job");
+}
+
+fn create_gr(
+    sea_pool: Arc<DatabaseConnection>,
+) -> GithubTopicsScraper<PgRepository, PgService, Github> {
+    let repository = PgRepository::new(sea_pool.clone());
+    let service = PgService::new(sea_pool);
+    let github_api_key = config::get("GITHUB_KEY").ok();
+
+    let github = match github_api_key {
+        Some(api_key) => Github::new_with_auth(api_key),
+        None => Github::default(),
+    };
+
+    GithubTopicsScraper::new(repository, service, github)
+}

backend/api/src/jobs/mod.rs

@@ -8,6 +8,7 @@ pub mod info;
 mod github_issues;
 mod github_project_info;
 mod github_topics;
+mod github_topics_scraper;
 
 pub fn setup(sea_pool: Arc<DatabaseConnection>) -> Cronus {
     let cron = Cronus::new();
@@ -16,6 +17,7 @@ pub fn setup(sea_pool: Arc<DatabaseConnection>) -> Cronus {
     github_repositories::setup(&cron, sea_pool.clone());
     github_project_info::setup(&cron, sea_pool.clone());
     github_topics::setup(&cron, sea_pool.clone());
+    github_topics_scraper::setup(&cron, sea_pool.clone());
     github_issues::setup(&cron, sea_pool);
 
     cron