Merge pull request #8 from 9AMPYLO/database

Database

.gitignore

@@ -1,3 +1,4 @@
 config/secrets.yml
 coverage/*
-!coverage/.resultset.json
+!coverage/.resultset.json
+*.db

.pryrc

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'hirb-unicode'
+
+Hirb.enable
+
+old_print = Pry.config.print
+Pry.config.print = proc do |*args|
+  Hirb::View.view_or_page_output(args[1]) || old_print.call(*args)
+end

Gemfile

@@ -2,41 +2,50 @@
 
 source 'https://rubygems.org'
 
+# Configuration and Utilities
+gem 'figaro', '~> 1.2'
+gem 'rake'
+
 # Web Application
-gem 'puma', '~> 5'
+gem 'puma', '~> 6'
 gem 'roda', '~> 3'
 gem 'slim', '~> 4'
 
+# Validation
+gem 'dry-struct', '~> 1'
+gem 'dry-types', '~> 1'
+
 # Networking
 gem 'http', '~> 5'
 
-# Testing
-gem 'minitest', '~> 5'
-gem 'minitest-rg', '~> 5'
-gem 'rerun', '~> 0'
-gem 'simplecov', '~> 0'
-gem 'vcr', '~> 6'
-gem 'webmock', '~> 3'
-
-# Utility Tools
-gem 'rake'
+# Database
+gem 'hirb', '~> 0'
+gem 'hirb-unicode', '~> 0'
+gem 'sequel', '~> 5.49'
 
-# Debugging
-gem 'pry'
+group :development, :test do
+  gem 'sqlite3', '~> 1.4'
+end
 
-# Code Quality
-gem 'flog'
-gem 'reek'
-gem 'rubocop'
+# Testing
+group :test do
+  gem 'minitest', '~> 5'
+  gem 'minitest-rg', '~> 5'
+  gem 'simplecov', '~> 0'
+  gem 'vcr', '~> 6'
+  gem 'webmock', '~> 3'
+end
+
+group :development do
+  gem 'rerun', '~> 0'
+end
 
 # Debugging
 gem 'pry'
 
-# Validation
-gem 'dry-struct', '~> 1'
-gem 'dry-types', '~> 1'
-
-# Web Application
-gem 'puma', '~> 5'
-gem 'roda', '~> 3'
-gem 'slim', '~> 4'
+# Code Quality
+group :development do
+  gem 'flog'
+  gem 'reek'
+  gem 'rubocop'
+end

Gemfile.lock

@@ -37,11 +37,17 @@ GEM
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
+    figaro (1.2.0)
+      thor (>= 0.14.0, < 2)
     flog (4.6.6)
       path_expander (~> 1.0)
       ruby_parser (~> 3.1, > 3.1.0)
       sexp_processor (~> 4.8)
     hashdiff (1.0.1)
+    hirb (0.7.3)
+    hirb-unicode (0.0.5)
+      hirb (~> 0.5)
+      unicode-display_width (~> 0.1.1)
     http (5.1.0)
       addressable (~> 2.8)
       http-cookie (~> 1.0)
@@ -51,7 +57,6 @@ GEM
       domain_name (~> 0.5)
     http-form_data (2.3.0)
     ice_nine (0.11.2)
-    json (2.6.2)
     kwalify (0.7.2)
     listen (3.7.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
@@ -60,19 +65,19 @@ GEM
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
     method_source (1.0.0)
+    mini_portile2 (2.8.0)
     minitest (5.16.3)
     minitest-rg (5.2.0)
       minitest (~> 5.0)
     nio4r (2.5.8)
-    parallel (1.22.1)
     parser (3.1.2.1)
       ast (~> 2.4.1)
     path_expander (1.1.1)
     pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (5.0.0)
-    puma (5.6.5)
+    puma (6.0.0)
       nio4r (~> 2.0)
     rack (3.0.0)
     rainbow (3.1.1)
@@ -84,27 +89,16 @@ GEM
       kwalify (~> 0.7.0)
       parser (~> 3.1.0)
       rainbow (>= 2.0, < 4.0)
-    regexp_parser (2.6.0)
     rerun (0.13.1)
       listen (~> 3.0)
     rexml (3.2.5)
     roda (3.61.0)
       rack
-    rubocop (1.36.0)
-      json (~> 2.3)
-      parallel (~> 1.10)
-      parser (>= 3.1.2.1)
-      rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.20.1, < 2.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.22.0)
-      parser (>= 3.1.1.0)
-    ruby-progressbar (1.11.0)
+    rubocop (0.7.2)
+      rainbow (>= 1.1.4)
     ruby_parser (3.19.1)
       sexp_processor (~> 4.16)
+    sequel (5.62.0)
     sexp_processor (4.16.1)
     simplecov (0.21.2)
       docile (~> 1.1)
@@ -115,12 +109,17 @@ GEM
     slim (4.1.0)
       temple (>= 0.7.6, < 0.9)
       tilt (>= 2.0.6, < 2.1)
+    sqlite3 (1.5.3)
+      mini_portile2 (~> 2.8.0)
+    sqlite3 (1.5.3-x86_64-darwin)
+    sqlite3 (1.5.3-x86_64-linux)
     temple (0.8.2)
+    thor (1.2.1)
     tilt (2.0.11)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8.2)
-    unicode-display_width (2.3.0)
+    unicode-display_width (0.1.1)
     vcr (6.1.0)
     webmock (3.18.1)
       addressable (>= 2.8.0)
@@ -129,25 +128,31 @@ GEM
     zeitwerk (2.6.1)
 
 PLATFORMS
+  ruby
   x86_64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
   dry-struct (~> 1)
   dry-types (~> 1)
+  figaro (~> 1.2)
   flog
+  hirb (~> 0)
+  hirb-unicode (~> 0)
   http (~> 5)
   minitest (~> 5)
   minitest-rg (~> 5)
   pry
-  puma (~> 5)
+  puma (~> 6)
   rake
   reek
   rerun (~> 0)
   roda (~> 3)
   rubocop
+  sequel (~> 5.49)
   simplecov (~> 0)
   slim (~> 4)
+  sqlite3 (~> 1.4)
   vcr (~> 6)
   webmock (~> 3)
 

README.md

@@ -27,3 +27,7 @@ Based on ther **trends** and **keywords** to recommend the latest papers, open s
 * Notifications on LINE or Telegram daily.
 
 * Customize **keywords** or **KOLs** to get personal recommended papers.
+
+## ER Diagram
+
+![Imgur](https://imgur.com/Xy56Rfo.png)

Rakefile

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 
 require 'rake/testtask'
+require_relative 'require_app'
 
 task :default do
   puts `rake -T`
 end
 
-desc 'run tests'
-task :spec do
-  sh 'ruby spec/gateway_paperswithcode_spec.rb'
+desc 'Run tests once'
+Rake::TestTask.new(:spec) do |t|
+  t.pattern = 'spec/*_spec.rb'
+  t.warning = false
 end
 
 desc 'Keep rerunning tests upon file changes'
@@ -20,6 +22,50 @@ task :run do
   sh 'bundle exec puma'
 end
 
+namespace :db do
+  task :config do
+    require 'sequel'
+    require_relative 'config/environment'
+    require_relative 'spec/helpers/database_helper'
+
+    def app = PapMon::App
+  end
+
+  desc 'Run migrations'
+  task migrate: :config do
+    Sequel.extension :migration
+    puts "Migrating #{app.environment} database to latest"
+    Sequel::Migrator.run(app.DB, 'db/migrations')
+  end
+
+  desc 'Wipe records from all tables'
+  task wipe: :config do
+    if app.environment == :production
+      puts 'Refusing to wipe production database'
+      return
+    end
+
+    require_app('infrastructure')
+    DatabaseHelper.wipe_database
+  end
+
+  desc 'Delete dev or test database file (set correct RACK_ENV)'
+  task drop: :config do
+    if app.environment == :production
+      puts 'Refusing to drop production database'
+      return
+    end
+
+    FileUtils.rm(PapMon::App.config.DB_FILENAME)
+    puts "Deleted #{PapMon::App.config.DB_FILENAME}"
+  end
+end
+
+desc 'Run application console'
+task :console do
+  sh 'pry -r ./load_all'
+end
+
 task :rerun do
   sh "rerun -c --ignore 'coverage/*' -- bundle exec puma"
 end

app/controllers/app.rb

@@ -17,7 +17,8 @@ class App < Roda
 
       # GET /
       routing.root do
-        view 'home'
+        papers = Repository::For.klass(Entity::Paper).all
+        view 'home', locals: { papers: }
       end
 
       routing.on 'paper' do
@@ -26,14 +27,23 @@ class App < Roda
           routing.post do
             id = routing.params['id']
             routing.halt 400, 'Missing paper id' unless id
+            # routing.redirect "paper/#{id}"
+
+            # Get paper from PapersWithCode
+            paperswithcode_paper = PapersWithCode::PaperMapper.new.find(id)
+
+            # Add paper to database
+            Repository::For.entity(paperswithcode_paper).create(paperswithcode_paper)
+
+            # Redirect viewer to paper page
             routing.redirect "paper/#{id}"
           end
         end
 
         routing.on String do |id|
           # GET /paper/[id]
           routing.get do
-            paperswithcode_paper = PapersWithCode::PaperMapper.new.find(id)
+            paperswithcode_paper = Repository::For.klass(Entity::Paper).find_origin_id(id)
             view 'paper', locals: { paper: paperswithcode_paper }
           end
         end

app/infrastructure/database/dataset_orm.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module PapMon
+  module Database
+    # Paper ORM
+    class DatasetOrm < Sequel::Model(:datasets)
+      many_to_many :papers,
+                   class: :'PapMon::Database::PaperOrm',
+                   join_table: :papers_datasets,
+                   left_key: :dataset_id, right_key: :paper_id
+      plugin :timestamps, update_on_create: true
+      def self.find_or_create(dataset_info)
+        first(id: dataset_info[:id]) || create(dataset_info)
+      end
+    end
+  end
+end

app/infrastructure/database/paper_orm.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module PapMon
+  module Database
+    # Paper ORM
+    class PaperOrm < Sequel::Model(:papers)
+      one_to_many :repositories,
+                  class: :'PapMon::Database::RepositoryOrm',
+                  key: :paper_id
+      many_to_many :datasets,
+                   class: :'PapMon::Database::DatasetOrm',
+                   join_table: :papers_datasets,
+                   left_key: :paper_id, right_key: :dataset_id
+      plugin :timestamps, update_on_create: true
+    end
+  end
+end

app/infrastructure/database/repository_orm.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module PapMon
+  module Database
+    # Paper ORM
+    class RepositoryOrm < Sequel::Model(:repositories)
+      many_to_one :paper,
+                  class: :'PapMon::Database::PaperOrm'
+
+      plugin :timestamps, update_on_create: true
+      def self.find_or_create(repo_info)
+        first(id: repo_info[:id]) || create(repo_info)
+      end
+    end
+  end
+end