Issue slovensko-digital#3:

 - first touch
 - parse order_id
 - parse order documents
 - store topic id with order id
 - create post for existed topic

Gemfile

@@ -5,11 +5,16 @@ gem 'curb'
 gem 'dotenv'
 gem 'discourse_api'
 gem 'abstract_type'
+gem 'activerecord'
+gem 'pg'
 
 group :development do
   gem 'rspec'
   gem 'rubocop'
   gem 'mutant-rspec'
+  gem 'pry'
+  gem 'pry-rails'
+  gem 'pry-byebug'
 end
 
 group :test do

Gemfile.lock

@@ -2,16 +2,35 @@ GEM
   remote: https://rubygems.org/
   specs:
     abstract_type (0.0.7)
+    activemodel (4.2.5.1)
+      activesupport (= 4.2.5.1)
+      builder (~> 3.1)
+    activerecord (4.2.5.1)
+      activemodel (= 4.2.5.1)
+      activesupport (= 4.2.5.1)
+      arel (~> 6.0)
+    activesupport (4.2.5.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
     adamantium (0.2.0)
       ice_nine (~> 0.11.0)
       memoizable (~> 0.4.0)
     anima (0.3.0)
       abstract_type (~> 0.0.7)
       adamantium (~> 0.2)
       equalizer (~> 0.0.11)
+    arel (6.0.3)
     ast (2.2.0)
+    builder (3.2.2)
+    byebug (5.0.0)
+      columnize (= 0.9.0)
     codeclimate-test-reporter (0.4.7)
       simplecov (>= 0.7.1, < 1.0.0)
+    coderay (1.1.0)
+    columnize (0.9.0)
     concord (0.1.5)
       adamantium (~> 0.2.0)
       equalizer (~> 0.0.9)
@@ -28,11 +47,14 @@ GEM
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.10.0)
       faraday (>= 0.7.4, < 0.10)
+    i18n (0.7.0)
     ice_nine (0.11.2)
     json (1.8.3)
     memoizable (0.4.2)
       thread_safe (~> 0.3, >= 0.3.1)
+    method_source (0.8.2)
     mini_portile2 (2.0.0)
+    minitest (5.8.4)
     morpher (0.2.6)
       abstract_type (~> 0.0.7)
       adamantium (~> 0.2.0)
@@ -66,8 +88,18 @@ GEM
     parallel (1.6.1)
     parser (2.3.0.2)
       ast (~> 2.2)
+    pg (0.18.3)
     powerpack (0.1.1)
     procto (0.0.2)
+    pry (0.10.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-byebug (3.2.0)
+      byebug (~> 5.0)
+      pry (~> 0.10)
+    pry-rails (0.3.4)
+      pry (>= 0.9.10)
     rack (1.6.4)
     rainbow (2.1.0)
     rake (10.4.2)
@@ -95,7 +127,10 @@ GEM
       json (~> 1.8)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
+    slop (3.6.0)
     thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
     unparser (0.2.5)
       abstract_type (~> 0.0.7)
       adamantium (~> 0.2.0)
@@ -110,12 +145,17 @@ PLATFORMS
 
 DEPENDENCIES
   abstract_type
+  activerecord
   codeclimate-test-reporter
   curb
   discourse_api
   dotenv
   mutant-rspec
   nokogiri
+  pg
+  pry
+  pry-byebug
+  pry-rails
   rake
   rspec
   rubocop

lib/uvobot.rb

@@ -5,6 +5,7 @@
 require_relative 'uvobot/notifications/discourse_notifier'
 require_relative 'uvobot/notifications/slack_notifier'
 require_relative 'uvobot/worker'
+require_relative 'uvobot/store'
 
 module Uvobot
 end

lib/uvobot/discourse_client.rb

@@ -2,14 +2,34 @@
 
 module Uvobot
   class DiscourseClient
-    def initialize(host, api_key, api_username)
+    def initialize(host: nil, api_key: nil, api_username: nil, local_store: nil)
       @client = DiscourseApi::Client.new(host, api_key, api_username)
+      @local_store = local_store
+    end
+
+    def store_topic(order_id: nil, topic: nil, category: nil)
+      if @local_store.check_topic?(order_id)
+        create_post(@local_store.get_topic_id(order_id), topic[:body])
+      else
+        response = create_topic(
+            title: topic[:title],
+            raw: topic[:body],
+            category: category
+        )
+        # TODO: get topic_id and store it
+      end
     end
 
     def create_topic(args = {})
       @client.create_topic(args)
     rescue DiscourseApi::Error
       return nil
     end
+
+    def create_post(topic_id, content)
+      @client.create_post(topic_id: topic_id, raw: content)
+    rescue DiscourseApi::Error
+      return nil
+    end
   end
 end

lib/uvobot/notifications/discourse_notifier.rb

@@ -20,33 +20,42 @@ def new_issue_not_published
       def matching_announcements_found(_page_info, announcements)
         announcements.each do |a|
           topic = announcement_to_topic(a)
-          @client.create_topic(
-            title: topic[:title],
-            raw: topic[:body],
-            category: @category
-          )
+          @client.store_topic(order_id: a[:order][:id], topic: topic, category: @category)
         end
       end
 
       private
 
       def announcement_to_topic(announcement)
-        detail = @scraper.get_announcement_detail(announcement[:link][:href])
-
-        {
-          title: announcement[:procurement_subject].to_s,
-          body: ["**Obstarávateľ:** #{announcement[:procurer]}",
-                 "**Predmet obstarávania:** #{announcement[:procurement_subject]}",
-                 detail_message(detail),
-                 "**Zdroj:** [#{announcement[:link][:text]}](#{announcement[:link][:href]})"].join("  \n")
-        }
+        details = @scraper.get_announcement_detail(announcement[:link][:href], announcement[:release_date])
+        response = {}
+        response[:title] = announcement[:procurement_subject].to_s
+        response[:body] = [
+            "**Obstarávateľ:** #{announcement[:procurer]}",
+            "**Predmet obstarávania:** #{announcement[:procurement_subject]}",
+            price_details(details),
+            order_documents(details),
+            "**Zdroj:** [#{announcement[:link][:text]}](#{announcement[:link][:href]})"
+        ].join("  \n")
       end
 
-      def detail_message(detail)
-        if detail
-          "**Cena:** #{detail[:amount]}"
+      def price_details(details)
+        if details && details[:amount]
+          "**Cena:** #{details[:amount]}"
         else
-          '**Detaily sa nepodarilo extrahovať.**'
+          '**Cena:** nepodarilo sa extrahovať'
+        end
+      end
+
+      def order_documents(details)
+        if details && details[:order] && details[:order][:documents]
+          response = [ "** Dokumenty zákazky:**" ]
+          details[:order][:documents].each do |document|
+            response << "#{document[:name]} [#{document[:href]}]"
+          end
+          response.join("  \n")
+        else
+          '** Dokumenty zákazky:** nepodarilo sa extrahovať.**'
         end
       end
     end

lib/uvobot/store.rb

@@ -0,0 +1,6 @@
+require_relative 'store/manager'
+
+module Uvobot
+  module Store
+  end
+end

lib/uvobot/store/create_topics_table.rb

@@ -0,0 +1,16 @@
+module Uvobot::Store
+  class CreateTopicsTable < ::ActiveRecord::Migration
+
+    def up
+      create_table :topics do |t|
+        t.string :order_id
+        t.string :topic_id
+      end
+      add_index :topics, :order_id
+    end
+
+    def down
+      drop_table :topics
+    end
+  end
+end

lib/uvobot/store/manager.rb

@@ -0,0 +1,34 @@
+require 'active_record'
+require_relative 'create_topics_table'
+require_relative 'topic'
+
+module Uvobot
+  module Store
+    class Manager
+      def initialize(database_url)
+        ActiveRecord::Base.establish_connection(database_url)
+        check_migration
+      end
+
+      def check_topic?(order_id)
+        Topic.where(order_id: order_id).count > 0
+      end
+
+      def get_topic_id(order_id)
+        Topic.where(order_id: order_id).first.topic_id
+      end
+
+      def check_migration
+        unless ActiveRecord::Base.connection.table_exists? 'topics'
+          run_migration
+        end
+      end
+
+      private
+
+      def run_migration
+        CreateTopicsTable.migrate(:up)
+      end
+    end
+  end
+end

lib/uvobot/store/topic.rb

@@ -0,0 +1,5 @@
+module Uvobot::Store
+  class Topic < ::ActiveRecord::Base
+
+  end
+end

lib/uvobot/uvo_parser.rb

@@ -2,25 +2,48 @@
 
 module Uvobot
   class UvoParser
-    def self.parse_announcements(html, bulletin_url)
+    def self.parse_announcements(html, bulletin_url, release_date)
       announcements = []
 
       doc(html).css('#lists-table tr[onclick]').each do |tr|
-        announcements << parse_table_line(tr, bulletin_url)
+        announcements << parse_table_line(tr, bulletin_url, release_date)
       end
       announcements
     end
 
-    def self.parse_table_line(tr_node, bulletin_url)
+    def self.parse_table_line(tr_node, bulletin_url, release_date)
       a_parts = tr_node.css('td').first.text.split("\n").map(&:strip)
 
       {
+        release_date: release_date,
         link: { text: a_parts[0], href: parse_detail_link(tr_node, bulletin_url) },
         procurer: a_parts[1],
         procurement_subject: a_parts[2]
       }
     end
 
+    def self.parse_order_documents_url(announcement_html)
+      url = doc(announcement_html).css('#procurer table tr:first td a').first['href']
+      [
+          url.split('/').last,
+          url.gsub('zdetail','zdokumenty')+ '?_profilObstaravatela_WAR_uvoprofil_sortKey=datum_dt&_profilObstaravatela_WAR_uvoprofil_sortOrder=desc'
+      ]
+    end
+
+    def self.parse_order_documents(document_html, bulletin_url, release_date)
+      documents = []
+      doc(document_html).css('.obst_search_container .reg_search:first tbody tr').each do |row|
+        td, td_date = row.css('td')
+        if release_date == Date.parse(td_date.text.strip)
+          document = {}
+          document[:name] = td.text
+          document[:href] = bulletin_url + td.css('a').first['href']
+          documents << document
+        end
+      end
+      documents
+    end
+
     def self.parse_detail_link(tr_node, bulletin_url)
       bulletin_url + tr_node.attributes['onclick'].text.scan(/'(.*)'/).first[0]
     end

lib/uvobot/uvo_scraper.rb

@@ -27,12 +27,18 @@ def get_announcements(release_date)
       search_query = { kcpv: IT_CONTRACTS_CODE, dzOd: date, dzDo: date }
       html = @html_client.post(SEARCH_URL, search_query).body
 
-      [@parser.parse_page_info(html), @parser.parse_announcements(html, BULLETIN_URL)]
+      [@parser.parse_page_info(html), @parser.parse_announcements(html, BULLETIN_URL, release_date)]
     end
 
-    def get_announcement_detail(url)
+    def get_announcement_detail(url, release_date)
       html = @html_client.get(url).body
-      @parser.parse_detail(html)
+      response = @parser.parse_detail(html)
+      order_id, order_url = @parser.parse_order_documents_url(html)
+      response[:order] = {
+          id: order_id,
+          documents: @parser.parse_order_documents(@html_client.get(order_url).body, BULLETIN_URL, release_date)
+      }
+      response
     end
   end
 end

uvobot.rb

@@ -4,9 +4,10 @@
 
 Dotenv.load
 discourse_client = Uvobot::DiscourseClient.new(
-  ENV.fetch('DISCOURSE_URL'),
-  ENV.fetch('DISCOURSE_API_KEY'),
-  ENV.fetch('DISCOURSE_USER')
+  host: ENV.fetch('DISCOURSE_URL'),
+  api_key: ENV.fetch('DISCOURSE_API_KEY'),
+  api_username: ENV.fetch('DISCOURSE_USER'),
+  local_store: Uvobot::Store::Manager.new(ENV.fetch('DATABASE_URL'))
 )
 
 notifiers = [