From ea77651df9a45fb7eca9863a5a985da436a7b53b Mon Sep 17 00:00:00 2001 From: Oliver Kriska Date: Sun, 7 Feb 2016 19:47:43 +0100 Subject: [PATCH] Issue #3: - first touch - parse order_id - parse order documents - store topic id with order id - create post for existed topic --- Gemfile | 5 +++ Gemfile.lock | 40 +++++++++++++++++ lib/uvobot.rb | 1 + lib/uvobot/discourse_client.rb | 22 ++++++++- .../notifications/discourse_notifier.rb | 45 +++++++++++-------- lib/uvobot/store.rb | 6 +++ lib/uvobot/store/create_topics_table.rb | 16 +++++++ lib/uvobot/store/manager.rb | 34 ++++++++++++++ lib/uvobot/store/topic.rb | 5 +++ lib/uvobot/uvo_parser.rb | 29 ++++++++++-- lib/uvobot/uvo_scraper.rb | 12 +++-- uvobot.rb | 7 +-- 12 files changed, 194 insertions(+), 28 deletions(-) create mode 100644 lib/uvobot/store.rb create mode 100644 lib/uvobot/store/create_topics_table.rb create mode 100644 lib/uvobot/store/manager.rb create mode 100644 lib/uvobot/store/topic.rb diff --git a/Gemfile b/Gemfile index 968ba4c..35fbd16 100644 --- a/Gemfile +++ b/Gemfile @@ -5,11 +5,16 @@ gem 'curb' gem 'dotenv' gem 'discourse_api' gem 'abstract_type' +gem 'activerecord' +gem 'pg' group :development do gem 'rspec' gem 'rubocop' gem 'mutant-rspec' + gem 'pry' + gem 'pry-rails' + gem 'pry-byebug' end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index a7792b5..2033798 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,6 +2,19 @@ GEM remote: https://rubygems.org/ specs: abstract_type (0.0.7) + activemodel (4.2.5.1) + activesupport (= 4.2.5.1) + builder (~> 3.1) + activerecord (4.2.5.1) + activemodel (= 4.2.5.1) + activesupport (= 4.2.5.1) + arel (~> 6.0) + activesupport (4.2.5.1) + i18n (~> 0.7) + json (~> 1.7, >= 1.7.7) + minitest (~> 5.1) + thread_safe (~> 0.3, >= 0.3.4) + tzinfo (~> 1.1) adamantium (0.2.0) ice_nine (~> 0.11.0) memoizable (~> 0.4.0) @@ -9,9 +22,15 @@ GEM abstract_type (~> 0.0.7) adamantium (~> 0.2) equalizer (~> 0.0.11) + arel (6.0.3) ast (2.2.0) + builder (3.2.2) + byebug (5.0.0) + columnize (= 0.9.0) codeclimate-test-reporter (0.4.7) simplecov (>= 0.7.1, < 1.0.0) + coderay (1.1.0) + columnize (0.9.0) concord (0.1.5) adamantium (~> 0.2.0) equalizer (~> 0.0.9) @@ -28,11 +47,14 @@ GEM multipart-post (>= 1.2, < 3) faraday_middleware (0.10.0) faraday (>= 0.7.4, < 0.10) + i18n (0.7.0) ice_nine (0.11.2) json (1.8.3) memoizable (0.4.2) thread_safe (~> 0.3, >= 0.3.1) + method_source (0.8.2) mini_portile2 (2.0.0) + minitest (5.8.4) morpher (0.2.6) abstract_type (~> 0.0.7) adamantium (~> 0.2.0) @@ -66,8 +88,18 @@ GEM parallel (1.6.1) parser (2.3.0.2) ast (~> 2.2) + pg (0.18.3) powerpack (0.1.1) procto (0.0.2) + pry (0.10.3) + coderay (~> 1.1.0) + method_source (~> 0.8.1) + slop (~> 3.4) + pry-byebug (3.2.0) + byebug (~> 5.0) + pry (~> 0.10) + pry-rails (0.3.4) + pry (>= 0.9.10) rack (1.6.4) rainbow (2.1.0) rake (10.4.2) @@ -95,7 +127,10 @@ GEM json (~> 1.8) simplecov-html (~> 0.10.0) simplecov-html (0.10.0) + slop (3.6.0) thread_safe (0.3.5) + tzinfo (1.2.2) + thread_safe (~> 0.1) unparser (0.2.5) abstract_type (~> 0.0.7) adamantium (~> 0.2.0) @@ -110,12 +145,17 @@ PLATFORMS DEPENDENCIES abstract_type + activerecord codeclimate-test-reporter curb discourse_api dotenv mutant-rspec nokogiri + pg + pry + pry-byebug + pry-rails rake rspec rubocop diff --git a/lib/uvobot.rb b/lib/uvobot.rb index ef8731a..1e6d734 100644 --- a/lib/uvobot.rb +++ b/lib/uvobot.rb @@ -5,6 +5,7 @@ require_relative 'uvobot/notifications/discourse_notifier' require_relative 'uvobot/notifications/slack_notifier' require_relative 'uvobot/worker' +require_relative 'uvobot/store' module Uvobot end diff --git a/lib/uvobot/discourse_client.rb b/lib/uvobot/discourse_client.rb index ae59dcc..1d1f925 100644 --- a/lib/uvobot/discourse_client.rb +++ b/lib/uvobot/discourse_client.rb @@ -2,8 +2,22 @@ module Uvobot class DiscourseClient - def initialize(host, api_key, api_username) + def initialize(host: nil, api_key: nil, api_username: nil, local_store: nil) @client = DiscourseApi::Client.new(host, api_key, api_username) + @local_store = local_store + end + + def store_topic(order_id: nil, topic: nil, category: nil) + if @local_store.check_topic?(order_id) + create_post(@local_store.get_topic_id(order_id), topic[:body]) + else + response = create_topic( + title: topic[:title], + raw: topic[:body], + category: category + ) + # TODO: get topic_id and store it + end end def create_topic(args = {}) @@ -11,5 +25,11 @@ def create_topic(args = {}) rescue DiscourseApi::Error return nil end + + def create_post(topic_id, content) + @client.create_post(topic_id: topic_id, raw: content) + rescue DiscourseApi::Error + return nil + end end end diff --git a/lib/uvobot/notifications/discourse_notifier.rb b/lib/uvobot/notifications/discourse_notifier.rb index 1b7427e..5b761c8 100644 --- a/lib/uvobot/notifications/discourse_notifier.rb +++ b/lib/uvobot/notifications/discourse_notifier.rb @@ -20,33 +20,42 @@ def new_issue_not_published def matching_announcements_found(_page_info, announcements) announcements.each do |a| topic = announcement_to_topic(a) - @client.create_topic( - title: topic[:title], - raw: topic[:body], - category: @category - ) + @client.store_topic(order_id: a[:order][:id], topic: topic, category: @category) end end private def announcement_to_topic(announcement) - detail = @scraper.get_announcement_detail(announcement[:link][:href]) - - { - title: announcement[:procurement_subject].to_s, - body: ["**Obstarávateľ:** #{announcement[:procurer]}", - "**Predmet obstarávania:** #{announcement[:procurement_subject]}", - detail_message(detail), - "**Zdroj:** [#{announcement[:link][:text]}](#{announcement[:link][:href]})"].join(" \n") - } + details = @scraper.get_announcement_detail(announcement[:link][:href], announcement[:release_date]) + response = {} + response[:title] = announcement[:procurement_subject].to_s + response[:body] = [ + "**Obstarávateľ:** #{announcement[:procurer]}", + "**Predmet obstarávania:** #{announcement[:procurement_subject]}", + price_details(details), + order_documents(details), + "**Zdroj:** [#{announcement[:link][:text]}](#{announcement[:link][:href]})" + ].join(" \n") end - def detail_message(detail) - if detail - "**Cena:** #{detail[:amount]}" + def price_details(details) + if details && details[:amount] + "**Cena:** #{details[:amount]}" else - '**Detaily sa nepodarilo extrahovať.**' + '**Cena:** nepodarilo sa extrahovať' + end + end + + def order_documents(details) + if details && details[:order] && details[:order][:documents] + response = [ "** Dokumenty zákazky:**" ] + details[:order][:documents].each do |document| + response << "#{document[:name]} [#{document[:href]}]" + end + response.join(" \n") + else + '** Dokumenty zákazky:** nepodarilo sa extrahovať.**' end end end diff --git a/lib/uvobot/store.rb b/lib/uvobot/store.rb new file mode 100644 index 0000000..6e170ab --- /dev/null +++ b/lib/uvobot/store.rb @@ -0,0 +1,6 @@ +require_relative 'store/manager' + +module Uvobot + module Store + end +end \ No newline at end of file diff --git a/lib/uvobot/store/create_topics_table.rb b/lib/uvobot/store/create_topics_table.rb new file mode 100644 index 0000000..32941b3 --- /dev/null +++ b/lib/uvobot/store/create_topics_table.rb @@ -0,0 +1,16 @@ +module Uvobot::Store + class CreateTopicsTable < ::ActiveRecord::Migration + + def up + create_table :topics do |t| + t.string :order_id + t.string :topic_id + end + add_index :topics, :order_id + end + + def down + drop_table :topics + end + end +end \ No newline at end of file diff --git a/lib/uvobot/store/manager.rb b/lib/uvobot/store/manager.rb new file mode 100644 index 0000000..ae24651 --- /dev/null +++ b/lib/uvobot/store/manager.rb @@ -0,0 +1,34 @@ +require 'active_record' +require_relative 'create_topics_table' +require_relative 'topic' + +module Uvobot + module Store + class Manager + def initialize(database_url) + ActiveRecord::Base.establish_connection(database_url) + check_migration + end + + def check_topic?(order_id) + Topic.where(order_id: order_id).count > 0 + end + + def get_topic_id(order_id) + Topic.where(order_id: order_id).first.topic_id + end + + def check_migration + unless ActiveRecord::Base.connection.table_exists? 'topics' + run_migration + end + end + + private + + def run_migration + CreateTopicsTable.migrate(:up) + end + end + end +end \ No newline at end of file diff --git a/lib/uvobot/store/topic.rb b/lib/uvobot/store/topic.rb new file mode 100644 index 0000000..e1ad5d5 --- /dev/null +++ b/lib/uvobot/store/topic.rb @@ -0,0 +1,5 @@ +module Uvobot::Store + class Topic < ::ActiveRecord::Base + + end +end \ No newline at end of file diff --git a/lib/uvobot/uvo_parser.rb b/lib/uvobot/uvo_parser.rb index 434cc8f..db37ec8 100644 --- a/lib/uvobot/uvo_parser.rb +++ b/lib/uvobot/uvo_parser.rb @@ -2,25 +2,48 @@ module Uvobot class UvoParser - def self.parse_announcements(html, bulletin_url) + def self.parse_announcements(html, bulletin_url, release_date) announcements = [] doc(html).css('#lists-table tr[onclick]').each do |tr| - announcements << parse_table_line(tr, bulletin_url) + announcements << parse_table_line(tr, bulletin_url, release_date) end announcements end - def self.parse_table_line(tr_node, bulletin_url) + def self.parse_table_line(tr_node, bulletin_url, release_date) a_parts = tr_node.css('td').first.text.split("\n").map(&:strip) { + release_date: release_date, link: { text: a_parts[0], href: parse_detail_link(tr_node, bulletin_url) }, procurer: a_parts[1], procurement_subject: a_parts[2] } end + def self.parse_order_documents_url(announcement_html) + url = doc(announcement_html).css('#procurer table tr:first td a').first['href'] + [ + url.split('/').last, + url.gsub('zdetail','zdokumenty')+ '?_profilObstaravatela_WAR_uvoprofil_sortKey=datum_dt&_profilObstaravatela_WAR_uvoprofil_sortOrder=desc' + ] + end + + def self.parse_order_documents(document_html, bulletin_url, release_date) + documents = [] + doc(document_html).css('.obst_search_container .reg_search:first tbody tr').each do |row| + td, td_date = row.css('td') + if release_date == Date.parse(td_date.text.strip) + document = {} + document[:name] = td.text + document[:href] = bulletin_url + td.css('a').first['href'] + documents << document + end + end + documents + end + def self.parse_detail_link(tr_node, bulletin_url) bulletin_url + tr_node.attributes['onclick'].text.scan(/'(.*)'/).first[0] end diff --git a/lib/uvobot/uvo_scraper.rb b/lib/uvobot/uvo_scraper.rb index 588071b..8dcb7f9 100644 --- a/lib/uvobot/uvo_scraper.rb +++ b/lib/uvobot/uvo_scraper.rb @@ -27,12 +27,18 @@ def get_announcements(release_date) search_query = { kcpv: IT_CONTRACTS_CODE, dzOd: date, dzDo: date } html = @html_client.post(SEARCH_URL, search_query).body - [@parser.parse_page_info(html), @parser.parse_announcements(html, BULLETIN_URL)] + [@parser.parse_page_info(html), @parser.parse_announcements(html, BULLETIN_URL, release_date)] end - def get_announcement_detail(url) + def get_announcement_detail(url, release_date) html = @html_client.get(url).body - @parser.parse_detail(html) + response = @parser.parse_detail(html) + order_id, order_url = @parser.parse_order_documents_url(html) + response[:order] = { + id: order_id, + documents: @parser.parse_order_documents(@html_client.get(order_url).body, BULLETIN_URL, release_date) + } + response end end end diff --git a/uvobot.rb b/uvobot.rb index 4ca0468..c9a5d60 100644 --- a/uvobot.rb +++ b/uvobot.rb @@ -4,9 +4,10 @@ Dotenv.load discourse_client = Uvobot::DiscourseClient.new( - ENV.fetch('DISCOURSE_URL'), - ENV.fetch('DISCOURSE_API_KEY'), - ENV.fetch('DISCOURSE_USER') + host: ENV.fetch('DISCOURSE_URL'), + api_key: ENV.fetch('DISCOURSE_API_KEY'), + api_username: ENV.fetch('DISCOURSE_USER'), + local_store: Uvobot::Store::Manager.new(ENV.fetch('DATABASE_URL')) ) notifiers = [