From 864c2e71db1a3aa99e4131e2fd79600323073827 Mon Sep 17 00:00:00 2001 From: Lika Lex Date: Tue, 24 Jul 2018 23:00:38 +0300 Subject: [PATCH 1/8] Create basic sinatra application. --- 2199/3/Gemfile | 10 +++++ 2199/3/Gemfile.lock | 45 ++++++++++++++++++++ 2199/3/config.ru | 3 ++ 2199/3/controllers/application_controller.rb | 3 ++ 2199/3/controllers/links_controller.rb | 7 +++ 2199/3/link.rb | 34 +++++++++++++++ 2199/3/main.rb | 12 ++++++ 2199/3/views/index.erb | 28 ++++++++++++ 2199/3/views/style.css | 11 +++++ 9 files changed, 153 insertions(+) create mode 100644 2199/3/Gemfile create mode 100644 2199/3/Gemfile.lock create mode 100644 2199/3/config.ru create mode 100644 2199/3/controllers/application_controller.rb create mode 100644 2199/3/controllers/links_controller.rb create mode 100644 2199/3/link.rb create mode 100644 2199/3/main.rb create mode 100644 2199/3/views/index.erb create mode 100644 2199/3/views/style.css diff --git a/2199/3/Gemfile b/2199/3/Gemfile new file mode 100644 index 000000000..776d50858 --- /dev/null +++ b/2199/3/Gemfile @@ -0,0 +1,10 @@ +source 'https://rubygems.org' + +gem 'sinatra' +gem 'shotgun' +gem 'redis' +gem 'ohm' + +group :development do + gem 'pry' +end \ No newline at end of file diff --git a/2199/3/Gemfile.lock b/2199/3/Gemfile.lock new file mode 100644 index 000000000..0b1652006 --- /dev/null +++ b/2199/3/Gemfile.lock @@ -0,0 +1,45 @@ +GEM + remote: https://rubygems.org/ + specs: + coderay (1.1.2) + hiredis (0.6.1) + method_source (0.9.0) + mustermann (1.0.2) + nest (3.1.1) + redic + ohm (3.1.1) + nest (~> 3) + redic (~> 1.5.0) + stal + pry (0.11.3) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + rack (2.0.5) + rack-protection (2.0.3) + rack + redic (1.5.0) + hiredis + redis (4.0.1) + shotgun (0.9.2) + rack (>= 1.0) + sinatra (2.0.3) + mustermann (~> 1.0) + rack (~> 2.0) + rack-protection (= 2.0.3) + tilt (~> 2.0) + stal (0.3.0) + redic (~> 1.5) + tilt (2.0.8) + +PLATFORMS + ruby + +DEPENDENCIES + ohm + pry + redis + shotgun + sinatra + +BUNDLED WITH + 1.16.1 diff --git a/2199/3/config.ru b/2199/3/config.ru new file mode 100644 index 000000000..8abdd4668 --- /dev/null +++ b/2199/3/config.ru @@ -0,0 +1,3 @@ +require 'sinatra/base' +#Dir.glob('./{helpers, controllers}/*.rb').each { |file| require file } +map('/') { run ApplicationController } diff --git a/2199/3/controllers/application_controller.rb b/2199/3/controllers/application_controller.rb new file mode 100644 index 000000000..0407ae618 --- /dev/null +++ b/2199/3/controllers/application_controller.rb @@ -0,0 +1,3 @@ +require 'sinatra' +class ApplicationController < Sinatra::Base +end \ No newline at end of file diff --git a/2199/3/controllers/links_controller.rb b/2199/3/controllers/links_controller.rb new file mode 100644 index 000000000..fbc5a1969 --- /dev/null +++ b/2199/3/controllers/links_controller.rb @@ -0,0 +1,7 @@ +require 'sinatra' +# links controller +class LinksController < ApplicationController + get '/' do + 'index' + end +end \ No newline at end of file diff --git a/2199/3/link.rb b/2199/3/link.rb new file mode 100644 index 000000000..feb224977 --- /dev/null +++ b/2199/3/link.rb @@ -0,0 +1,34 @@ +# class link +class Link + def index + @link = Link.all.order('created_at DESC') + end + + def new + @link = Link.new + + end + + def create + @link = Link.new(link_params) + @link.save + end + + def show + @link = Link.find(params[:id]) + end + + def destroy + @link.destroy + end + + private + + def link_params + params.require(:link).permit(:name) + end + + def find_age + @age = Link.find(params[:id]) + end +end \ No newline at end of file diff --git a/2199/3/main.rb b/2199/3/main.rb new file mode 100644 index 000000000..8ec1ccaec --- /dev/null +++ b/2199/3/main.rb @@ -0,0 +1,12 @@ +require 'sinatra' +require 'redis' +require 'ohm' +require 'pry' + +#Ohm.redis = Redis.new('redis://127.0.0.1:6379') +get '/' do + #Ohm.redis.call 'SET', 'Foo', 'Bar' + + #Ohm.redis.call 'GET', 'Foo' + erb :index +end \ No newline at end of file diff --git a/2199/3/views/index.erb b/2199/3/views/index.erb new file mode 100644 index 000000000..c7b5e2bc4 --- /dev/null +++ b/2199/3/views/index.erb @@ -0,0 +1,28 @@ + + + + Your Page Title + + + +

Hello))) I am analyser!! Let's play))

+

Add link to new article

+ + + + + + + + + + + + + +
Number Link Rating Link to comments with rating
+ + + + + \ No newline at end of file diff --git a/2199/3/views/style.css b/2199/3/views/style.css new file mode 100644 index 000000000..d572cf0e4 --- /dev/null +++ b/2199/3/views/style.css @@ -0,0 +1,11 @@ + +body { + font-family: 'Merriweather', serif; + color: orangered; +} + +h1, h2, h3, h4, h5, h6 { + font-family: 'Oswald', sans-serif; + font-weight: 300; + color: brown; +} \ No newline at end of file From 2cf4112aa4102efd9b4f1efe59aa7dcff66ec3fe Mon Sep 17 00:00:00 2001 From: Lika Lex Date: Thu, 26 Jul 2018 14:10:59 +0300 Subject: [PATCH 2/8] Create Onliner parser --- 2199/3/Gemfile | 9 ++--- 2199/3/Gemfile.lock | 28 ++++++++++++++++ 2199/3/app/capybara_initializer.rb | 8 +++++ 2199/3/app/comment_analyzer.rb | 32 ++++++++++++++++++ 2199/3/app/comment_parser.rb | 13 ++++++++ 2199/3/app/data_store.rb | 11 ++++++ 2199/3/app/onliner_page_parser.rb | 35 ++++++++++++++++++++ 2199/3/assets/stylesheets/style.css | 11 ++++++ 2199/3/config.ru | 3 ++ 2199/3/controllers/application_controller.rb | 16 +++++++++ 2199/3/controllers/links_controller.rb | 7 ---- 2199/3/link.rb | 34 ------------------- 2199/3/main.rb | 12 ------- 2199/3/views/index.erb | 5 ++- 2199/3/views/new.erb | 15 +++++++++ 2199/3/views/show.erb | 25 ++++++++++++++ 16 files changed, 204 insertions(+), 60 deletions(-) create mode 100644 2199/3/app/capybara_initializer.rb create mode 100644 2199/3/app/comment_analyzer.rb create mode 100644 2199/3/app/comment_parser.rb create mode 100644 2199/3/app/data_store.rb create mode 100644 2199/3/app/onliner_page_parser.rb create mode 100644 2199/3/assets/stylesheets/style.css delete mode 100644 2199/3/controllers/links_controller.rb delete mode 100644 2199/3/link.rb delete mode 100644 2199/3/main.rb create mode 100644 2199/3/views/new.erb create mode 100644 2199/3/views/show.erb diff --git a/2199/3/Gemfile b/2199/3/Gemfile index 776d50858..e0ded29a7 100644 --- a/2199/3/Gemfile +++ b/2199/3/Gemfile @@ -1,10 +1,11 @@ source 'https://rubygems.org' -gem 'sinatra' -gem 'shotgun' -gem 'redis' +gem 'capybara' gem 'ohm' - +gem 'poltergeist' +gem 'redis' +gem 'shotgun' +gem 'sinatra' group :development do gem 'pry' end \ No newline at end of file diff --git a/2199/3/Gemfile.lock b/2199/3/Gemfile.lock index 0b1652006..12780a6cb 100644 --- a/2199/3/Gemfile.lock +++ b/2199/3/Gemfile.lock @@ -1,22 +1,43 @@ GEM remote: https://rubygems.org/ specs: + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) + capybara (3.4.1) + addressable + mini_mime (>= 0.1.3) + nokogiri (~> 1.8) + rack (>= 1.6.0) + rack-test (>= 0.6.3) + xpath (~> 3.1) + cliver (0.3.2) coderay (1.1.2) hiredis (0.6.1) method_source (0.9.0) + mini_mime (1.0.0) + mini_portile2 (2.3.0) mustermann (1.0.2) nest (3.1.1) redic + nokogiri (1.8.4) + mini_portile2 (~> 2.3.0) ohm (3.1.1) nest (~> 3) redic (~> 1.5.0) stal + poltergeist (1.18.1) + capybara (>= 2.1, < 4) + cliver (~> 0.3.1) + websocket-driver (>= 0.2.0) pry (0.11.3) coderay (~> 1.1.0) method_source (~> 0.9.0) + public_suffix (3.0.2) rack (2.0.5) rack-protection (2.0.3) rack + rack-test (1.1.0) + rack (>= 1.0, < 3) redic (1.5.0) hiredis redis (4.0.1) @@ -30,12 +51,19 @@ GEM stal (0.3.0) redic (~> 1.5) tilt (2.0.8) + websocket-driver (0.7.0) + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.3) + xpath (3.1.0) + nokogiri (~> 1.8) PLATFORMS ruby DEPENDENCIES + capybara ohm + poltergeist pry redis shotgun diff --git a/2199/3/app/capybara_initializer.rb b/2199/3/app/capybara_initializer.rb new file mode 100644 index 000000000..5e7de86e1 --- /dev/null +++ b/2199/3/app/capybara_initializer.rb @@ -0,0 +1,8 @@ +require 'capybara' +require 'capybara/poltergeist' +Capybara.register_driver :poltergeist do |app| + Capybara::Poltergeist::Driver.new(app, js_errors: false) +end + +# Configure Capybara to use Poltergeist as the driver +Capybara.default_driver = :poltergeist diff --git a/2199/3/app/comment_analyzer.rb b/2199/3/app/comment_analyzer.rb new file mode 100644 index 000000000..1e6d7e3e6 --- /dev/null +++ b/2199/3/app/comment_analyzer.rb @@ -0,0 +1,32 @@ +require_relative 'data_store' +class CommentsAnalyzer + attr_reader :text + + def initialize(text) + @text = text + end + + def analyze(text) + + count_rating(rating) + end + + def count_rating(rating) + (rating * 2) - 100 + end +end + +=begin +def parse_page(article) + article = Nokogiri::HTML(open(article_url)) + comments = [] + comments << article.html() +end + +def analyze_comment(comment) + session = Capybara::Session.new(:selenium) + session.visit('https://azure.microsoft.com/en-u¡¡s/services/cognitive-services/text-analytics/?v=18.05') + session.fill_in('InputText', with: comment) + session.click_button('Analyze') +end +=end \ No newline at end of file diff --git a/2199/3/app/comment_parser.rb b/2199/3/app/comment_parser.rb new file mode 100644 index 000000000..f2910e2bf --- /dev/null +++ b/2199/3/app/comment_parser.rb @@ -0,0 +1,13 @@ +class CommentParser + def initialize(node) + @node = node + end + + def text + @text ||= @node.find('.news-comment__speech p').text + end + + def rating + @rating ||= @node.all('.like-control span').sum { |mark| mark.text.to_i } + end +end \ No newline at end of file diff --git a/2199/3/app/data_store.rb b/2199/3/app/data_store.rb new file mode 100644 index 000000000..f58023d33 --- /dev/null +++ b/2199/3/app/data_store.rb @@ -0,0 +1,11 @@ +# storage of data +class DataStore + def initialize(page) + @page = page + @comments = {} + end + + def add_comment_to_storage(comment) + @comments << comment + end +end \ No newline at end of file diff --git a/2199/3/app/onliner_page_parser.rb b/2199/3/app/onliner_page_parser.rb new file mode 100644 index 000000000..5009befd2 --- /dev/null +++ b/2199/3/app/onliner_page_parser.rb @@ -0,0 +1,35 @@ +require 'pry' +require_relative '../app/comment_parser' +require_relative 'capybara_initializer' + +# class for parsing +class OnlinerPageParser + COMMENTS_TO_TAKE = 50 + + def initialize(link) + @link = link + end + + def browser + @browser ||= Capybara.current_session + end + + def top_comment_texts + browser.visit(@link) + binding.pry + browser.find('.news-comment__all-button_visible').click + top_comments.map(&:text) + end + + def comment_nodes + browser.all('.news-comment__unit').drop(1) + end + + def comments + comment_nodes.map { |comment_node| CommentParser.new(comment_node) } + end + + def top_comments + comments.sort_by(&:rating).reverse.first(COMMENTS_TO_TAKE) + end +end diff --git a/2199/3/assets/stylesheets/style.css b/2199/3/assets/stylesheets/style.css new file mode 100644 index 000000000..d572cf0e4 --- /dev/null +++ b/2199/3/assets/stylesheets/style.css @@ -0,0 +1,11 @@ + +body { + font-family: 'Merriweather', serif; + color: orangered; +} + +h1, h2, h3, h4, h5, h6 { + font-family: 'Oswald', sans-serif; + font-weight: 300; + color: brown; +} \ No newline at end of file diff --git a/2199/3/config.ru b/2199/3/config.ru index 8abdd4668..f8a07400c 100644 --- a/2199/3/config.ru +++ b/2199/3/config.ru @@ -1,3 +1,6 @@ require 'sinatra/base' +require_relative 'app/onliner_page_parser' +require_relative 'controllers/application_controller' + #Dir.glob('./{helpers, controllers}/*.rb').each { |file| require file } map('/') { run ApplicationController } diff --git a/2199/3/controllers/application_controller.rb b/2199/3/controllers/application_controller.rb index 0407ae618..2bd72e9db 100644 --- a/2199/3/controllers/application_controller.rb +++ b/2199/3/controllers/application_controller.rb @@ -1,3 +1,19 @@ require 'sinatra' +require 'pry' + class ApplicationController < Sinatra::Base + set views: 'views/' + + get '/' do + erb :index + end + + get '/new' do + erb :new + end + + post '/analyze' do + binding.pry + OnlinerPageParser.new(params[:link]).top_comment_texts + end end \ No newline at end of file diff --git a/2199/3/controllers/links_controller.rb b/2199/3/controllers/links_controller.rb deleted file mode 100644 index fbc5a1969..000000000 --- a/2199/3/controllers/links_controller.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'sinatra' -# links controller -class LinksController < ApplicationController - get '/' do - 'index' - end -end \ No newline at end of file diff --git a/2199/3/link.rb b/2199/3/link.rb deleted file mode 100644 index feb224977..000000000 --- a/2199/3/link.rb +++ /dev/null @@ -1,34 +0,0 @@ -# class link -class Link - def index - @link = Link.all.order('created_at DESC') - end - - def new - @link = Link.new - - end - - def create - @link = Link.new(link_params) - @link.save - end - - def show - @link = Link.find(params[:id]) - end - - def destroy - @link.destroy - end - - private - - def link_params - params.require(:link).permit(:name) - end - - def find_age - @age = Link.find(params[:id]) - end -end \ No newline at end of file diff --git a/2199/3/main.rb b/2199/3/main.rb deleted file mode 100644 index 8ec1ccaec..000000000 --- a/2199/3/main.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'sinatra' -require 'redis' -require 'ohm' -require 'pry' - -#Ohm.redis = Redis.new('redis://127.0.0.1:6379') -get '/' do - #Ohm.redis.call 'SET', 'Foo', 'Bar' - - #Ohm.redis.call 'GET', 'Foo' - erb :index -end \ No newline at end of file diff --git a/2199/3/views/index.erb b/2199/3/views/index.erb index c7b5e2bc4..09259049e 100644 --- a/2199/3/views/index.erb +++ b/2199/3/views/index.erb @@ -2,14 +2,13 @@ Your Page Title - +

Hello))) I am analyser!! Let's play))

Add link to new article

- - +Analyse new link!! diff --git a/2199/3/views/new.erb b/2199/3/views/new.erb new file mode 100644 index 000000000..7033eb9b6 --- /dev/null +++ b/2199/3/views/new.erb @@ -0,0 +1,15 @@ + + + + My Parser - Add Link + + +

Add New Link

+> + + + + +Back to Index + + \ No newline at end of file diff --git a/2199/3/views/show.erb b/2199/3/views/show.erb new file mode 100644 index 000000000..d75f84515 --- /dev/null +++ b/2199/3/views/show.erb @@ -0,0 +1,25 @@ + + + + + About article <%= @article.title %> + + +

<%= @article.title %>

+
+ + + + + + + + + + + + +
Title:<%= @article.title %>
Link:<%= @article.link %>
Average rating:<%= @article.rating %>
+Back to Index + + \ No newline at end of file From 8d6ee8d67809721ef63302a8f2a25024f36cfcf5 Mon Sep 17 00:00:00 2001 From: Lika Lex Date: Thu, 26 Jul 2018 22:45:23 +0300 Subject: [PATCH 3/8] Create models --- 2199/3/models/comment.rb | 5 +++++ 2199/3/models/page.rb | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 2199/3/models/comment.rb create mode 100644 2199/3/models/page.rb diff --git a/2199/3/models/comment.rb b/2199/3/models/comment.rb new file mode 100644 index 000000000..de19c2a50 --- /dev/null +++ b/2199/3/models/comment.rb @@ -0,0 +1,5 @@ +class Comment < Ohm::Model + attribute :text + attribute :rating + reference :page, Page +end \ No newline at end of file diff --git a/2199/3/models/page.rb b/2199/3/models/page.rb new file mode 100644 index 000000000..2287c84cf --- /dev/null +++ b/2199/3/models/page.rb @@ -0,0 +1,4 @@ +class Page < Ohm::Model + attribute :title + collection :comments, Comment +end \ No newline at end of file From f9a449e5203caabf2c785984f9c12d8274835630 Mon Sep 17 00:00:00 2001 From: Lika Lex Date: Fri, 27 Jul 2018 11:45:19 +0300 Subject: [PATCH 4/8] Create comment analyzer --- 2199/3/.gitignore | 1 + 2199/3/Gemfile | 1 + 2199/3/Gemfile.lock | 2 + 2199/3/app/comment_analyzer.rb | 59 ++++++++++++-------- 2199/3/app/comment_parser.rb | 2 +- 2199/3/app/data_store.rb | 11 ---- 2199/3/app/onliner_page_parser.rb | 27 +++++++-- 2199/3/config.ru | 6 +- 2199/3/controllers/application_controller.rb | 13 ++++- 2199/3/models/comment.rb | 2 +- 2199/3/models/page.rb | 9 ++- 2199/3/views/index.erb | 10 ++-- 2199/3/views/show.erb | 15 +++-- 13 files changed, 100 insertions(+), 58 deletions(-) create mode 100644 2199/3/.gitignore delete mode 100644 2199/3/app/data_store.rb diff --git a/2199/3/.gitignore b/2199/3/.gitignore new file mode 100644 index 000000000..2eea525d8 --- /dev/null +++ b/2199/3/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/2199/3/Gemfile b/2199/3/Gemfile index e0ded29a7..b24cf5a97 100644 --- a/2199/3/Gemfile +++ b/2199/3/Gemfile @@ -1,6 +1,7 @@ source 'https://rubygems.org' gem 'capybara' +gem 'dotenv' gem 'ohm' gem 'poltergeist' gem 'redis' diff --git a/2199/3/Gemfile.lock b/2199/3/Gemfile.lock index 12780a6cb..b4ad02891 100644 --- a/2199/3/Gemfile.lock +++ b/2199/3/Gemfile.lock @@ -12,6 +12,7 @@ GEM xpath (~> 3.1) cliver (0.3.2) coderay (1.1.2) + dotenv (2.4.0) hiredis (0.6.1) method_source (0.9.0) mini_mime (1.0.0) @@ -62,6 +63,7 @@ PLATFORMS DEPENDENCIES capybara + dotenv ohm poltergeist pry diff --git a/2199/3/app/comment_analyzer.rb b/2199/3/app/comment_analyzer.rb index 1e6d7e3e6..f60a60624 100644 --- a/2199/3/app/comment_analyzer.rb +++ b/2199/3/app/comment_analyzer.rb @@ -1,32 +1,47 @@ -require_relative 'data_store' -class CommentsAnalyzer - attr_reader :text +require 'dotenv/load' +class CommentAnalyzer + ACCESS_KEY = ENV['ACCESS_KEY'] + AZURE_ENDPOINT = ENV['AZURE_ENDPOINT'] - def initialize(text) - @text = text + def initialize(texts) + @texts = texts end - def analyze(text) + def analyze + JSON.parse(run_request.body)['documents'].each_with_object([]) do |result, store| + document = documents.find { |document| document[:id].to_s == result['id'] } + store << { + text: document[:text], + rating: result['score'] * 200 - 100 + } + end + end - count_rating(rating) + def endpoint + @endpoint ||= URI(AZURE_ENDPOINT) end - def count_rating(rating) - (rating * 2) - 100 + def request + request = Net::HTTP::Post.new(endpoint) + request['Content-Type'] = 'application/json' + request['Ocp-Apim-Subscription-Key'] = ACCESS_KEY + request.body = serialized_texts + request end -end -=begin -def parse_page(article) - article = Nokogiri::HTML(open(article_url)) - comments = [] - comments << article.html() -end + def documents + @documents ||= @texts.map do |text| + {id: text.object_id, language: 'ru', text: text} + end + end -def analyze_comment(comment) - session = Capybara::Session.new(:selenium) - session.visit('https://azure.microsoft.com/en-u¡¡s/services/cognitive-services/text-analytics/?v=18.05') - session.fill_in('InputText', with: comment) - session.click_button('Analyze') + def serialized_texts + {documents: documents}.to_json + end + + def run_request + https = Net::HTTP.new(endpoint.host, endpoint.port) + https.use_ssl = true + https.request(request) + end end -=end \ No newline at end of file diff --git a/2199/3/app/comment_parser.rb b/2199/3/app/comment_parser.rb index f2910e2bf..0c9abf36e 100644 --- a/2199/3/app/comment_parser.rb +++ b/2199/3/app/comment_parser.rb @@ -4,7 +4,7 @@ def initialize(node) end def text - @text ||= @node.find('.news-comment__speech p').text + @text ||= @node.all('.news-comment__speech > div > p').map(&:text).reduce(&:+) end def rating diff --git a/2199/3/app/data_store.rb b/2199/3/app/data_store.rb deleted file mode 100644 index f58023d33..000000000 --- a/2199/3/app/data_store.rb +++ /dev/null @@ -1,11 +0,0 @@ -# storage of data -class DataStore - def initialize(page) - @page = page - @comments = {} - end - - def add_comment_to_storage(comment) - @comments << comment - end -end \ No newline at end of file diff --git a/2199/3/app/onliner_page_parser.rb b/2199/3/app/onliner_page_parser.rb index 5009befd2..ac2183e1f 100644 --- a/2199/3/app/onliner_page_parser.rb +++ b/2199/3/app/onliner_page_parser.rb @@ -14,15 +14,34 @@ def browser @browser ||= Capybara.current_session end + def visit_page + attempts = 0 + begin + browser.visit(@link) + rescue StandardError => e + attempts += 1 + sleep(2 * attempts) + if attempts <= 3 + retry + else + raise e + end + end + end + + def show_all_comments + browser.find('.news-form__control_condensed .news-form__button').click + sleep(5) + end + def top_comment_texts - browser.visit(@link) - binding.pry - browser.find('.news-comment__all-button_visible').click + visit_page + show_all_comments top_comments.map(&:text) end def comment_nodes - browser.all('.news-comment__unit').drop(1) + browser.all('[id^="comment-"]').drop(1) end def comments diff --git a/2199/3/config.ru b/2199/3/config.ru index f8a07400c..c272c369a 100644 --- a/2199/3/config.ru +++ b/2199/3/config.ru @@ -1,6 +1,10 @@ require 'sinatra/base' +require 'ohm' require_relative 'app/onliner_page_parser' +require_relative 'app/comment_analyzer' require_relative 'controllers/application_controller' +require_relative 'models/page' +require_relative 'models/comment' -#Dir.glob('./{helpers, controllers}/*.rb').each { |file| require file } +# Dir.glob('./{helpers, controllers}/*.rb').each { |file| require file } map('/') { run ApplicationController } diff --git a/2199/3/controllers/application_controller.rb b/2199/3/controllers/application_controller.rb index 2bd72e9db..bbb618f10 100644 --- a/2199/3/controllers/application_controller.rb +++ b/2199/3/controllers/application_controller.rb @@ -13,7 +13,16 @@ class ApplicationController < Sinatra::Base end post '/analyze' do - binding.pry - OnlinerPageParser.new(params[:link]).top_comment_texts + link = params[:link] + @page = Page.find(link: link).first + unless @page + comment_texts = OnlinerPageParser.new(link).top_comment_texts + comments_with_score = CommentAnalyzer.new(comment_texts).analyze + @page = Page.create(link: link) + comments_with_score.each do |comment_data| + Comment.create(comment_data.merge(page: @page)) + end + end + erb :show end end \ No newline at end of file diff --git a/2199/3/models/comment.rb b/2199/3/models/comment.rb index de19c2a50..77ceb40a9 100644 --- a/2199/3/models/comment.rb +++ b/2199/3/models/comment.rb @@ -1,5 +1,5 @@ class Comment < Ohm::Model attribute :text attribute :rating - reference :page, Page + reference :page, 'Page' end \ No newline at end of file diff --git a/2199/3/models/page.rb b/2199/3/models/page.rb index 2287c84cf..2e3f2e4ef 100644 --- a/2199/3/models/page.rb +++ b/2199/3/models/page.rb @@ -1,4 +1,9 @@ class Page < Ohm::Model - attribute :title - collection :comments, Comment + attribute :link + collection :comments, 'Comment' + index :link + + def rating + comments.sum { |comment| comment.rating.to_f } / comments.count + end end \ No newline at end of file diff --git a/2199/3/views/index.erb b/2199/3/views/index.erb index 09259049e..137ceaf34 100644 --- a/2199/3/views/index.erb +++ b/2199/3/views/index.erb @@ -1,15 +1,13 @@ - Your Page Title - + Analyzer -

Hello))) I am analyser!! Let's play))

-

Add link to new article

+

Hello))) I am analyzer!! Let's play))

Analyse new link!! - + diff --git a/2199/3/views/show.erb b/2199/3/views/show.erb index d75f84515..740f8564e 100644 --- a/2199/3/views/show.erb +++ b/2199/3/views/show.erb @@ -2,24 +2,23 @@ - About article <%= @article.title %> + About article <%= @page.link %> -

<%= @article.title %>

- - - - - + - +
Title:<%= @article.title %>
Link:<%= @article.link %><%= @page.link %>
Average rating:<%= @article.rating %><%= @page.rating %>
+<% @page.comments.each do |comment| %> + <%= "#{comment.rating} --- #{comment.text}" %> +
+<% end %> Back to Index \ No newline at end of file From 94cac61124a483cadff2a6e87c8bd87176d28f6b Mon Sep 17 00:00:00 2001 From: Lika Lex Date: Fri, 27 Jul 2018 15:04:48 +0300 Subject: [PATCH 5/8] Update pages style, fix minor issues. --- 2199/3/Gemfile | 2 +- 2199/3/app/comment_analyzer.rb | 7 ++-- 2199/3/app/comment_parser.rb | 3 +- 2199/3/app/onliner_page_parser.rb | 12 +++--- 2199/3/controllers/application_controller.rb | 10 ++++- 2199/3/models/comment.rb | 3 +- 2199/3/models/page.rb | 3 +- 2199/3/views/index.erb | 43 +++++++++----------- 2199/3/views/layout.erb | 10 +++++ 2199/3/views/new.erb | 9 ---- 2199/3/views/show.erb | 34 +++++++++------- 2199/3/views/style.css | 11 ----- 12 files changed, 73 insertions(+), 74 deletions(-) create mode 100644 2199/3/views/layout.erb delete mode 100644 2199/3/views/style.css diff --git a/2199/3/Gemfile b/2199/3/Gemfile index b24cf5a97..296b0c4eb 100644 --- a/2199/3/Gemfile +++ b/2199/3/Gemfile @@ -9,4 +9,4 @@ gem 'shotgun' gem 'sinatra' group :development do gem 'pry' -end \ No newline at end of file +end diff --git a/2199/3/app/comment_analyzer.rb b/2199/3/app/comment_analyzer.rb index f60a60624..ff0d5e3ca 100644 --- a/2199/3/app/comment_analyzer.rb +++ b/2199/3/app/comment_analyzer.rb @@ -1,4 +1,5 @@ require 'dotenv/load' +# analyze comment class CommentAnalyzer ACCESS_KEY = ENV['ACCESS_KEY'] AZURE_ENDPOINT = ENV['AZURE_ENDPOINT'] @@ -9,7 +10,7 @@ def initialize(texts) def analyze JSON.parse(run_request.body)['documents'].each_with_object([]) do |result, store| - document = documents.find { |document| document[:id].to_s == result['id'] } + document = documents.find { |doc| doc[:id].to_s == result['id'] } store << { text: document[:text], rating: result['score'] * 200 - 100 @@ -31,12 +32,12 @@ def request def documents @documents ||= @texts.map do |text| - {id: text.object_id, language: 'ru', text: text} + { id: text.object_id, language: 'ru', text: text } end end def serialized_texts - {documents: documents}.to_json + { documents: documents }.to_json end def run_request diff --git a/2199/3/app/comment_parser.rb b/2199/3/app/comment_parser.rb index 0c9abf36e..b6d09a16d 100644 --- a/2199/3/app/comment_parser.rb +++ b/2199/3/app/comment_parser.rb @@ -1,3 +1,4 @@ +# parse node and find text and rating class CommentParser def initialize(node) @node = node @@ -10,4 +11,4 @@ def text def rating @rating ||= @node.all('.like-control span').sum { |mark| mark.text.to_i } end -end \ No newline at end of file +end diff --git a/2199/3/app/onliner_page_parser.rb b/2199/3/app/onliner_page_parser.rb index ac2183e1f..fa4512519 100644 --- a/2199/3/app/onliner_page_parser.rb +++ b/2199/3/app/onliner_page_parser.rb @@ -18,19 +18,17 @@ def visit_page attempts = 0 begin browser.visit(@link) - rescue StandardError => e + rescue StandardError => exception attempts += 1 sleep(2 * attempts) - if attempts <= 3 - retry - else - raise e - end + retry if attempts <= 3 + raise exception end end def show_all_comments - browser.find('.news-form__control_condensed .news-form__button').click + browser.find('.button-style.button-style_subsidiary.button-style_big.news-form__button'\ + '.news-form__button_width_full.news-form__button_font-weight_semibold').click sleep(5) end diff --git a/2199/3/controllers/application_controller.rb b/2199/3/controllers/application_controller.rb index bbb618f10..4d767c8af 100644 --- a/2199/3/controllers/application_controller.rb +++ b/2199/3/controllers/application_controller.rb @@ -1,10 +1,11 @@ require 'sinatra' require 'pry' - +# class ApplicationController class ApplicationController < Sinatra::Base set views: 'views/' get '/' do + @pages = Page.all erb :index end @@ -12,6 +13,11 @@ class ApplicationController < Sinatra::Base erb :new end + get '/show/:id' do + @page = Page[params[:id]] + erb :show + end + post '/analyze' do link = params[:link] @page = Page.find(link: link).first @@ -25,4 +31,4 @@ class ApplicationController < Sinatra::Base end erb :show end -end \ No newline at end of file +end diff --git a/2199/3/models/comment.rb b/2199/3/models/comment.rb index 77ceb40a9..5d9f47f16 100644 --- a/2199/3/models/comment.rb +++ b/2199/3/models/comment.rb @@ -1,5 +1,6 @@ +# Comment model class Comment < Ohm::Model attribute :text attribute :rating reference :page, 'Page' -end \ No newline at end of file +end diff --git a/2199/3/models/page.rb b/2199/3/models/page.rb index 2e3f2e4ef..2929d0d84 100644 --- a/2199/3/models/page.rb +++ b/2199/3/models/page.rb @@ -1,3 +1,4 @@ +# Page model class Page < Ohm::Model attribute :link collection :comments, 'Comment' @@ -6,4 +7,4 @@ class Page < Ohm::Model def rating comments.sum { |comment| comment.rating.to_f } / comments.count end -end \ No newline at end of file +end diff --git a/2199/3/views/index.erb b/2199/3/views/index.erb index 137ceaf34..437632363 100644 --- a/2199/3/views/index.erb +++ b/2199/3/views/index.erb @@ -1,25 +1,20 @@ - - - - Analyzer - - -

Hello))) I am analyzer!! Let's play))

- Analyse new link!! - - - - - - \ No newline at end of file + + + + + + + + + + <% @pages.each do |page| %> + + + + + + <% end %> + + +
Number Link Rating
"><%= page.id %><%= page.link %><%= page.rating %>
diff --git a/2199/3/views/layout.erb b/2199/3/views/layout.erb new file mode 100644 index 000000000..30db9b08e --- /dev/null +++ b/2199/3/views/layout.erb @@ -0,0 +1,10 @@ + + + + Analyzer + + + + <%= yield %> + + diff --git a/2199/3/views/new.erb b/2199/3/views/new.erb index 7033eb9b6..e41041668 100644 --- a/2199/3/views/new.erb +++ b/2199/3/views/new.erb @@ -1,15 +1,6 @@ - - - - My Parser - Add Link - - -

Add New Link

>
Back to Index - - \ No newline at end of file diff --git a/2199/3/views/show.erb b/2199/3/views/show.erb index 740f8564e..efa1c9138 100644 --- a/2199/3/views/show.erb +++ b/2199/3/views/show.erb @@ -1,10 +1,3 @@ - - - - - About article <%= @page.link %> - - @@ -12,13 +5,26 @@ - +
Link:
Average rating:<%= @page.rating %><%= @page.rating.to_i %>
-<% @page.comments.each do |comment| %> - <%= "#{comment.rating} --- #{comment.text}" %> -
-<% end %> + + + + + + + + + + <% @page.comments.each do |comment| %> + + + + + <% end %> + +
Rating Comment
<%= "#{comment.rating.to_i}" %><%= "#{comment.text}" %>
+ + Back to Index - - \ No newline at end of file diff --git a/2199/3/views/style.css b/2199/3/views/style.css deleted file mode 100644 index d572cf0e4..000000000 --- a/2199/3/views/style.css +++ /dev/null @@ -1,11 +0,0 @@ - -body { - font-family: 'Merriweather', serif; - color: orangered; -} - -h1, h2, h3, h4, h5, h6 { - font-family: 'Oswald', sans-serif; - font-weight: 300; - color: brown; -} \ No newline at end of file From 49401ad1cd1fd31fa4f7650d8074b97ecb5c2044 Mon Sep 17 00:00:00 2001 From: Lika Lex Date: Fri, 27 Jul 2018 15:52:48 +0300 Subject: [PATCH 6/8] reek/rubocop disable --- 2199/3/app/comment_analyzer.rb | 4 ++++ 2199/3/app/onliner_page_parser.rb | 1 + 2199/3/assets/stylesheets/style.css | 11 ----------- 3 files changed, 5 insertions(+), 11 deletions(-) delete mode 100644 2199/3/assets/stylesheets/style.css diff --git a/2199/3/app/comment_analyzer.rb b/2199/3/app/comment_analyzer.rb index ff0d5e3ca..47744ff1f 100644 --- a/2199/3/app/comment_analyzer.rb +++ b/2199/3/app/comment_analyzer.rb @@ -8,6 +8,8 @@ def initialize(texts) @texts = texts end + # :reek:NestedIterators + # rubocop:disable Metrics/AbcSize def analyze JSON.parse(run_request.body)['documents'].each_with_object([]) do |result, store| document = documents.find { |doc| doc[:id].to_s == result['id'] } @@ -17,11 +19,13 @@ def analyze } end end + # rubocop:enable Metrics/AbcSize def endpoint @endpoint ||= URI(AZURE_ENDPOINT) end + # :reek:FeatureEnvy def request request = Net::HTTP::Post.new(endpoint) request['Content-Type'] = 'application/json' diff --git a/2199/3/app/onliner_page_parser.rb b/2199/3/app/onliner_page_parser.rb index fa4512519..212366fe3 100644 --- a/2199/3/app/onliner_page_parser.rb +++ b/2199/3/app/onliner_page_parser.rb @@ -14,6 +14,7 @@ def browser @browser ||= Capybara.current_session end + # :reek:TooManyStatements def visit_page attempts = 0 begin diff --git a/2199/3/assets/stylesheets/style.css b/2199/3/assets/stylesheets/style.css deleted file mode 100644 index d572cf0e4..000000000 --- a/2199/3/assets/stylesheets/style.css +++ /dev/null @@ -1,11 +0,0 @@ - -body { - font-family: 'Merriweather', serif; - color: orangered; -} - -h1, h2, h3, h4, h5, h6 { - font-family: 'Oswald', sans-serif; - font-weight: 300; - color: brown; -} \ No newline at end of file From 8a427102092d6802504d418c450ff36e86eb46ec Mon Sep 17 00:00:00 2001 From: Lika Lex Date: Fri, 27 Jul 2018 17:16:09 +0300 Subject: [PATCH 7/8] Stick to RESTful routes. --- 2199/3/.gitignore | 2 +- 2199/3/controllers/application_controller.rb | 10 +++++++--- 2199/3/views/index.erb | 4 ++-- 2199/3/views/new.erb | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/2199/3/.gitignore b/2199/3/.gitignore index 2eea525d8..4c49bd78f 100644 --- a/2199/3/.gitignore +++ b/2199/3/.gitignore @@ -1 +1 @@ -.env \ No newline at end of file +.env diff --git a/2199/3/controllers/application_controller.rb b/2199/3/controllers/application_controller.rb index 4d767c8af..cbe1e2fb2 100644 --- a/2199/3/controllers/application_controller.rb +++ b/2199/3/controllers/application_controller.rb @@ -5,20 +5,24 @@ class ApplicationController < Sinatra::Base set views: 'views/' get '/' do + redirect '/pages' + end + + get '/pages' do @pages = Page.all erb :index end - get '/new' do + get '/pages/new' do erb :new end - get '/show/:id' do + get '/pages/:id' do @page = Page[params[:id]] erb :show end - post '/analyze' do + post '/pages' do link = params[:link] @page = Page.find(link: link).first unless @page diff --git a/2199/3/views/index.erb b/2199/3/views/index.erb index 437632363..6d5ab5ac1 100644 --- a/2199/3/views/index.erb +++ b/2199/3/views/index.erb @@ -1,4 +1,4 @@ -Analyse new link!! +Analyse new link!! @@ -10,7 +10,7 @@ <% @pages.each do |page| %> - + diff --git a/2199/3/views/new.erb b/2199/3/views/new.erb index e41041668..7a0ac5b33 100644 --- a/2199/3/views/new.erb +++ b/2199/3/views/new.erb @@ -1,4 +1,4 @@ -> +> From 9490ddc467f5b0dc29093898cedb60e88f449c54 Mon Sep 17 00:00:00 2001 From: Lika Lex Date: Mon, 30 Jul 2018 08:49:11 +0300 Subject: [PATCH 8/8] Renamed the necessary classes and methods, used tap --- 2199/3/app/comment_analyzer.rb | 22 ++++++++++---------- 2199/3/app/onliner_page_parser.rb | 2 +- 2199/3/controllers/application_controller.rb | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/2199/3/app/comment_analyzer.rb b/2199/3/app/comment_analyzer.rb index 47744ff1f..86de4dd46 100644 --- a/2199/3/app/comment_analyzer.rb +++ b/2199/3/app/comment_analyzer.rb @@ -1,8 +1,8 @@ require 'dotenv/load' # analyze comment -class CommentAnalyzer +class CommentsAnalyzer ACCESS_KEY = ENV['ACCESS_KEY'] - AZURE_ENDPOINT = ENV['AZURE_ENDPOINT'] + AZURE_ENDPOINT = 'https://westcentralus.api.cognitive.microsoft.com/text/analytics/v2.0/sentiment'.freeze def initialize(texts) @texts = texts @@ -21,17 +21,17 @@ def analyze end # rubocop:enable Metrics/AbcSize - def endpoint - @endpoint ||= URI(AZURE_ENDPOINT) + def azure_endpoint + @azure_endpoint ||= URI(AZURE_ENDPOINT) end # :reek:FeatureEnvy def request - request = Net::HTTP::Post.new(endpoint) - request['Content-Type'] = 'application/json' - request['Ocp-Apim-Subscription-Key'] = ACCESS_KEY - request.body = serialized_texts - request + Net::HTTP::Post.new(azure_endpoint).tap do |request| + request['Content-Type'] = 'application/json' + request['Ocp-Apim-Subscription-Key'] = ACCESS_KEY + request.body = serialized_texts_json + end end def documents @@ -40,12 +40,12 @@ def documents end end - def serialized_texts + def serialized_texts_json { documents: documents }.to_json end def run_request - https = Net::HTTP.new(endpoint.host, endpoint.port) + https = Net::HTTP.new(azure_endpoint.host, azure_endpoint.port) https.use_ssl = true https.request(request) end diff --git a/2199/3/app/onliner_page_parser.rb b/2199/3/app/onliner_page_parser.rb index 212366fe3..966f9285e 100644 --- a/2199/3/app/onliner_page_parser.rb +++ b/2199/3/app/onliner_page_parser.rb @@ -3,7 +3,7 @@ require_relative 'capybara_initializer' # class for parsing -class OnlinerPageParser +class OnlinerPage COMMENTS_TO_TAKE = 50 def initialize(link) diff --git a/2199/3/controllers/application_controller.rb b/2199/3/controllers/application_controller.rb index cbe1e2fb2..7fa0e9804 100644 --- a/2199/3/controllers/application_controller.rb +++ b/2199/3/controllers/application_controller.rb @@ -26,8 +26,8 @@ class ApplicationController < Sinatra::Base link = params[:link] @page = Page.find(link: link).first unless @page - comment_texts = OnlinerPageParser.new(link).top_comment_texts - comments_with_score = CommentAnalyzer.new(comment_texts).analyze + comment_texts = OnlinerPage.new(link).top_comment_texts + comments_with_score = CommentsAnalyzer.new(comment_texts).analyze @page = Page.create(link: link) comments_with_score.each do |comment_data| Comment.create(comment_data.merge(page: @page))
"><%= page.id %>"><%= page.id %> <%= page.link %> <%= page.rating %>