From bc6a063f30e7c0705db6479037f6b2c66aa07b59 Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Mon, 2 Sep 2013 14:42:59 +0200 Subject: [PATCH 01/19] Change version number to 0.1.0 --- Gemfile | 2 +- Gemfile.lock | 2 +- lib/oss_active_record/version.rb | 2 +- oss_active_record.gemspec | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index 9c03aea..c4dc375 100644 --- a/Gemfile +++ b/Gemfile @@ -12,4 +12,4 @@ gemspec # To use debugger gem 'pry' -gem 'oss_rb', :git => "https://github.com/emmanuel-keller/oss_rb" +gem 'oss_rb', :git => "https://github.com/jaeksoft/oss_rb" diff --git a/Gemfile.lock b/Gemfile.lock index b45dcb8..a1002f5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,7 +2,7 @@ GIT remote: https://github.com/emmanuel-keller/oss_rb revision: fc29f72083004be1100e21ab4b5690a60924dc8c specs: - oss_rb (0.0.1) + oss_rb (0.1.0) activesupport rest-client diff --git a/lib/oss_active_record/version.rb b/lib/oss_active_record/version.rb index c0320f5..73f808d 100644 --- a/lib/oss_active_record/version.rb +++ b/lib/oss_active_record/version.rb @@ -1,3 +1,3 @@ module OssActiveRecord - VERSION = "0.0.1" + VERSION = "0.1.0" end diff --git a/oss_active_record.gemspec b/oss_active_record.gemspec index 052dfa7..ac88960 100644 --- a/oss_active_record.gemspec +++ b/oss_active_record.gemspec @@ -10,8 +10,8 @@ Gem::Specification.new do |s| s.authors = ["Ori Pekelman", "Emmanuel Keller"] s.email = ["ori@pekelman.com", "ekeller@open-search-server.com"] s.homepage = "http://www.open-search-server.com" - s.summary = "Open search server ActiveRecord integration" - s.description = "Open search server ActiveRecord integration" + s.summary = "OpenSearchServer ActiveRecord integration" + s.description = "OpenSearchServer ActiveRecord integration" s.files = Dir["{app,config,db,lib}/**/*", "MIT-LICENSE", "Rakefile", "README.rdoc"] s.test_files = Dir["test/**/*"] From e6db2b6c53f6c00d4de652f104932eeeb9ad0efa Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Mon, 2 Sep 2013 15:49:34 +0200 Subject: [PATCH 02/19] Few documentation updates --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7f35ebd..619c703 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,33 @@ A very initial active_record intergration for OpenSearchServer http://www.open-search-server.com -This is very very initial, no tests yet, but there is a running dummy application +This is very very initial, but there is a running dummy application with test cases. -Depends on OriPekelman/oss_rb +Depends on jaeksoft/oss_rb ## Installation Add this line to your application's Gemfile: - gem 'oss_rb', :github => "OriPekelman/oss_rb" - gem 'oss_active_record', :github => "OriPekelman/oss_active_record" + gem 'oss_rb' + gem 'oss_active_record' And then execute: $ bundle -Or install it yourself as (not on rubygems yet!): +Or install it yourself as: $ gem install oss_active_record ## Configuration + Add `config/initializers/oss_active_record.rb` with the url of opens-search-server Rails.configuration.open_search_server_url = "http://localhost:8080" ## Usage - ```ruby class Person < ActiveRecord::Base searchable do From 3e56905b6ced0c3d6c89afc27fbc71f9351165df Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Mon, 2 Sep 2013 22:08:35 +0200 Subject: [PATCH 03/19] Update documentation and gem spec --- Gemfile.lock | 16 ++++++++-------- README.md | 7 ++++++- oss_active_record.gemspec | 4 +++- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a1002f5..33dcb5f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT - remote: https://github.com/emmanuel-keller/oss_rb - revision: fc29f72083004be1100e21ab4b5690a60924dc8c + remote: https://github.com/jaeksoft/oss_rb + revision: 227c80c3c56c945e0cd2a727ca7e373a7f3658c0 specs: oss_rb (0.1.0) activesupport @@ -9,7 +9,7 @@ GIT PATH remote: . specs: - oss_active_record (0.0.1) + oss_active_record (0.1.0) rails (~> 4.0.0) GEM @@ -40,19 +40,19 @@ GEM thread_safe (~> 0.1) tzinfo (~> 0.3.37) arel (4.0.0) - atomic (1.1.10) + atomic (1.1.13) builder (3.1.4) coderay (1.0.9) erubis (2.7.0) hike (1.2.3) - i18n (0.6.4) + i18n (0.6.5) mail (2.5.4) mime-types (~> 1.16) treetop (~> 1.4.8) method_source (0.8.2) - mime-types (1.23) + mime-types (1.25) minitest (4.7.5) - multi_json (1.7.7) + multi_json (1.7.9) polyglot (0.3.3) pry (0.9.12.2) coderay (~> 1.0.5) @@ -92,7 +92,7 @@ GEM thread_safe (0.1.2) atomic tilt (1.4.1) - treetop (1.4.14) + treetop (1.4.15) polyglot polyglot (>= 0.3.1) tzinfo (0.3.37) diff --git a/README.md b/README.md index 619c703..6751409 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,14 @@ -A very initial active_record intergration for OpenSearchServer http://www.open-search-server.com +An active_record intergration for OpenSearchServer This is very very initial, but there is a running dummy application with test cases. Depends on jaeksoft/oss_rb +## Requirement + +A running OpenSearchServer instance (>= v1.5) +http://www.open-search-server.com + ## Installation Add this line to your application's Gemfile: diff --git a/oss_active_record.gemspec b/oss_active_record.gemspec index ac88960..4533e83 100644 --- a/oss_active_record.gemspec +++ b/oss_active_record.gemspec @@ -9,10 +9,12 @@ Gem::Specification.new do |s| s.version = OssActiveRecord::VERSION s.authors = ["Ori Pekelman", "Emmanuel Keller"] s.email = ["ori@pekelman.com", "ekeller@open-search-server.com"] - s.homepage = "http://www.open-search-server.com" + s.homepage = "https://github.com/jaeksoft/oss_active_record" s.summary = "OpenSearchServer ActiveRecord integration" s.description = "OpenSearchServer ActiveRecord integration" + s.metadata = { "Issues" => "https://github.com/jaeksoft/oss_active_record/issues" } + s.files = Dir["{app,config,db,lib}/**/*", "MIT-LICENSE", "Rakefile", "README.rdoc"] s.test_files = Dir["test/**/*"] From b26e469074e18842422ac7b4168dbbb0093a1189 Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Thu, 5 Sep 2013 20:43:50 +0200 Subject: [PATCH 04/19] #1 fixed. The non required yield call has been removed. document_test has been updated. --- Gemfile | 2 +- Gemfile.lock | 14 +++++-------- lib/oss_active_record/searchable.rb | 1 - oss_active_record.gemspec | 1 + test/dummy/app/models/document.rb | 3 ++- test/dummy/test/fixtures/articles.yml | 8 ++++---- test/dummy/test/fixtures/documents.yml | 10 ++++----- test/dummy/test/models/article_test.rb | 27 +++++++++++++------------ test/dummy/test/models/document_test.rb | 12 ++++++++--- 9 files changed, 41 insertions(+), 37 deletions(-) diff --git a/Gemfile b/Gemfile index c4dc375..1a3014d 100644 --- a/Gemfile +++ b/Gemfile @@ -12,4 +12,4 @@ gemspec # To use debugger gem 'pry' -gem 'oss_rb', :git => "https://github.com/jaeksoft/oss_rb" +gem 'oss_rb' diff --git a/Gemfile.lock b/Gemfile.lock index 33dcb5f..37fa222 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,15 +1,8 @@ -GIT - remote: https://github.com/jaeksoft/oss_rb - revision: 227c80c3c56c945e0cd2a727ca7e373a7f3658c0 - specs: - oss_rb (0.1.0) - activesupport - rest-client - PATH remote: . specs: oss_active_record (0.1.0) + oss_rb (>= 0.1.0) rails (~> 4.0.0) GEM @@ -53,6 +46,9 @@ GEM mime-types (1.25) minitest (4.7.5) multi_json (1.7.9) + oss_rb (0.1.0) + activesupport + rest-client polyglot (0.3.3) pry (0.9.12.2) coderay (~> 1.0.5) @@ -102,6 +98,6 @@ PLATFORMS DEPENDENCIES oss_active_record! - oss_rb! + oss_rb pry sqlite3 diff --git a/lib/oss_active_record/searchable.rb b/lib/oss_active_record/searchable.rb index 81d861a..b97ee46 100644 --- a/lib/oss_active_record/searchable.rb +++ b/lib/oss_active_record/searchable.rb @@ -82,7 +82,6 @@ def create_schema_field!(field) end def method_missing(method, *args, &block) - yield unless block.nil? add_field args[0], method, block if @@field_types.include? method end diff --git a/oss_active_record.gemspec b/oss_active_record.gemspec index 4533e83..e02cb50 100644 --- a/oss_active_record.gemspec +++ b/oss_active_record.gemspec @@ -19,5 +19,6 @@ Gem::Specification.new do |s| s.test_files = Dir["test/**/*"] s.add_dependency "rails", "~> 4.0.0" + s.add_dependency 'oss_rb', '>= 0.1.0' s.add_development_dependency "sqlite3" end diff --git a/test/dummy/app/models/document.rb b/test/dummy/app/models/document.rb index e80f64e..b2ca37d 100644 --- a/test/dummy/app/models/document.rb +++ b/test/dummy/app/models/document.rb @@ -27,7 +27,8 @@ def room_id end class Document < ActiveRecord::Base - # belongs_to :current_revision + + belongs_to :current_revision def self.folder Folder.new end diff --git a/test/dummy/test/fixtures/articles.yml b/test/dummy/test/fixtures/articles.yml index 4c14eeb..d78a7a7 100644 --- a/test/dummy/test/fixtures/articles.yml +++ b/test/dummy/test/fixtures/articles.yml @@ -2,12 +2,12 @@ one: id: 1 - title: Talking - content: My beautiful article + title: Weather report + content: Sunny weather today updated_at: 2013-07-11 17:06:19 two: id: 2 - title: Weather report - content: Another news about weather + title: Active record rocks + content: Using OpenSearchServer with active record is simple updated_at: 2013-07-11 17:06:19 diff --git a/test/dummy/test/fixtures/documents.yml b/test/dummy/test/fixtures/documents.yml index eb6024b..13c0bdc 100644 --- a/test/dummy/test/fixtures/documents.yml +++ b/test/dummy/test/fixtures/documents.yml @@ -13,13 +13,13 @@ one: state: MyString two: - id: 1 - folder_id: 1 - room_id: 1 + id: 2 + folder_id: 2 + room_id: 2 name: MyString updated_at: 2013-07-11 17:06:19 uuid: 1 - user_id: 1 - file_size: 1 + user_id: 2 + file_size: 2 file_content_type: MyString state: MyString diff --git a/test/dummy/test/models/article_test.rb b/test/dummy/test/models/article_test.rb index 1c1af3a..32bb9fc 100644 --- a/test/dummy/test/models/article_test.rb +++ b/test/dummy/test/models/article_test.rb @@ -1,18 +1,19 @@ require_relative '../../../test_helper' class ArticleTest < ActiveSupport::TestCase - - test "Index" do - article = Article.new(id: 1, title: 'Weather report', content: 'Sunny weather today') - article.save - article = Article.new(id: 2, title: 'Active record rocks', content: 'Using OpenSearchServer with active record is simple') - article.save - - articles = Article.search do - fulltext 'weather' - end - assert articles.length == 1 - assert articles[0]['title'] == 'Weather report' + fixtures :articles + + setup do + Article.reindex! end - + + test "Index article" do + + result = Article.search do + fulltext 'weather' + end + assert result.length == 1 + assert result[0]['title'] == 'Weather report' + end + end diff --git a/test/dummy/test/models/document_test.rb b/test/dummy/test/models/document_test.rb index e08f62a..3b21fda 100644 --- a/test/dummy/test/models/document_test.rb +++ b/test/dummy/test/models/document_test.rb @@ -1,19 +1,25 @@ require_relative '../../../test_helper' class DocumentTest < ActiveSupport::TestCase + fixtures :documents + + setup do + Document.reindex! + end + test "the truth" do assert true end - test "Search" do - @documents = Document.search(:include => { :current_revision => :user }) do + test "Search document" do + result = Document.search(:include => { :current_revision => :user }) do fulltext 'test' with :room_id, 1 paginate page: 1, per_page: 10 order_by :score, :desc order_by :id end - @documents.each { |document| puts document } + result.each { |document| puts document } assert true end end \ No newline at end of file From d605833c3c187db3de2294999e68df85e333d445 Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Fri, 6 Sep 2013 10:58:21 +0200 Subject: [PATCH 05/19] Fixes #2, integer and decimal types use now respectively the IndexAnalyzer and the DecimalAnalyzer. Filtering (with, without) and sorting (order_by) use now the right field name. --- lib/oss_active_record/search_request.rb | 24 ++++++++--------- lib/oss_active_record/searchable.rb | 26 ++++++++++++++++--- .../config/initializers/oss_active_record.rb | 2 ++ test/dummy/test/fixtures/documents.yml | 8 +++--- test/dummy/test/models/document_test.rb | 7 ++--- 5 files changed, 45 insertions(+), 22 deletions(-) diff --git a/lib/oss_active_record/search_request.rb b/lib/oss_active_record/search_request.rb index a163019..6403d7b 100644 --- a/lib/oss_active_record/search_request.rb +++ b/lib/oss_active_record/search_request.rb @@ -1,12 +1,10 @@ module OssActiveRecord class SearchRequest - def initialize(index, text_fields, sortable_fields) - @index = index + def initialize(searchable) @params = {'start' => 0, 'rows' => 10} - @filters = []; - @text_fields = text_fields; - @sortable_fields = sortable_fields; - @order_by = {}; + @filters = [] + @searchable = searchable + @order_by = {} end def fulltext(keywords, &block) @@ -14,15 +12,17 @@ def fulltext(keywords, &block) end def filter(field, value, negative) - @filters << {"type"=> "QueryFilter", "negative"=> negative, "query"=> "#{field}:#{value}"} + @filters << {"type"=> "QueryFilter", "negative"=> negative, "query"=> "#{field}:(#{value})"} end def with(field, value) - filter field, value, false + index_field = @searchable.find_field_name(field) + filter index_field, value, false unless index_field.nil? end def without(field, value) - filter field, value, true + index_field = @searchable.find_field_name(field) + filter index_field, value, true unless index_field.nil? end def returns(fields) @@ -37,7 +37,7 @@ def paginate(params) end def order_by(field = nil, direction = :asc) - field = field == :score ? 'score' :@sortable_fields[field] + field = field == :score ? 'score' :@searchable.find_sortable_name(field) @order_by[field.to_s] = direction.to_s.upcase unless field.nil? end @@ -45,7 +45,7 @@ def execute(&block) self.instance_eval(&block) unless block.nil? @params['filters'] = @filters unless @filters.length == 0 fields = [] - @text_fields.each do |key, value| + @searchable._text_fields.each do |key, value| fields<<{ "field"=> value,"phrase"=> true,"boost"=> 1.0} end @params['searchFields'] = fields unless fields.length == 0 @@ -54,7 +54,7 @@ def execute(&block) sorts<<{ "field"=> key,"direction"=> value} end @params['sorts'] = sorts unless sorts.length == 0 - return @index.search_field(@params) + return @searchable.oss_index.search_field(@params) end end diff --git a/lib/oss_active_record/searchable.rb b/lib/oss_active_record/searchable.rb index b97ee46..d3123bc 100644 --- a/lib/oss_active_record/searchable.rb +++ b/lib/oss_active_record/searchable.rb @@ -17,12 +17,25 @@ module ClassMethods @@_field_id = nil @@_text_fields = {} @@_sortable_fields = {} + @@_all_fields = {} @@index = nil + def _text_fields + @@_text_fields + end + def _fields @@_fields end + def find_sortable_name(field_name) + field == :score ? 'score' : @@_sortable_fields[field_name] unless field_name.nil? + end + + def find_field_name(field_name) + @@_all_fields[field_name] unless field_name.nil? + end + def searchable(options = {}, &block) yield unless options[:auto_index] == false @@ -33,7 +46,10 @@ def searchable(options = {}, &block) def oss_index if @@index.nil? @@index_name ||= self.name.downcase - @@index = Oss::Index.new(@@index_name, Rails.configuration.open_search_server_url) + @@index = Oss::Index.new(@@index_name, + Rails.configuration.open_search_server_url, + Rails.configuration.open_search_server_login, + Rails.configuration.open_search_server_apikey) create_schema! end @@index @@ -63,7 +79,10 @@ def reindex! end def create_schema_field!(field) - analyzers = { :text => 'StandardAnalyzer', :integer => 'DecimalAnalyzer'} + analyzers = { + :text => 'StandardAnalyzer', + :integer => 'IntegerAnalyzer', + :decimal => 'DecimalAnalyzer'} analyzer = analyzers[field[:type]] if field[:name] != :id termVectors = { :text => 'POSITIONS_OFFSETS'} termVector = termVectors[field[:type]] || 'NO' @@ -77,6 +96,7 @@ def create_schema_field!(field) } @@_text_fields[field[:name]] = name if field[:type] == :text @@_sortable_fields[field[:name]] = name unless field[:type] == :text + @@_all_fields[field[:name]] = name self.oss_index.set_field(params) self.oss_index.set_field_default_unique(name, name) if field[:name] == :id end @@ -86,7 +106,7 @@ def method_missing(method, *args, &block) end def search(*args, &block) - searchRequest = SearchRequest.new(self.oss_index, @@_text_fields, @@_sortable_fields) + searchRequest = SearchRequest.new(self) searchRequest.returns @@_fields.map {|f|"#{f[:name]}|#{f[:type]}"} active_record_from_result searchRequest.execute(&block) end diff --git a/test/dummy/config/initializers/oss_active_record.rb b/test/dummy/config/initializers/oss_active_record.rb index 5d18541..a7c92de 100644 --- a/test/dummy/config/initializers/oss_active_record.rb +++ b/test/dummy/config/initializers/oss_active_record.rb @@ -1 +1,3 @@ Rails.configuration.open_search_server_url = "http://localhost:8080" +Rails.configuration.open_search_server_login = nil +Rails.configuration.open_search_server_apikey = nil diff --git a/test/dummy/test/fixtures/documents.yml b/test/dummy/test/fixtures/documents.yml index 13c0bdc..9f2dc1f 100644 --- a/test/dummy/test/fixtures/documents.yml +++ b/test/dummy/test/fixtures/documents.yml @@ -4,22 +4,22 @@ one: id: 1 folder_id: 1 room_id: 1 - name: MyString + name: my wonderful document updated_at: 2013-07-11 17:06:19 uuid: 1 user_id: 1 file_size: 1 - file_content_type: MyString + file_content_type: application/pdf state: MyString two: id: 2 folder_id: 2 room_id: 2 - name: MyString + name: another amazing document updated_at: 2013-07-11 17:06:19 uuid: 1 user_id: 2 file_size: 2 - file_content_type: MyString + file_content_type: application/xml state: MyString diff --git a/test/dummy/test/models/document_test.rb b/test/dummy/test/models/document_test.rb index 3b21fda..e2fd295 100644 --- a/test/dummy/test/models/document_test.rb +++ b/test/dummy/test/models/document_test.rb @@ -13,13 +13,14 @@ class DocumentTest < ActiveSupport::TestCase test "Search document" do result = Document.search(:include => { :current_revision => :user }) do - fulltext 'test' - with :room_id, 1 + fulltext 'document' + with(:room_id, 42) + without(:room_id, 43) paginate page: 1, per_page: 10 order_by :score, :desc order_by :id end result.each { |document| puts document } - assert true + assert result.length == 2 end end \ No newline at end of file From 0c8647668547aed5e2dcc08853cb76a47a034bf0 Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Fri, 6 Sep 2013 13:08:12 +0200 Subject: [PATCH 06/19] #3 fix: the order_by clause works now. The ordering was corrupted when documents were extracted from the ActiveRecord instance. Two related test cases have been added. --- lib/oss_active_record/search_request.rb | 4 +- lib/oss_active_record/searchable.rb | 15 +++----- test/dummy/app/models/article.rb | 1 + .../migrate/20130901161100_create_articles.rb | 1 + test/dummy/db/schema.rb | 1 + test/dummy/test/fixtures/articles.yml | 9 +++++ test/dummy/test/models/article_test.rb | 37 +++++++++++++++++-- test/dummy/test/models/document_test.rb | 3 +- 8 files changed, 54 insertions(+), 17 deletions(-) diff --git a/lib/oss_active_record/search_request.rb b/lib/oss_active_record/search_request.rb index 6403d7b..88de013 100644 --- a/lib/oss_active_record/search_request.rb +++ b/lib/oss_active_record/search_request.rb @@ -37,8 +37,8 @@ def paginate(params) end def order_by(field = nil, direction = :asc) - field = field == :score ? 'score' :@searchable.find_sortable_name(field) - @order_by[field.to_s] = direction.to_s.upcase unless field.nil? + index_field = field == :score ? 'score' :@searchable.find_sortable_name(field) + @order_by[index_field] = direction.to_s.upcase unless index_field.nil? end def execute(&block) diff --git a/lib/oss_active_record/searchable.rb b/lib/oss_active_record/searchable.rb index d3123bc..a182ec6 100644 --- a/lib/oss_active_record/searchable.rb +++ b/lib/oss_active_record/searchable.rb @@ -108,22 +108,19 @@ def method_missing(method, *args, &block) def search(*args, &block) searchRequest = SearchRequest.new(self) searchRequest.returns @@_fields.map {|f|"#{f[:name]}|#{f[:type]}"} - active_record_from_result searchRequest.execute(&block) + find_results searchRequest.execute(&block) end - def get_ids_from_results(search_result) - ids = [] + def find_results(search_result) id_field_name = "#{@@_field_id[:name]}|#{@@_field_id[:type]}" + results = [] search_result['documents'].each do |document| document['fields'].each do |field| - ids << field['values'].map {|f|f.to_i}.uniq if field['fieldName'] == id_field_name + id = field['values'].map {|f|f.to_i}.uniq if field['fieldName'] == id_field_name + results<= article[:category_id], 'Order is wrong') unless cat_id.nil? + cat_id = article[:category_id] + end + end end diff --git a/test/dummy/test/models/document_test.rb b/test/dummy/test/models/document_test.rb index e2fd295..7b5fa60 100644 --- a/test/dummy/test/models/document_test.rb +++ b/test/dummy/test/models/document_test.rb @@ -20,7 +20,6 @@ class DocumentTest < ActiveSupport::TestCase order_by :score, :desc order_by :id end - result.each { |document| puts document } - assert result.length == 2 + assert result.length == 2, 'The number of document returned is wrong' end end \ No newline at end of file From 7e9882df43ae7fdcdf76c924dadf255472946632 Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Fri, 6 Sep 2013 13:19:07 +0200 Subject: [PATCH 07/19] Documentation update. New gem version (0.1.1). --- README.md | 7 ++++--- oss_active_record.gemspec | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6751409..ef1e599 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,6 @@ Add this line to your application's Gemfile: gem 'oss_rb' gem 'oss_active_record' - And then execute: $ bundle @@ -27,9 +26,11 @@ Or install it yourself as: ## Configuration -Add `config/initializers/oss_active_record.rb` with the url of opens-search-server +Add `config/initializers/oss_active_record.rb` with the URL, login and api key of opens-search-server - Rails.configuration.open_search_server_url = "http://localhost:8080" + Rails.configuration.open_search_server_url = 'http://localhost:8080' + Rails.configuration.open_search_server_login = 'my_login' + Rails.configuration.open_search_server_apikey = 'my_api_key' ## Usage diff --git a/oss_active_record.gemspec b/oss_active_record.gemspec index e02cb50..22d3c90 100644 --- a/oss_active_record.gemspec +++ b/oss_active_record.gemspec @@ -19,6 +19,6 @@ Gem::Specification.new do |s| s.test_files = Dir["test/**/*"] s.add_dependency "rails", "~> 4.0.0" - s.add_dependency 'oss_rb', '>= 0.1.0' + s.add_dependency 'oss_rb', '>= 0.1.1' s.add_development_dependency "sqlite3" end From 1fe8203a14905eba310c8a7a17ec86efdb9d79d3 Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Fri, 6 Sep 2013 13:38:38 +0200 Subject: [PATCH 08/19] New release 0.1.1 --- .gitignore | 1 + lib/oss_active_record/version.rb | 2 +- oss_active_record.gemspec | 23 ++++++++++++----------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 40f2060..7c7db89 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ test/dummy/.sass-cache /.project /.DS_Store /.buildpath +/oss_active_record-*.gem diff --git a/lib/oss_active_record/version.rb b/lib/oss_active_record/version.rb index 73f808d..da732da 100644 --- a/lib/oss_active_record/version.rb +++ b/lib/oss_active_record/version.rb @@ -1,3 +1,3 @@ module OssActiveRecord - VERSION = "0.1.0" + VERSION = "0.1.1" end diff --git a/oss_active_record.gemspec b/oss_active_record.gemspec index 22d3c90..e7a5529 100644 --- a/oss_active_record.gemspec +++ b/oss_active_record.gemspec @@ -5,20 +5,21 @@ require "oss_active_record/version" # Describe your gem and declare its dependencies: Gem::Specification.new do |s| - s.name = "oss_active_record" + s.name = 'oss_active_record' s.version = OssActiveRecord::VERSION - s.authors = ["Ori Pekelman", "Emmanuel Keller"] - s.email = ["ori@pekelman.com", "ekeller@open-search-server.com"] - s.homepage = "https://github.com/jaeksoft/oss_active_record" - s.summary = "OpenSearchServer ActiveRecord integration" - s.description = "OpenSearchServer ActiveRecord integration" + s.authors = ['Ori Pekelman', 'Emmanuel Keller'] + s.email = ['ori@pekelman.com', 'ekeller@open-search-server.com'] + s.homepage = 'https://github.com/jaeksoft/oss_active_record' + s.summary = 'OpenSearchServer ActiveRecord integration' + s.description = 'oss_active_record is a library providing an all-ruby API for the OpenSearchServer search engine.' + s.license = 'MIT' - s.metadata = { "Issues" => "https://github.com/jaeksoft/oss_active_record/issues" } + s.metadata = { 'Issues' => 'https://github.com/jaeksoft/oss_active_record/issues' } - s.files = Dir["{app,config,db,lib}/**/*", "MIT-LICENSE", "Rakefile", "README.rdoc"] + s.files = Dir["{app,config,db,lib}/**/*", 'MIT-LICENSE', 'Rakefile', 'README.rdoc'] s.test_files = Dir["test/**/*"] - s.add_dependency "rails", "~> 4.0.0" - s.add_dependency 'oss_rb', '>= 0.1.1' - s.add_development_dependency "sqlite3" + s.add_dependency 'rails', '~> 4.0.0' + s.add_dependency 'oss_rb', '>= 0.1.0' + s.add_development_dependency 'sqlite3' end From 6a16c5e3f6b5e2f697886eed2d1942d73587c347 Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Thu, 12 Sep 2013 17:48:25 +0200 Subject: [PATCH 09/19] Fixes #1. All the fields were merged in the same array structure. The tests was correct when each model item was checked separately. Now the test checks the full model. --- Gemfile.lock | 2 +- lib/oss_active_record.rb | 2 + lib/oss_active_record/index_instance.rb | 90 ++++++++++++++++++ lib/oss_active_record/search_request.rb | 14 +-- lib/oss_active_record/searchable.rb | 116 ++++++------------------ lib/oss_active_record/version.rb | 2 +- test/dummy/test/models/article_test.rb | 1 - 7 files changed, 127 insertions(+), 100 deletions(-) create mode 100644 lib/oss_active_record/index_instance.rb diff --git a/Gemfile.lock b/Gemfile.lock index 37fa222..0c51252 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - oss_active_record (0.1.0) + oss_active_record (0.1.2) oss_rb (>= 0.1.0) rails (~> 4.0.0) diff --git a/lib/oss_active_record.rb b/lib/oss_active_record.rb index 6d77873..2c35677 100644 --- a/lib/oss_active_record.rb +++ b/lib/oss_active_record.rb @@ -1,4 +1,6 @@ require "oss_active_record/searchable" require "oss_active_record/search_request" +require "oss_active_record/index_instance" + module OssActiveRecord end diff --git a/lib/oss_active_record/index_instance.rb b/lib/oss_active_record/index_instance.rb new file mode 100644 index 0000000..c01f48e --- /dev/null +++ b/lib/oss_active_record/index_instance.rb @@ -0,0 +1,90 @@ +module OssActiveRecord + class IndexInstance + + @@_analyzers = { + :text => 'StandardAnalyzer', + :integer => 'IntegerAnalyzer', + :decimal => 'DecimalAnalyzer'} + + def initialize(index_name) + @_index_name ||= index_name + @_fields= [] + @_field_id = nil + @_text_fields = {} + @_sortable_fields = {} + @_all_fields = {} + @_index = nil + end + + def text_fields + @_text_fields + end + + def fields + @_fields + end + + def field_id + @_field_id + end + + def oss_index + if @_index.nil? + @_index = Oss::Index.new(@_index_name, + Rails.configuration.open_search_server_url, + Rails.configuration.open_search_server_login, + Rails.configuration.open_search_server_apikey) + create_schema! + end + @_index + end + + def find_sortable_name(field_name) + field_name == :score ? 'score' : @_sortable_fields[field_name] unless field_name.nil? + end + + def find_field_name(field_name) + @_all_fields[field_name] unless field_name.nil? + end + + def add_field(name, type, block=nil) + @_fields<<{:name => name, :type => type,:block => block} + end + + def create_schema! + @_index.create('EMPTY_INDEX') unless @_index.list.include? @_index_name + @_field_id = @_fields.detect {|f| f[:name] == :id } + @_field_id = {:name => 'id', :type => 'integer',:block => nil} if @_field_id.nil? + @_fields << @_field_id + @_fields.each do |field| + create_schema_field!(field) + end + end + + def create_schema_field!(field) + analyzer = @@_analyzers[field[:type]] if field[:name] != :id + termVectors = { :text => 'POSITIONS_OFFSETS'} + termVector = termVectors[field[:type]] || 'NO' + name = "#{field[:name]}|#{field[:type]}" + params = { + 'name' => name, + 'analyzer' => analyzer, + 'stored' => 'NO', + 'indexed' => 'YES', + 'termVector' => termVector + } + @_text_fields[field[:name]] = name if field[:type] == :text + @_sortable_fields[field[:name]] = name unless field[:type] == :text + @_all_fields[field[:name]] = name + @_index.set_field(params) + @_index.set_field_default_unique(name, name) if field[:name] == :id + end + + def index(docs) + @_index.documents << docs + @_index.index! + end + + end + +end \ No newline at end of file diff --git a/lib/oss_active_record/search_request.rb b/lib/oss_active_record/search_request.rb index 88de013..36a6584 100644 --- a/lib/oss_active_record/search_request.rb +++ b/lib/oss_active_record/search_request.rb @@ -1,9 +1,9 @@ module OssActiveRecord class SearchRequest - def initialize(searchable) + def initialize(index_instance) @params = {'start' => 0, 'rows' => 10} @filters = [] - @searchable = searchable + @index_instance = index_instance @order_by = {} end @@ -16,12 +16,12 @@ def filter(field, value, negative) end def with(field, value) - index_field = @searchable.find_field_name(field) + index_field = @index_instance.find_field_name(field) filter index_field, value, false unless index_field.nil? end def without(field, value) - index_field = @searchable.find_field_name(field) + index_field = @index_instance.find_field_name(field) filter index_field, value, true unless index_field.nil? end @@ -37,7 +37,7 @@ def paginate(params) end def order_by(field = nil, direction = :asc) - index_field = field == :score ? 'score' :@searchable.find_sortable_name(field) + index_field = field == :score ? 'score' :@index_instance.find_sortable_name(field) @order_by[index_field] = direction.to_s.upcase unless index_field.nil? end @@ -45,7 +45,7 @@ def execute(&block) self.instance_eval(&block) unless block.nil? @params['filters'] = @filters unless @filters.length == 0 fields = [] - @searchable._text_fields.each do |key, value| + @index_instance.text_fields.each do |key, value| fields<<{ "field"=> value,"phrase"=> true,"boost"=> 1.0} end @params['searchFields'] = fields unless fields.length == 0 @@ -54,7 +54,7 @@ def execute(&block) sorts<<{ "field"=> key,"direction"=> value} end @params['sorts'] = sorts unless sorts.length == 0 - return @searchable.oss_index.search_field(@params) + return @index_instance.oss_index.search_field(@params) end end diff --git a/lib/oss_active_record/searchable.rb b/lib/oss_active_record/searchable.rb index a182ec6..a46b442 100644 --- a/lib/oss_active_record/searchable.rb +++ b/lib/oss_active_record/searchable.rb @@ -1,40 +1,9 @@ module OssActiveRecord module Searchable extend ActiveSupport::Concern - def index - doc = self.to_indexable - oss_doc = Oss::Document.new - doc.each do |name,value| - oss_doc.fields << Oss::Field.new(name, value) - end - self.class.oss_index.documents << oss_doc - self.class.oss_index.index! - end - module ClassMethods @@field_types= [:integer, :text, :string, :time] #supported field types - @@_fields=[] - @@_field_id = nil - @@_text_fields = {} - @@_sortable_fields = {} - @@_all_fields = {} - @@index = nil - - def _text_fields - @@_text_fields - end - - def _fields - @@_fields - end - - def find_sortable_name(field_name) - field == :score ? 'score' : @@_sortable_fields[field_name] unless field_name.nil? - end - - def find_field_name(field_name) - @@_all_fields[field_name] unless field_name.nil? - end + @@index_instances = {} def searchable(options = {}, &block) yield @@ -43,76 +12,35 @@ def searchable(options = {}, &block) end end - def oss_index - if @@index.nil? - @@index_name ||= self.name.downcase - @@index = Oss::Index.new(@@index_name, - Rails.configuration.open_search_server_url, - Rails.configuration.open_search_server_login, - Rails.configuration.open_search_server_apikey) - create_schema! - end - @@index - end - - def add_field(name, type, block=nil) - @@_fields<<{:name => name, :type => type,:block => block} - end - - def create_schema! - @@index.create('EMPTY_INDEX') unless @@index.list.include? @@index_name - @@_field_id = @@_fields.detect {|f| f[:name] == :id } - @@_field_id = {:name => 'id', :type => 'integer',:block => nil} if @@_field_id.nil? - @@_fields << @@_field_id - @@_fields.each do |field| - create_schema_field!(field) + def index_instance + idx_inst = @@index_instances[self.name.downcase] + if idx_inst.nil? + idx_inst = IndexInstance.new(self.name.downcase) + @@index_instances[self.name.downcase] = idx_inst end + idx_inst end def reindex! - self.oss_index.delete! - self.create_schema! - + index_instance.oss_index.delete! + index_instance.create_schema! self.all.find_in_batches do |group| - group.each { |doc| doc.index } + group.each { |doc| doc.index(index_instance) } end end - def create_schema_field!(field) - analyzers = { - :text => 'StandardAnalyzer', - :integer => 'IntegerAnalyzer', - :decimal => 'DecimalAnalyzer'} - analyzer = analyzers[field[:type]] if field[:name] != :id - termVectors = { :text => 'POSITIONS_OFFSETS'} - termVector = termVectors[field[:type]] || 'NO' - name = "#{field[:name]}|#{field[:type]}" - params = { - 'name' => name, - 'analyzer' => analyzer, - 'stored' => 'NO', - 'indexed' => 'YES', - 'termVector' => termVector - } - @@_text_fields[field[:name]] = name if field[:type] == :text - @@_sortable_fields[field[:name]] = name unless field[:type] == :text - @@_all_fields[field[:name]] = name - self.oss_index.set_field(params) - self.oss_index.set_field_default_unique(name, name) if field[:name] == :id - end - def method_missing(method, *args, &block) - add_field args[0], method, block if @@field_types.include? method + index_instance.add_field(args[0], method, block) if @@field_types.include? method end def search(*args, &block) - searchRequest = SearchRequest.new(self) - searchRequest.returns @@_fields.map {|f|"#{f[:name]}|#{f[:type]}"} - find_results searchRequest.execute(&block) + searchRequest = SearchRequest.new(index_instance) + searchRequest.returns index_instance.fields.map {|f|"#{f[:name]}|#{f[:type]}"} + find_results(searchRequest.execute(&block), index_instance.field_id) end - def find_results(search_result) - id_field_name = "#{@@_field_id[:name]}|#{@@_field_id[:type]}" + def find_results(search_result, field_id) + id_field_name = "#{field_id[:name]}|#{field_id[:type]}" results = [] search_result['documents'].each do |document| document['fields'].each do |field| @@ -122,12 +50,20 @@ def find_results(search_result) end return results end + end + def index(index_instance) + doc = self.to_indexable(index_instance.fields) + oss_doc = Oss::Document.new + doc.each do |name,value| + oss_doc.fields << Oss::Field.new(name, value) + end + index_instance.index(oss_doc) end - def to_indexable + def to_indexable(fields) doc={} - self.class._fields.each do |field| + fields.each do |field| if field[:block].nil? val = self.send(field[:name].to_sym) else diff --git a/lib/oss_active_record/version.rb b/lib/oss_active_record/version.rb index da732da..a97f056 100644 --- a/lib/oss_active_record/version.rb +++ b/lib/oss_active_record/version.rb @@ -1,3 +1,3 @@ module OssActiveRecord - VERSION = "0.1.1" + VERSION = "0.1.2" end diff --git a/test/dummy/test/models/article_test.rb b/test/dummy/test/models/article_test.rb index 036c3a5..2b5ac75 100644 --- a/test/dummy/test/models/article_test.rb +++ b/test/dummy/test/models/article_test.rb @@ -13,7 +13,6 @@ class ArticleTest < ActiveSupport::TestCase fulltext 'weather' end - puts results.inspect assert results.length == 1, 'The number of result is wrong, should be one' assert results[0][:title] == 'Weather report', 'The returned title is wrong' end From 4a2d95f70942b9ca80ccea6f5a9d1c6451ab4f61 Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Mon, 7 Oct 2013 11:54:25 +0200 Subject: [PATCH 10/19] Workaroung on #4 and new suggestion support #5 --- Gemfile.lock | 14 ++++++------ lib/oss_active_record/index_instance.rb | 3 ++- lib/oss_active_record/search_request.rb | 21 +++++++++++++---- lib/oss_active_record/searchable.rb | 5 +++-- lib/oss_active_record/version.rb | 2 +- oss_active_record.gemspec | 2 +- test/dummy/app/models/document.rb | 1 + test/dummy/test/models/document_test.rb | 30 ++++++++++++++++--------- 8 files changed, 52 insertions(+), 26 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0c51252..3a3947c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,8 +1,8 @@ PATH remote: . specs: - oss_active_record (0.1.2) - oss_rb (>= 0.1.0) + oss_active_record (0.2.0) + oss_rb (>= 0.1.2) rails (~> 4.0.0) GEM @@ -33,7 +33,7 @@ GEM thread_safe (~> 0.1) tzinfo (~> 0.3.37) arel (4.0.0) - atomic (1.1.13) + atomic (1.1.14) builder (3.1.4) coderay (1.0.9) erubis (2.7.0) @@ -45,8 +45,8 @@ GEM method_source (0.8.2) mime-types (1.25) minitest (4.7.5) - multi_json (1.7.9) - oss_rb (0.1.0) + multi_json (1.8.1) + oss_rb (0.1.2) activesupport rest-client polyglot (0.3.3) @@ -83,9 +83,9 @@ GEM actionpack (>= 3.0) activesupport (>= 3.0) sprockets (~> 2.8) - sqlite3 (1.3.7) + sqlite3 (1.3.8) thor (0.18.1) - thread_safe (0.1.2) + thread_safe (0.1.3) atomic tilt (1.4.1) treetop (1.4.15) diff --git a/lib/oss_active_record/index_instance.rb b/lib/oss_active_record/index_instance.rb index c01f48e..41bf108 100644 --- a/lib/oss_active_record/index_instance.rb +++ b/lib/oss_active_record/index_instance.rb @@ -4,7 +4,8 @@ class IndexInstance @@_analyzers = { :text => 'StandardAnalyzer', :integer => 'IntegerAnalyzer', - :decimal => 'DecimalAnalyzer'} + :decimal => 'DecimalAnalyzer', + :suggestion => 'SuggestionAnalyzer'} def initialize(index_name) @_index_name ||= index_name diff --git a/lib/oss_active_record/search_request.rb b/lib/oss_active_record/search_request.rb index 36a6584..f141731 100644 --- a/lib/oss_active_record/search_request.rb +++ b/lib/oss_active_record/search_request.rb @@ -5,10 +5,24 @@ def initialize(index_instance) @filters = [] @index_instance = index_instance @order_by = {} + @fields = [] end def fulltext(keywords, &block) @params['query'] = keywords + self.instance_eval(&block) unless block.nil? + end + + def field(field, phrase = false, boost = 1.0) + @fields<<{ "field"=> @index_instance.find_field_name(field), "phrase" => phrase, "boost" => boost} if field.is_a?Symbol + @fields<<{ "field"=> field, "phrase" => phrase, "boost" => boost} if field.is_a?String + end + + def fields(fields) + fields.each do |key, value| + field key, false, value + end if fields.is_a?Hash + field fields unless fields.is_a?Hash end def filter(field, value, negative) @@ -44,11 +58,10 @@ def order_by(field = nil, direction = :asc) def execute(&block) self.instance_eval(&block) unless block.nil? @params['filters'] = @filters unless @filters.length == 0 - fields = [] @index_instance.text_fields.each do |key, value| - fields<<{ "field"=> value,"phrase"=> true,"boost"=> 1.0} - end - @params['searchFields'] = fields unless fields.length == 0 + field value, true + end unless @fields.any? + @params['searchFields'] = @fields unless @fields.length == 0 sorts = [] @order_by.each do |key, value| sorts<<{ "field"=> key,"direction"=> value} diff --git a/lib/oss_active_record/searchable.rb b/lib/oss_active_record/searchable.rb index a46b442..abe62d1 100644 --- a/lib/oss_active_record/searchable.rb +++ b/lib/oss_active_record/searchable.rb @@ -2,7 +2,7 @@ module OssActiveRecord module Searchable extend ActiveSupport::Concern module ClassMethods - @@field_types= [:integer, :text, :string, :time] #supported field types + @@field_types= [:integer, :text, :string, :time, :suggestion] #supported field types @@index_instances = {} def searchable(options = {}, &block) @@ -30,7 +30,8 @@ def reindex! end def method_missing(method, *args, &block) - index_instance.add_field(args[0], method, block) if @@field_types.include? method + return index_instance.add_field(args[0], method, block) if @@field_types.include? method + super end def search(*args, &block) diff --git a/lib/oss_active_record/version.rb b/lib/oss_active_record/version.rb index a97f056..9236ef0 100644 --- a/lib/oss_active_record/version.rb +++ b/lib/oss_active_record/version.rb @@ -1,3 +1,3 @@ module OssActiveRecord - VERSION = "0.1.2" + VERSION = "0.2.0" end diff --git a/oss_active_record.gemspec b/oss_active_record.gemspec index e7a5529..a292016 100644 --- a/oss_active_record.gemspec +++ b/oss_active_record.gemspec @@ -20,6 +20,6 @@ Gem::Specification.new do |s| s.test_files = Dir["test/**/*"] s.add_dependency 'rails', '~> 4.0.0' - s.add_dependency 'oss_rb', '>= 0.1.0' + s.add_dependency 'oss_rb', '>= 0.1.2' s.add_development_dependency 'sqlite3' end diff --git a/test/dummy/app/models/document.rb b/test/dummy/app/models/document.rb index b2ca37d..a22d130 100644 --- a/test/dummy/app/models/document.rb +++ b/test/dummy/app/models/document.rb @@ -44,6 +44,7 @@ def self.current_revision text :name # fulltext string :name # order_by + suggestion :name # Suggestion (autocompletion) time :updated_at integer :uuid do current_revision.uuid end diff --git a/test/dummy/test/models/document_test.rb b/test/dummy/test/models/document_test.rb index 7b5fa60..5f1bd2c 100644 --- a/test/dummy/test/models/document_test.rb +++ b/test/dummy/test/models/document_test.rb @@ -2,7 +2,7 @@ class DocumentTest < ActiveSupport::TestCase fixtures :documents - + setup do Document.reindex! end @@ -10,16 +10,26 @@ class DocumentTest < ActiveSupport::TestCase test "the truth" do assert true end - + test "Search document" do result = Document.search(:include => { :current_revision => :user }) do - fulltext 'document' - with(:room_id, 42) - without(:room_id, 43) - paginate page: 1, per_page: 10 - order_by :score, :desc - order_by :id - end - assert result.length == 2, 'The number of document returned is wrong' + fulltext 'document' + with(:room_id, 42) + without(:room_id, 43) + paginate page: 1, per_page: 10 + order_by :score, :desc + order_by :id + end + assert result.length == 2, 'The number of document returned is wrong' end + + test "Suggestion" do + result = Document.search do + fulltext 'An' do + fields('name|suggestion') + end + end + assert result.length == 1, 'The number of returned suggestion is wrong' + end + end \ No newline at end of file From f9684b34356852e665859b4781718017515af1ec Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 7 Oct 2013 20:39:29 +0200 Subject: [PATCH 11/19] Fixes & optimizations to Searchable. - Loading all records from the database at once + manual reordering to match OSS result. - Added :include option to #oss_search & #find_results to eager load associations. - Using after_commit instead of after_save for indexing documents. Other minor things: - Renamed #search to #oss_search (with alias) for simpler custom searches in models. - Overall ruby syntax & functional code refactoring. --- lib/oss_active_record/searchable.rb | 88 +++++++++++++------------ test/dummy/app/models/document.rb | 9 ++- test/dummy/test/models/document_test.rb | 8 +-- 3 files changed, 52 insertions(+), 53 deletions(-) diff --git a/lib/oss_active_record/searchable.rb b/lib/oss_active_record/searchable.rb index abe62d1..714c413 100644 --- a/lib/oss_active_record/searchable.rb +++ b/lib/oss_active_record/searchable.rb @@ -1,81 +1,85 @@ +require 'thread' + module OssActiveRecord module Searchable extend ActiveSupport::Concern + module ClassMethods - @@field_types= [:integer, :text, :string, :time, :suggestion] #supported field types - @@index_instances = {} + @@oss_field_types = [:integer, :text, :string, :time, :suggestion] # supported field types + @@oss_mutex = Mutex.new def searchable(options = {}, &block) yield unless options[:auto_index] == false - after_save :index + after_commit :index end end def index_instance - idx_inst = @@index_instances[self.name.downcase] - if idx_inst.nil? - idx_inst = IndexInstance.new(self.name.downcase) - @@index_instances[self.name.downcase] = idx_inst + @@oss_mutex.synchronize do + @index_instance ||= IndexInstance.new(self.name.downcase) end - idx_inst end def reindex! index_instance.oss_index.delete! index_instance.create_schema! - self.all.find_in_batches do |group| - group.each { |doc| doc.index(index_instance) } - end + self.all.find_in_batches { |group| group.each(&:index) } end def method_missing(method, *args, &block) - return index_instance.add_field(args[0], method, block) if @@field_types.include? method + return index_instance.add_field(args[0], method, block) if @@oss_field_types.include? method super end - def search(*args, &block) - searchRequest = SearchRequest.new(index_instance) - searchRequest.returns index_instance.fields.map {|f|"#{f[:name]}|#{f[:type]}"} - find_results(searchRequest.execute(&block), index_instance.field_id) + def oss_search(*args, &block) + options = args.extract_options! + search_request = SearchRequest.new(index_instance) + search_request.returns index_instance.fields.map { |f| "#{f[:name]}|#{f[:type]}" } + find_results(search_request.execute(&block), index_instance.field_id, options) end + alias_method :search, :oss_search - def find_results(search_result, field_id) + # FIXME: load all records in a single request then rebuild an ordered array. + def find_results(search_result, field_id, options) id_field_name = "#{field_id[:name]}|#{field_id[:type]}" - results = [] - search_result['documents'].each do |document| - document['fields'].each do |field| - id = field['values'].map {|f|f.to_i}.uniq if field['fieldName'] == id_field_name - results< { :current_revision => :user }) do + result = Document.search do fulltext 'document' with(:room_id, 42) without(:room_id, 43) @@ -32,4 +28,4 @@ class DocumentTest < ActiveSupport::TestCase assert result.length == 1, 'The number of returned suggestion is wrong' end -end \ No newline at end of file +end From 21514f994d35e6166008137887c4561a8355e75c Mon Sep 17 00:00:00 2001 From: Julien Portalier Date: Mon, 7 Oct 2013 20:51:25 +0200 Subject: [PATCH 12/19] removed fixme. --- lib/oss_active_record/searchable.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/oss_active_record/searchable.rb b/lib/oss_active_record/searchable.rb index 714c413..9d41db7 100644 --- a/lib/oss_active_record/searchable.rb +++ b/lib/oss_active_record/searchable.rb @@ -40,7 +40,6 @@ def oss_search(*args, &block) end alias_method :search, :oss_search - # FIXME: load all records in a single request then rebuild an ordered array. def find_results(search_result, field_id, options) id_field_name = "#{field_id[:name]}|#{field_id[:type]}" From e7fd2391e79d4357b7b7add8f1569780a761f285 Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Wed, 23 Oct 2013 09:21:15 +0200 Subject: [PATCH 13/19] Design improvements (thx Julien !), fixes authentication issue on dependecy (oss_rb) --- Gemfile.lock | 12 ++++++------ lib/oss_active_record/searchable.rb | 15 +++++++++++++++ oss_active_record.gemspec | 2 +- .../config/initializers/oss_active_record.rb | 4 ++-- test/dummy/test/models/article_test.rb | 9 +++++++++ 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3a3947c..b74b0a0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,7 +2,7 @@ PATH remote: . specs: oss_active_record (0.2.0) - oss_rb (>= 0.1.2) + oss_rb (>= 0.1.3) rails (~> 4.0.0) GEM @@ -32,7 +32,7 @@ GEM multi_json (~> 1.3) thread_safe (~> 0.1) tzinfo (~> 0.3.37) - arel (4.0.0) + arel (4.0.1) atomic (1.1.14) builder (3.1.4) coderay (1.0.9) @@ -45,8 +45,8 @@ GEM method_source (0.8.2) mime-types (1.25) minitest (4.7.5) - multi_json (1.8.1) - oss_rb (0.1.2) + multi_json (1.8.2) + oss_rb (0.1.3) activesupport rest-client polyglot (0.3.3) @@ -79,7 +79,7 @@ GEM multi_json (~> 1.0) rack (~> 1.0) tilt (~> 1.1, != 1.3.0) - sprockets-rails (2.0.0) + sprockets-rails (2.0.1) actionpack (>= 3.0) activesupport (>= 3.0) sprockets (~> 2.8) @@ -91,7 +91,7 @@ GEM treetop (1.4.15) polyglot polyglot (>= 0.3.1) - tzinfo (0.3.37) + tzinfo (0.3.38) PLATFORMS ruby diff --git a/lib/oss_active_record/searchable.rb b/lib/oss_active_record/searchable.rb index abe62d1..c5ed779 100644 --- a/lib/oss_active_record/searchable.rb +++ b/lib/oss_active_record/searchable.rb @@ -1,6 +1,12 @@ module OssActiveRecord module Searchable extend ActiveSupport::Concern + + included do + extend ClassMethods + include InstanceMethods + end + module ClassMethods @@field_types= [:integer, :text, :string, :time, :suggestion] #supported field types @@index_instances = {} @@ -51,6 +57,7 @@ def find_results(search_result, field_id) end return results end + end def index(index_instance) @@ -76,6 +83,14 @@ def to_indexable(fields) end end + + #TODO Working on deletion + module InstanceMethods + def delete! + self.class.index_instance + end + alias :delete :delete! + end end ActiveRecord::Base.send :include, OssActiveRecord::Searchable \ No newline at end of file diff --git a/oss_active_record.gemspec b/oss_active_record.gemspec index a292016..92e1ea3 100644 --- a/oss_active_record.gemspec +++ b/oss_active_record.gemspec @@ -20,6 +20,6 @@ Gem::Specification.new do |s| s.test_files = Dir["test/**/*"] s.add_dependency 'rails', '~> 4.0.0' - s.add_dependency 'oss_rb', '>= 0.1.2' + s.add_dependency 'oss_rb', '>= 0.1.3' s.add_development_dependency 'sqlite3' end diff --git a/test/dummy/config/initializers/oss_active_record.rb b/test/dummy/config/initializers/oss_active_record.rb index a7c92de..2a566cf 100644 --- a/test/dummy/config/initializers/oss_active_record.rb +++ b/test/dummy/config/initializers/oss_active_record.rb @@ -1,3 +1,3 @@ Rails.configuration.open_search_server_url = "http://localhost:8080" -Rails.configuration.open_search_server_login = nil -Rails.configuration.open_search_server_apikey = nil +Rails.configuration.open_search_server_login = "admin" +Rails.configuration.open_search_server_apikey = "54a51ee4f27cbbcb7a771352b980567f" diff --git a/test/dummy/test/models/article_test.rb b/test/dummy/test/models/article_test.rb index 2b5ac75..1452f5a 100644 --- a/test/dummy/test/models/article_test.rb +++ b/test/dummy/test/models/article_test.rb @@ -44,4 +44,13 @@ class ArticleTest < ActiveSupport::TestCase end + test "Deletion" do + + results = Article.search do + fulltext 'weather' + end + + results[0].delete + end + end From 8e8245e917268ba7a64f1c1a7b8660ce58715f8c Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Wed, 23 Oct 2013 13:15:45 +0200 Subject: [PATCH 14/19] Merge with upstream --- lib/oss_active_record/searchable.rb | 85 +++++++++++++------------ test/dummy/app/models/document.rb | 9 ++- test/dummy/test/models/document_test.rb | 8 +-- 3 files changed, 49 insertions(+), 53 deletions(-) diff --git a/lib/oss_active_record/searchable.rb b/lib/oss_active_record/searchable.rb index 12af1ac..02753ea 100644 --- a/lib/oss_active_record/searchable.rb +++ b/lib/oss_active_record/searchable.rb @@ -1,3 +1,5 @@ +require 'thread' + module OssActiveRecord module Searchable extend ActiveSupport::Concern @@ -8,8 +10,8 @@ module Searchable end module ClassMethods - @@field_types= [:integer, :text, :string, :time, :suggestion] #supported field types - @@index_instances = {} + @@oss_field_types = [:integer, :text, :string, :time, :suggestion] # supported field types + @@oss_mutex = Mutex.new def searchable(options = {}, &block) yield @@ -19,69 +21,68 @@ def searchable(options = {}, &block) end def index_instance - idx_inst = @@index_instances[self.name.downcase] - if idx_inst.nil? - idx_inst = IndexInstance.new(self.name.downcase) - @@index_instances[self.name.downcase] = idx_inst + @@oss_mutex.synchronize do + @index_instance ||= IndexInstance.new(self.name.downcase) end - idx_inst end def reindex! index_instance.oss_index.delete! index_instance.create_schema! - self.all.find_in_batches do |group| - group.each { |doc| doc.index(index_instance) } - end + self.all.find_in_batches { |group| group.each(&:index) } end def method_missing(method, *args, &block) - return index_instance.add_field(args[0], method, block) if @@field_types.include? method + return index_instance.add_field(args[0], method, block) if @@oss_field_types.include? method super end - def search(*args, &block) - searchRequest = SearchRequest.new(index_instance) - searchRequest.returns index_instance.fields.map {|f|"#{f[:name]}|#{f[:type]}"} - find_results(searchRequest.execute(&block), index_instance.field_id) + def oss_search(*args, &block) + options = args.extract_options! + search_request = SearchRequest.new(index_instance) + search_request.returns index_instance.fields.map { |f| "#{f[:name]}|#{f[:type]}" } + find_results(search_request.execute(&block), index_instance.field_id, options) end + alias_method :search, :oss_search - def find_results(search_result, field_id) + def find_results(search_result, field_id, options) id_field_name = "#{field_id[:name]}|#{field_id[:type]}" - results = [] - search_result['documents'].each do |document| - document['fields'].each do |field| - id = field['values'].map {|f|f.to_i}.uniq if field['fieldName'] == id_field_name - results< { :current_revision => :user }) do + result = Document.search do fulltext 'document' with(:room_id, 42) without(:room_id, 43) @@ -32,4 +28,4 @@ class DocumentTest < ActiveSupport::TestCase assert result.length == 1, 'The number of returned suggestion is wrong' end -end \ No newline at end of file +end From d08d69596bad181468e592ca3a7af582421af765 Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Wed, 23 Oct 2013 13:42:34 +0200 Subject: [PATCH 15/19] v0.3 --- lib/oss_active_record/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oss_active_record/version.rb b/lib/oss_active_record/version.rb index 9236ef0..9a04780 100644 --- a/lib/oss_active_record/version.rb +++ b/lib/oss_active_record/version.rb @@ -1,3 +1,3 @@ module OssActiveRecord - VERSION = "0.2.0" + VERSION = "0.3.0" end From 21f00179e66a50c515cf6e16baed5b2ec438e51f Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Wed, 23 Oct 2013 17:18:00 +0200 Subject: [PATCH 16/19] Add after_destroy callback support --- Gemfile.lock | 2 +- lib/oss_active_record/index_instance.rb | 5 ++++ lib/oss_active_record/searchable.rb | 33 +++++++++++-------------- test/dummy/test/models/article_test.rb | 2 +- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b74b0a0..73e1c17 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - oss_active_record (0.2.0) + oss_active_record (0.3.0) oss_rb (>= 0.1.3) rails (~> 4.0.0) diff --git a/lib/oss_active_record/index_instance.rb b/lib/oss_active_record/index_instance.rb index 41bf108..7a3f21b 100644 --- a/lib/oss_active_record/index_instance.rb +++ b/lib/oss_active_record/index_instance.rb @@ -86,6 +86,11 @@ def index(docs) @_index.index! end + def delete_by_id(id) + id_field = find_sortable_name(:id) + @_index.delete_document_by_query("#{id_field}:#{id}") + end + end end \ No newline at end of file diff --git a/lib/oss_active_record/searchable.rb b/lib/oss_active_record/searchable.rb index 29743ad..bcd8a0c 100644 --- a/lib/oss_active_record/searchable.rb +++ b/lib/oss_active_record/searchable.rb @@ -5,10 +5,9 @@ module Searchable extend ActiveSupport::Concern included do - extend ClassMethods - include InstanceMethods + after_destroy :oss_delete end - + module ClassMethods @@oss_field_types = [:integer, :text, :string, :time, :suggestion] # supported field types @@oss_mutex = Mutex.new @@ -54,16 +53,21 @@ def find_results(search_result, field_id, options) end.compact query = if options[:include] - unscoped.includes(options[:include]) - else - unscoped - end + unscoped.includes(options[:include]) + else + unscoped + end records = query.find(ids) ids.map do |id| records.find { |record| record.id == id.to_i } end end + + end + + def oss_delete + self.class.index_instance.delete_by_id(self.id) end def index @@ -75,23 +79,16 @@ def index def to_indexable self.class.index_instance.fields.reduce({}) do |doc, field| val = if field[:block].nil? - send(field[:name].to_sym) - else - instance_eval(&field[:block]) - end + send(field[:name].to_sym) + else + instance_eval(&field[:block]) + end doc["#{field[:name]}|#{field[:type]}"] = val doc end end end - #TODO Working on deletion - module InstanceMethods - def delete! - self.class.index_instance - end - alias :delete :delete! - end end ActiveRecord::Base.send :include, OssActiveRecord::Searchable diff --git a/test/dummy/test/models/article_test.rb b/test/dummy/test/models/article_test.rb index 1452f5a..8a6165e 100644 --- a/test/dummy/test/models/article_test.rb +++ b/test/dummy/test/models/article_test.rb @@ -50,7 +50,7 @@ class ArticleTest < ActiveSupport::TestCase fulltext 'weather' end - results[0].delete + results[0].destroy end end From 17f85784e1bad9e33a100d1e96c80ea4f6d74b4b Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Wed, 23 Oct 2013 20:03:24 +0200 Subject: [PATCH 17/19] Add support of deletion --- .gitignore | 1 + lib/oss_active_record/index_instance.rb | 2 +- lib/oss_active_record/version.rb | 2 +- oss_active_record.gemspec | 2 +- test/dummy/config/application.rb | 1 - test/dummy/config/environments/test.rb | 1 + 6 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 7c7db89..60abb0c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ test/dummy/.sass-cache /.DS_Store /.buildpath /oss_active_record-*.gem +/Gemfile.lock diff --git a/lib/oss_active_record/index_instance.rb b/lib/oss_active_record/index_instance.rb index 7a3f21b..6d699d7 100644 --- a/lib/oss_active_record/index_instance.rb +++ b/lib/oss_active_record/index_instance.rb @@ -88,7 +88,7 @@ def index(docs) def delete_by_id(id) id_field = find_sortable_name(:id) - @_index.delete_document_by_query("#{id_field}:#{id}") + @_index.delete_document_by_value(id_field, id) unless id_field.nil? end end diff --git a/lib/oss_active_record/version.rb b/lib/oss_active_record/version.rb index 9a04780..5e9a465 100644 --- a/lib/oss_active_record/version.rb +++ b/lib/oss_active_record/version.rb @@ -1,3 +1,3 @@ module OssActiveRecord - VERSION = "0.3.0" + VERSION = "0.3.1" end diff --git a/oss_active_record.gemspec b/oss_active_record.gemspec index 92e1ea3..f84f94f 100644 --- a/oss_active_record.gemspec +++ b/oss_active_record.gemspec @@ -20,6 +20,6 @@ Gem::Specification.new do |s| s.test_files = Dir["test/**/*"] s.add_dependency 'rails', '~> 4.0.0' - s.add_dependency 'oss_rb', '>= 0.1.3' + s.add_dependency 'oss_rb', '>= 0.2.0' s.add_development_dependency 'sqlite3' end diff --git a/test/dummy/config/application.rb b/test/dummy/config/application.rb index 4d0d845..c203b06 100644 --- a/test/dummy/config/application.rb +++ b/test/dummy/config/application.rb @@ -1,5 +1,4 @@ require File.expand_path('../boot', __FILE__) - require 'rails/all' Bundler.require(*Rails.groups) diff --git a/test/dummy/config/environments/test.rb b/test/dummy/config/environments/test.rb index afbc0ae..3b598f8 100644 --- a/test/dummy/config/environments/test.rb +++ b/test/dummy/config/environments/test.rb @@ -33,4 +33,5 @@ # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr + end From e541659cbaa788c4334d6b1063af0a9c48b6cab2 Mon Sep 17 00:00:00 2001 From: Alexander Paramonov Date: Fri, 7 Mar 2014 13:52:17 +0200 Subject: [PATCH 18/19] Crete index object when needed --- lib/oss_active_record/index_instance.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/oss_active_record/index_instance.rb b/lib/oss_active_record/index_instance.rb index 6d699d7..f81eb38 100644 --- a/lib/oss_active_record/index_instance.rb +++ b/lib/oss_active_record/index_instance.rb @@ -52,6 +52,7 @@ def add_field(name, type, block=nil) @_fields<<{:name => name, :type => type,:block => block} end + # FIXME: private method def create_schema! @_index.create('EMPTY_INDEX') unless @_index.list.include? @_index_name @_field_id = @_fields.detect {|f| f[:name] == :id } @@ -62,6 +63,7 @@ def create_schema! end end + # FIXME: private method def create_schema_field!(field) analyzer = @@_analyzers[field[:type]] if field[:name] != :id termVectors = { :text => 'POSITIONS_OFFSETS'} @@ -82,15 +84,15 @@ def create_schema_field!(field) end def index(docs) - @_index.documents << docs - @_index.index! + oss_index.documents << docs + oss_index.index! end def delete_by_id(id) id_field = find_sortable_name(:id) - @_index.delete_document_by_value(id_field, id) unless id_field.nil? + oss_index.delete_document_by_value(id_field, id) unless id_field.nil? end end -end \ No newline at end of file +end From 44fad8f757e06bce08c059483e10cce0f98bc995 Mon Sep 17 00:00:00 2001 From: Alexander Paramonov Date: Fri, 7 Mar 2014 14:38:37 +0200 Subject: [PATCH 19/19] delete_by_id is also pseudo public. Force index build FIXME: refactor IndexInstance to either receive index as dependency or lazy build it on each public method. --- lib/oss_active_record/index_instance.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/oss_active_record/index_instance.rb b/lib/oss_active_record/index_instance.rb index f81eb38..9f254d5 100644 --- a/lib/oss_active_record/index_instance.rb +++ b/lib/oss_active_record/index_instance.rb @@ -89,6 +89,7 @@ def index(docs) end def delete_by_id(id) + oss_index id_field = find_sortable_name(:id) oss_index.delete_document_by_value(id_field, id) unless id_field.nil? end