[feature] Ability to query API resources with KEYWORDS (#1466)

Addresses: #1374 **DEMO** https://user-images.githubusercontent.com/25191509/214291392-23e3c962-31ce-4df3-b3fa-1a2565c978be.mov ### **For client-engineers** For http-request to search by `KEYWORDS`, we would need to pass `option: KEYWORDS` as key-value pair for the attributes we want to search by in `properties` payload. In the clip below, we are filtering `cars` by searching in its attributes, namely: `make`, `model` and `type` with `KEYWORDS` option. https://user-images.githubusercontent.com/25191509/224547568-37ff1e9c-51e0-4212-bfa4-dd123c9857ac.mov Payload Example: ``` properties: {"make":{"value":"kia tesla","option":"KEYWORDS"},"model":{"value":"kia tesla","option":"KEYWORDS"},"type":{"value":"kia tesla","option":"KEYWORDS"}}, match: ANY ``` Co-authored-by: Prashant Khadka <[email protected]>
restarone · Mar 15, 2023 · 89fb1c8 · 89fb1c8
1 parent bd50172
commit 89fb1c8
Show file tree

Hide file tree

Showing 7 changed files with 330 additions and 16 deletions.
diff --git a/app/controllers/comfy/admin/api_namespaces_controller.rb b/app/controllers/comfy/admin/api_namespaces_controller.rb
@@ -43,13 +43,6 @@ def show
 
     field, direction = params[:q].key?(:s) ? params[:q][:s].split(" ") : [nil, nil]
     fields_in_properties = @api_namespace.properties.keys
-    @custom_properties = {}
-    @api_namespace.properties.values.each_with_index do |obj,index|
-      if(obj.present? && obj != "nil" && obj != "\"\"")
-        @custom_properties[fields_in_properties[index]] = obj;
-      end
-    end
-    @custom_properties = JSON.parse(@custom_properties.to_json, object_class: OpenStruct).to_s.gsub(/=/,': ').gsub(/#<OpenStruct/,'{').gsub(/>/,'}').gsub("\\", "'").gsub(/"'"/,'"').gsub(/'""/,'"')
     @image_options = @api_namespace.non_primitive_properties.select { |non_primitive_property| non_primitive_property.field_type == 'file' }.pluck(:label)
     # check if we are sorting by a field inside properties jsonb column
     if field && fields_in_properties.include?(field)

diff --git a/app/helpers/api_namespaces_helper.rb b/app/helpers/api_namespaces_helper.rb
@@ -10,4 +10,27 @@ def graphql_base_url(subdomain, namespace)
   def system_paths
     Comfy::Cms::Page.all.pluck(:full_path)
   end
+
+  def api_html_renderer_dynamic_properties(namespace, search_option = nil)
+    custom_properties = {}
+    fields_in_properties = namespace.properties.keys
+
+    namespace.properties.values.each_with_index do |obj,index|
+      if obj.present? && obj != "nil" && obj != "\"\""
+        if search_option.present?
+          next if !obj.is_a?(Array) && !obj.is_a?(String)
+
+          custom_properties[fields_in_properties[index]] = {
+            value: obj.is_a?(Array) ? obj.first(1) : obj.split.first,
+            option: search_option
+          }
+        else
+          custom_properties[fields_in_properties[index]] = obj;
+        end
+      end
+    end
+
+    # sanitize the text to properly display
+    JSON.parse(custom_properties.to_json, object_class: OpenStruct).to_s.gsub(/=/,': ').gsub(/#<OpenStruct/,'{').gsub(/>/,' }').gsub("\\", "'").gsub(/"'"/,'"').gsub(/'""/,'"')
+  end
 end
diff --git a/app/models/concerns/jsonb_search/query_builder.rb b/app/models/concerns/jsonb_search/query_builder.rb
@@ -2,13 +2,19 @@ module JsonbSearch
   module QueryBuilder
     QUERY_OPTION = {
       EXACT: 'EXACT',
-      PARTIAL: 'PARTIAL'
+      PARTIAL: 'PARTIAL',
+      KEYWORDS: 'KEYWORDS'
     }.freeze
 
     MATCH_OPTION = {
       ALL: 'ALL',
       ANY: 'ANY'
-  }.freeze
+    }.freeze
+
+    # https://github.com/Altoros/belarus-ruby-on-rails/blob/master/solr/conf/stopwords.txt
+    STOP_WORDS = [
+      'a', 'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'for', 'if', 'in', 'into', 'is', 'it', 'no', 'not', 'of', 'on', 'or', 's', 'such', 't', 'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was', 'will', 'with'
+    ]
 
     class << self
       def build_jsonb_query(column_name, query_params, match = nil)
@@ -23,12 +29,15 @@ def parse_params(query_params)
 
         query_params.each do |key, value|
           if value.is_a?(Hash) && value.key?(:value)
+            next unless value[:value].present?
             # { name: { value: 'violet', query }} || { name: { value: 'violet', option: 'EXACT' }}
             queries << { option: value[:option] || QUERY_OPTION[:EXACT], key: key, value: value[:value], match: value[:match] }
           elsif value.is_a?(Hash)
             # { foo: { bar: 'baz', wat: 'up' }}
             value.each do |k, v|
               if v.is_a?(Hash) && v.key?(:value)
+                next unless v[:value].present?
+
                 queries <<  { key: key, value: [{ option: v[:option] || QUERY_OPTION[:EXACT], key: k, value: v[:value], match: v[:match] }] }
               else
                 queries << { key: key, value: [{ option: QUERY_OPTION[:EXACT], key: k, value: v }]}
@@ -75,12 +84,33 @@ def generate_query(param, query_string)
         end
       end
 
-      # "column ->> 'property' = 'term'" 
+      # "column ->> 'property' = 'term'"
       def string_query(key, term, option, query)
-        if option == QUERY_OPTION[:PARTIAL]
+        if option == QUERY_OPTION[:KEYWORDS]
+          query_array = []
+          operator = 'LIKE'
+
+          terms = term.split(' ') - STOP_WORDS
+          terms = terms.map { |txt| "%#{txt}%" }
+
+          terms.each do |txt|
+            query_array << generate_string_sql(query, key, txt, operator)
+          end
+
+          query_array << generate_string_sql(query, key, "%#{term}%", operator) if terms.size > 1
+
+          query_array.join(' OR ')
+        elsif option == QUERY_OPTION[:PARTIAL]
           term = "%#{term}%"
           operator = 'LIKE'
+
+          generate_string_sql(query, key, term, operator)
+        else
+          generate_string_sql(query, key, term)
         end
+      end
+
+      def generate_string_sql(query, key, term, operator = nil)
         # A ' inside a string quoted with ' may be written as ''.
         # https://stackoverflow.com/questions/54144340/how-to-query-jsonb-fields-and-values-containing-single-quote-in-rails#comment95120456_54144340
         # https://dev.mysql.com/doc/refman/8.0/en/string-literals.html#character-escape-sequences
@@ -95,7 +125,21 @@ def hash_query(key, term, option, query)
 
       # "column -> 'property' ? '['term']'" 
       def array_query(key, term, option, query, match)
-        if option == QUERY_OPTION[:PARTIAL]
+        if option == QUERY_OPTION[:KEYWORDS]
+          query_array = []
+          operator = 'LIKE'
+
+          term.each do |data|
+            items = data.split(' ') - STOP_WORDS
+            items = items.map { |txt| "%#{txt}%" }
+
+            items.each do |item|
+              query_array  << "lower(#{query} ->> '#{key}'::text) LIKE lower('#{item.to_s.gsub("'", "''")}')"
+            end
+          end
+
+          query_array.join(match == MATCH_OPTION[:ANY] ? ' OR ' : ' AND ')
+        elsif option == QUERY_OPTION[:PARTIAL]
           match == MATCH_OPTION[:ANY] ? term.map { |q| "#{query} -> '#{key}' ? '#{q}'" }.join(' OR ') : "#{query} -> '#{key}' @> '#{term.to_json.gsub("'", "''")}'"
         else
           "#{query} -> '#{key}' @> '#{term.to_json}' AND #{query} -> '#{key}' <@ '#{term.to_json.gsub("'", "''")}'"

diff --git a/app/views/comfy/admin/api_namespaces/show.html.haml b/app/views/comfy/admin/api_namespaces/show.html.haml
@@ -115,10 +115,16 @@
       %pre= @api_namespace.snippet
       %p
         %b API HTML Renderer index snippet:
-      %pre= "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: #{@custom_properties} } }}"
+      %pre= "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: #{api_html_renderer_dynamic_properties(@api_namespace)} } }}"
+      %p
+        %b API HTML Renderer index snippet (KEYWORDS - works for array and string data type only):
+      %pre= "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: #{api_html_renderer_dynamic_properties(@api_namespace, 'KEYWORDS')} } }}"
       %p
         %b API HTML Renderer show snippet:
-      %pre= "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: #{@custom_properties} } }}"
+      %pre= "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: #{api_html_renderer_dynamic_properties(@api_namespace)} } }}"
+      %p
+        %b API HTML Renderer show snippet (KEYWORDS - works for array and string data type only):
+      %pre= "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: #{api_html_renderer_dynamic_properties(@api_namespace, 'KEYWORDS')} } }}"
       %p
         .d-flex.justify-content-between
           %b Preview (outer border is present in preview only):

diff --git a/test/controllers/admin/comfy/api_namespaces_controller_test.rb b/test/controllers/admin/comfy/api_namespaces_controller_test.rb
@@ -783,9 +783,14 @@ class Comfy::Admin::ApiNamespacesControllerTest < ActionDispatch::IntegrationTes
     assert_select "b", {count: 1, text: "Form rendering snippet:"}
     assert_select "pre", {count: 1, text: @api_namespace.snippet}
     assert_select "b", {count: 1, text: "API HTML Renderer index snippet:"}
-    assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: { arr: [1, 2, 3], obj: { a: \"b\", c: \"d\"}, title: \"Hello World\", test_id: 123, alpha_arr: [\"a\", \"b\"], published: true} } }}"}
+    assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: { arr: [1, 2, 3], obj: { a: \"b\", c: \"d\" }, title: \"Hello World\", test_id: 123, alpha_arr: [\"a\", \"b\"], published: true } } }}"}
     assert_select "b", {count: 1, text: "API HTML Renderer show snippet:"}
-    assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: { arr: [1, 2, 3], obj: { a: \"b\", c: \"d\"}, title: \"Hello World\", test_id: 123, alpha_arr: [\"a\", \"b\"], published: true} } }}"}
+    assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: { arr: [1, 2, 3], obj: { a: \"b\", c: \"d\" }, title: \"Hello World\", test_id: 123, alpha_arr: [\"a\", \"b\"], published: true } } }}"}
+    # Dynamic renderer snippet for KEYWORDS based search
+    assert_select "b", {count: 1, text: "API HTML Renderer index snippet (KEYWORDS - works for array and string data type only):"}
+    assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource_index '#{@api_namespace.slug}', scope: { properties: { arr: { value: [1], option: \"KEYWORDS\" }, title: { value: \"Hello\", option: \"KEYWORDS\" }, alpha_arr: { value: [\"a\"], option: \"KEYWORDS\" } } } }}"}
+    assert_select "b", {count: 1, text: "API HTML Renderer show snippet (KEYWORDS - works for array and string data type only):"}
+    assert_select "pre", {count: 1, text: "{{ cms:helper render_api_namespace_resource '#{@api_namespace.slug}', scope: { properties: { arr: { value: [1], option: \"KEYWORDS\" }, title: { value: \"Hello\", option: \"KEYWORDS\" }, alpha_arr: { value: [\"a\"], option: \"KEYWORDS\" } } } }}"}
   end
 
   ######## API Accessibility Tests - START #########

diff --git a/test/controllers/api/resource_controller_test.rb b/test/controllers/api/resource_controller_test.rb
@@ -300,6 +300,122 @@ class Api::ResourceControllerTest < ActionDispatch::IntegrationTest
     assert_empty response.parsed_body["data"]
   end
 
+  test '#index search jsonb field - string - KEYWORDS: multi word string' do
+    @api_resource_1.update(properties: {name: 'Professional Writer'})
+    @api_resource_2.update(properties: {name: 'Physical Development'})
+    @api_resource_3.update(properties: {name: 'Professional Development'})
+
+    payload = { 
+      properties: { 
+        name: { 
+          value: 'professional development',
+          option: 'KEYWORDS'
+        }
+      }
+    }
+    get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json
+    assert_response :success
+
+    assert_equal response.parsed_body["data"].pluck("id").map(&:to_i).sort, [@api_resource_1.id, @api_resource_2.id, @api_resource_3.id].sort
+  end
+
+  test '#index search jsonb field - string - KEYWORDS: multi word string (unhappy)' do
+    @api_resource_1.update(properties: {name: 'Professional Writer'})
+    @api_resource_2.update(properties: {name: 'Physical Development'})
+    @api_resource_3.update(properties: {name: 'Professional Development'})
+
+    payload = { 
+      properties: { 
+        name: { 
+          value: 'hello world',
+          option: 'KEYWORDS'
+        }
+      }
+    }
+    get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json
+    assert_response :success
+
+    assert_empty response.parsed_body["data"]
+  end
+
+  test '#index search jsonb field - Array - KEYWORDS: match ALL' do
+    @api_resource_1.update(properties: {tags: ['Professional Writer', 'zebra']})
+    @api_resource_2.update(properties: {tags: ['Physical Development', 'cow']})
+    @api_resource_3.update(properties: {tags: ['Professional Development', 'animal']})
+
+    payload = { 
+      properties: { 
+        tags: { 
+          value: ['professional development'],
+          option: 'KEYWORDS'
+        }
+      }
+    }
+    get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json
+    assert_response :success
+
+    assert_equal response.parsed_body["data"].pluck("id").map(&:to_i).sort, [@api_resource_3.id].sort
+  end
+
+  test '#index search jsonb field - Array - KEYWORDS: match ALL (unhappy)' do
+    @api_resource_1.update(properties: {tags: ['Professional Writer', 'zebra']})
+    @api_resource_2.update(properties: {tags: ['Physical Development', 'cow']})
+    @api_resource_3.update(properties: {tags: ['Professional Development', 'animal']})
+
+    payload = { 
+      properties: { 
+        tags: { 
+          value: ['hello world'],
+          option: 'KEYWORDS'
+        }
+      }
+    }
+    get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json
+    assert_response :success
+
+    assert_empty response.parsed_body["data"]
+  end
+
+  test '#index search jsonb field - Array - KEYWORDS: match ANY' do
+    @api_resource_1.update(properties: {tags: ['Professional Writer', 'zebra']})
+    @api_resource_2.update(properties: {tags: ['Physical Development', 'cow']})
+    @api_resource_3.update(properties: {tags: ['Professional Development', 'animal']})
+
+    payload = { 
+      properties: { 
+        tags: { 
+          value: ['professional development'],
+          option: 'KEYWORDS',
+          match: 'ANY'
+        }
+      }
+    }
+    get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json
+    assert_response :success
+
+    assert_equal response.parsed_body["data"].pluck("id").map(&:to_i).sort, [@api_resource_1.id, @api_resource_2.id, @api_resource_3.id].sort
+  end
+
+  test '#index search jsonb field - Array - KEYWORDS: match ANY (unhappy)' do
+    @api_resource_1.update(properties: {tags: ['Professional Writer', 'zebra']})
+    @api_resource_2.update(properties: {tags: ['Physical Development', 'cow']})
+    @api_resource_3.update(properties: {tags: ['Professional Development', 'animal']})
+
+    payload = { 
+      properties: { 
+        tags: { 
+          value: ['hello world'],
+          option: 'KEYWORDS',
+          match: 'ANY'
+        }
+      }
+    }
+    get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json
+    assert_response :success
+
+    assert_empty response.parsed_body["data"]
+  end
+
   test '#index search jsonb field - nested string' do
     payload = { 
       properties: { 
@@ -484,4 +600,99 @@ class Api::ResourceControllerTest < ActionDispatch::IntegrationTest
 
     assert_empty response.parsed_body["data"]
   end
+
+  test '#index search jsonb field - array - KEYWORDS match ALL' do
+    @api_resource_2.update(properties: {interests: ['hello world', 'foo', 'bar']})
+    payload = { 
+        properties: { 
+          interests: {
+            value: ['hello world', 'foo'],
+            option: 'KEYWORDS'
+          }
+        }
+      }
+
+    get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json
+
+    assert_equal response.parsed_body["data"].pluck("id").map(&:to_i).sort, [@api_resource_2.id].sort
+  end
+
+  test '#index search jsonb field - array - KEYWORDS match ANY' do
+    @api_resource_1.update(properties: {interests: ['hello']})
+    @api_resource_2.update(properties: {interests: ['hello world', 'foo', 'bar']})
+
+    payload = { 
+        properties: { 
+          interests: {
+            value: ['hello world', 'foo'],
+            option: 'KEYWORDS',
+            match: 'ANY'
+          }
+        }
+      }
+
+    get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json
+
+    assert_equal response.parsed_body["data"].pluck("id").map(&:to_i).sort, [@api_resource_1.id, @api_resource_2.id].sort
+  end
+
+  test '#index search jsonb field - string - ignores empty values without throwing exception' do
+    @api_resource_1.update(properties: {name: 'Professional Writer', age: 11})
+    @api_resource_2.update(properties: {name: 'Physical Development', age: 22})
+    @api_resource_3.update(properties: {name: 'Professional Development', age: 33})
+
+    payload = { 
+      properties: { 
+        name: { 
+          value: '',
+          option: 'KEYWORDS'
+        },
+        age: { 
+          value: '',
+          option: 'KEYWORDS'
+        },
+      }
+    }
+
+    get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json
+    # Does not throw exception
+    assert_response :success
+
+    response_resource_ids = response.parsed_body["data"].pluck("id").map(&:to_i).sort
+
+    # Ignores empty value parameters and the filters are not applied
+    assert_includes response_resource_ids, @api_resource_1.id
+    assert_includes response_resource_ids, @api_resource_2.id
+    assert_includes response_resource_ids, @api_resource_3.id
+  end
+
+  test '#index search jsonb field - array - ignores empty values without throwing exception' do
+    @api_resource_1.update(properties: {name: 'Professional Writer', age: 11})
+    @api_resource_2.update(properties: {name: 'Physical Development', age: 22})
+    @api_resource_3.update(properties: {name: 'Professional Development', age: 33})
+
+    payload = { 
+      properties: { 
+        name: { 
+          value: [],
+          option: 'KEYWORDS'
+        },
+        age: { 
+          value: [],
+          option: 'KEYWORDS'
+        },
+      }
+    }
+
+    get api_url(version: @api_namespace.version, api_namespace: @api_namespace.slug), params: payload, as: :json
+    # Does not throw exception
+    assert_response :success
+
+    response_resource_ids = response.parsed_body["data"].pluck("id").map(&:to_i).sort
+
+    # Ignores empty value parameters and the filters are not applied
+    assert_includes response_resource_ids, @api_resource_1.id
+    assert_includes response_resource_ids, @api_resource_2.id
+    assert_includes response_resource_ids, @api_resource_3.id
+  end
 end