API-3791 support exponent on decimals

Something something scientific notation.
sparkapi · Feb 19, 2016 · ca5f08f · ca5f08f
1 parent b685896
commit ca5f08f
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 1 deletion.
diff --git a/lib/sparkql/token.rb b/lib/sparkql/token.rb
@@ -7,7 +7,7 @@ module Sparkql::Token
   STANDARD_FIELD = /[A-Z]+[A-Za-z0-9]*/
   CUSTOM_FIELD = /^(\"([^$."][^."]+)\".\"([^$."][^."]*)\")/
   INTEGER = /^\-?[0-9]+/
-  DECIMAL = /^\-?[0-9]+\.[0-9]+/
+  DECIMAL = /^\-?[0-9]+\.[0-9]+([Ee]-?[0-9]{1,2})?/
   CHARACTER = /^'([^'\\]*(\\.[^'\\]*)*)'/
   DATE = /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}/
   TIME = /^[0-9]{2}\:[0-9]{2}((\:[0-9]{2})(\.[0-9]{1,50})?)?/

diff --git a/test/unit/lexer_test.rb b/test/unit/lexer_test.rb
@@ -100,4 +100,13 @@ def test_utc_offsets
       assert_equal :DATETIME, token.first, op
     end
   end
+
+  def test_decimal_matches
+    ['-15.42', '1.0', '0.22', '9.0E-6', '-9.0E-3'].each do |op|
+      @lexer = Lexer.new(op)
+      token = @lexer.shift
+      assert_equal :DECIMAL, token.first, op
+    end
+  end
+
 end
diff --git a/test/unit/parser_compatability_test.rb b/test/unit/parser_compatability_test.rb
@@ -67,6 +67,21 @@ def setup
         :type => :decimal,
         :operator => "Eq"
       },
+      {
+        :string => "FloatField Eq 9.1E-6",
+        :type => :decimal,
+        :operator => "Eq"
+      },
+      {
+        :string => "FloatField Eq -9.1E-6",
+        :type => :decimal,
+        :operator => "Eq"
+      },
+      {
+        :string => "FloatField Eq 1.0E8",
+        :type => :decimal,
+        :operator => "Eq"
+      },
       {
         :string => "FloatField Eq -2001.120,-2002.0",
         :type => :decimal,
@@ -468,6 +483,13 @@ def find_operator(string)
     assert_equal true, parser.escape_value(expressions.first)
   end
 
+  test "escape decimal values" do
+    parser = Parser.new
+    expressions = parser.tokenize( "DecimalField Eq 0.00005 And DecimalField Eq 5.0E-5" )
+    assert_equal 5.0E-5, parser.escape_value(expressions.first)
+    assert_equal parser.escape_value(expressions.first), parser.escape_value(expressions.last)
+  end
+
   test "Between" do
     ["BathsFull Bt 10,20", "DateField Bt 2012-12-31,2013-01-31"].each do |f|
       parser = Parser.new