support json unicode validation

Change-Id: Ib88dd862bcde1c106ea0b84bffd3a63d6bb1e6ab
lock-free · Sep 25, 2020 · 0bc852e · 0bc852e
1 parent 2e1c3ec
commit 0bc852e
Show file tree

Hide file tree

Showing 3 changed files with 108 additions and 5 deletions.
diff --git a/src/main/scala/io/github/shopee/idata/sjson/TokenParser.scala b/src/main/scala/io/github/shopee/idata/sjson/TokenParser.scala
@@ -43,11 +43,15 @@ object TransitionState {
 
   val STRING_IN     = 11
   val STRING_ESCAPE = 12
+  val STRING_UNICODE_IN_0 = 13
+  val STRING_UNICODE_IN_1 = 14
+  val STRING_UNICODE_IN_2 = 15
+  val STRING_UNICODE_IN_3 = 16
 
-  val NUMBER_INTEGER          = 13
-  val NUMBER_FRAGMENT_INTEGER = 14
-  val NUMBER_SCIENCE_PART     = 15
-  val NUMBER_SCIENCE_INTEGER  = 16
+  val NUMBER_INTEGER          = 17
+  val NUMBER_FRAGMENT_INTEGER = 18
+  val NUMBER_SCIENCE_PART     = 19
+  val NUMBER_SCIENCE_INTEGER  = 20
 
   // transitions
   val NEW_TRANS                      = Transition(NEW)
@@ -63,6 +67,10 @@ object TransitionState {
   val FALSE_L_TRANS                  = Transition(FALSE_L)
   val FALSE_S_TRANS                  = Transition(FALSE_S)
   val STRING_ESCAPE_TRANS            = Transition(STRING_ESCAPE)
+  val STRING_UNICODE_IN_0_TRANS      = Transition(STRING_UNICODE_IN_0)
+  val STRING_UNICODE_IN_1_TRANS      = Transition(STRING_UNICODE_IN_1)
+  val STRING_UNICODE_IN_2_TRANS      = Transition(STRING_UNICODE_IN_2)
+  val STRING_UNICODE_IN_3_TRANS      = Transition(STRING_UNICODE_IN_3)
   val STRING_IN_TRANS                = Transition(STRING_IN)
   val NUMBER_INTEGER_TRANS           = Transition(NUMBER_INTEGER)
   val NUMBER_SCIENCE_INTEGER_TRANS   = Transition(NUMBER_SCIENCE_INTEGER)
@@ -126,6 +134,49 @@ class ParseToken(onTokenCallback: (JSONToken) => Unit) {
       }
 
       case STRING_ESCAPE => {
+        if (ch == '"' || ch == '\\' || ch == '/' || ch == 'b' || ch == 'f' || ch == 'n' || ch == 'r' || ch == 't') {
+          txtBuilder.append(ch)
+          STRING_IN_TRANS
+        } else if (ch == 'u') { // code
+          txtBuilder.append(ch)
+          STRING_UNICODE_IN_0_TRANS
+        } else { // error
+          throw new Exception(s"errored char ${ch} after escape char \\")
+        }
+      }
+
+      case STRING_UNICODE_IN_0 => { // u8
+        if (ch < '0' || ch > '9') {
+          throw new Exception(s"illegal char ${ch} in unicode string, unicode format example: \\u1000")
+        }
+
+        txtBuilder.append(ch)
+        STRING_UNICODE_IN_1_TRANS
+      }
+
+      case STRING_UNICODE_IN_1 => { // u82
+        if (ch < '0' || ch > '9') {
+          throw new Exception(s"illegal char ${ch} in unicode string, unicode format example: \\u1000")
+        }
+
+        txtBuilder.append(ch)
+        STRING_UNICODE_IN_2_TRANS
+      }
+
+      case STRING_UNICODE_IN_2 => { // u822
+        if (ch < '0' || ch > '9') {
+          throw new Exception(s"illegal char ${ch} in unicode string, unicode format example: \\u1000")
+        }
+
+        txtBuilder.append(ch)
+        STRING_UNICODE_IN_3_TRANS
+      }
+
+      case STRING_UNICODE_IN_3 => { // u8223
+        if (ch < '0' || ch > '9') {
+          throw new Exception(s"illegal char ${ch} in unicode string, unicode format example: \\u1000")
+        }
+
         txtBuilder.append(ch)
         STRING_IN_TRANS
       }

diff --git a/src/test/scala/io/github/shopee/idata/sjson/ErrorJSON.scala b/src/test/scala/io/github/shopee/idata/sjson/ErrorJSON.scala
@@ -49,4 +49,48 @@ class ErrorJSONTest extends org.scalatest.FunSuite {
     val fun = () => {}
     assert(JSON.stringify(fun()) == "null")
   }
+
+  test("tokenParse: escape error") {
+    try {
+      JSON.parse(s""""\\a"""")
+      throw new Exception("should have error")
+    } catch {
+      case e: Exception => {
+        assert(e.getMessage.indexOf("errored char a after escape char") !== -1)
+      }
+    }
+  }
+
+  test("tokenParse: unicode error") {
+    try {
+      JSON.parse(s""""\\u"""")
+      throw new Exception("should have error")
+    } catch {
+      case e: Exception => {
+        assert(e.getMessage.indexOf("unicode format example") !== -1)
+      }
+    }
+  }
+
+  test("tokenParse: unicode error2") {
+    try {
+      JSON.parse(s""""\\u12"""")
+      throw new Exception("should have error")
+    } catch {
+      case e: Exception => {
+        assert(e.getMessage.indexOf("unicode format example") !== -1)
+      }
+    }
+  }
+
+  test("tokenParse: unicode error3") {
+    try {
+      JSON.parse(s""""\\u12b"""")
+      throw new Exception("should have error")
+    } catch {
+      case e: Exception => {
+        assert(e.getMessage.indexOf("unicode format example") !== -1)
+      }
+    }
+  }
 }
diff --git a/src/test/scala/io/github/shopee/idata/sjson/TokenParserTest.scala b/src/test/scala/io/github/shopee/idata/sjson/TokenParserTest.scala
@@ -29,7 +29,15 @@ class TokenParseTest extends org.scalatest.FunSuite {
   }
 
   test("toTokens: single string") {
-    List(s"""""""", s""""hello, world"""", s""""123"""", s""""\\""""", s""""\n"""", s""""\t"""", s""""\\\\"""").map((txt) => {
+    List(s"""""""", s""""hello, world"""", s""""123"""", s""""\\""""",
+      s""""\\n"""", s""""\t"""", s""""\\\\"""", s""""\\r"""", s""""\\b"""",
+      s""""\\f"""", s""""\\/"""").map((txt) => {
+      testToToken(txt, List(JSONToken(JSONToken.STRING, txt)))
+    })
+  }
+
+  test("toTokens: unicode") {
+    List(s""""\\u1234"""").map((txt) => {
       testToToken(txt, List(JSONToken(JSONToken.STRING, txt)))
     })
   }