Merge pull request #45 from target/adds-byte-to-columnsumcheck

Adds support for Bytes to ColumnSumCheck
target · May 28, 2020 · 6fed17c · 6fed17c
2 parents d829d66 + 01515df
commit 6fed17c
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 1 deletion.
diff --git a/src/main/scala/com/target/data_validator/validator/ColumnSumCheck.scala b/src/main/scala/com/target/data_validator/validator/ColumnSumCheck.scala
@@ -63,6 +63,7 @@ case class ColumnSumCheck(
       case LongType => evaluate(r.getLong(idx))
       case FloatType => evaluate(r.getFloat(idx))
       case DoubleType => evaluate(r.getDouble(idx))
+      case ByteType => evaluate(r.getByte(idx))
       case ut => throw new Exception(s"Unsupported type for $name found in schema: $ut")
     }
 

diff --git a/src/test/scala/com/target/data_validator/TestHelpers.scala b/src/test/scala/com/target/data_validator/TestHelpers.scala
@@ -25,6 +25,7 @@ object TestHelpers {
     case "java.lang.Double" => DoubleType
     case "java.lang.Boolean" => BooleanType
     case "java.lang.Long" => LongType
+    case "java.lang.Byte" => ByteType
     case _ => throw new IllegalArgumentException(s"Unknown type '${v.getClass.getCanonicalName}'")
   }
 

diff --git a/src/test/scala/com/target/data_validator/validator/ColumnSumCheckSpec.scala b/src/test/scala/com/target/data_validator/validator/ColumnSumCheckSpec.scala
@@ -200,14 +200,22 @@ class ColumnSumCheckSpec extends FunSpec with Matchers with TestingSparkSession
         assert(!sut.failed)
       }
 
-      it("upper bound success") {
+      it("upper bound success with short") {
         val check = ColumnSumCheck("foo", maxValue = Some(Json.fromDouble(10).get)) // scalastyle:ignore magic.number
         val df = mkDf(spark, "foo" -> List[Short](1, 2, 1))
         val sut = ValidatorDataFrame(df, None, None, List(check))
         assert(!sut.quickChecks(spark, mkDict())(config))
         assert(!sut.failed)
       }
 
+      it("upper bound success with byte") {
+        val check = ColumnSumCheck("foo", maxValue = Some(Json.fromDouble(10).get)) // scalastyle:ignore magic.number
+        val df = mkDf(spark, "foo" -> List[Byte](1, 2, 1))
+        val sut = ValidatorDataFrame(df, None, None, List(check))
+        assert(!sut.quickChecks(spark, mkDict())(config))
+        assert(!sut.failed)
+      }
+
       it("upper bound failure") {
         val check = ColumnSumCheck("foo", maxValue = Some(Json.fromFloat(1).get)) // scalastyle:ignore magic.number
         val df = mkDf(spark, "foo" -> List(1L, 1L, 1L))