Fuzz measurement (#42)

* add a fuzz target for measurement, and remove the fuzz target for string with flags since that didn't seem to do much.

* update circle ci to create artifacts for the measurement test

* update the fuzzing target and quick fuzz target and the is_valid for measurements to trap error and invalid units

* add a check for parenthesis after a leading multiplier

* try trapping root powers of 2 and negative numbers

* try using normalcy as a check in the measurement fuzzer

* add another fuzz failure, try manipulating the tests a little to not have some odd conditions

* make a fix to handle cases like cindex} which somehow registers as valid unit when it shouldn't

* add a few more things to the dictionary

* add some measurement tests for negative values in roots and string outputs.

* update dockerfile and fuzz_script.sh to use the measurement for a while

.circleci/config.yml

@@ -69,7 +69,7 @@ jobs:
       - store_artifacts:
           path: /root/project/build/FuzzTargets/units_fail_string_artifact.txt
       - store_artifacts:
-          path: /root/project/build/FuzzTargets/units_fail_flags_artifact.txt
+          path: /root/project/build/FuzzTargets/units_fail_measurement_artifact.txt
 
 workflows:
   version: 2

FuzzTargets/CMakeLists.txt

@@ -1,5 +1,5 @@
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# Copyright (c) 2019,
+# Copyright (c) 2019-2020,
 # Lawrence Livermore National Security, LLC;
 # See the top-level NOTICE for additional details. All rights reserved.
 # SPDX-License-Identifier: BSD-3-Clause
@@ -9,18 +9,18 @@ add_executable(fuzz_from_string fuzz_target_from_string.cpp)
 target_link_libraries(fuzz_from_string units::units compile_flags_target)
 target_include_directories(fuzz_from_string PRIVATE ${CMAKE_SOURCE_DIR}/ThirdParty)
 
-add_executable(fuzz_with_flags fuzz_target_from_string_flags.cpp)
-target_link_libraries(fuzz_with_flags units::units compile_flags_target)
-target_include_directories(fuzz_with_flags PRIVATE ${CMAKE_SOURCE_DIR}/ThirdParty)
+add_executable(fuzz_measurement fuzz_target_measurement_from_string.cpp)
+target_link_libraries(fuzz_measurement units::units compile_flags_target)
+target_include_directories(fuzz_measurement PRIVATE ${CMAKE_SOURCE_DIR}/ThirdParty)
 
 add_custom_target(
     QUICK_RAW_FUZZ
-    COMMAND ${CMAKE_COMMAND} -E make_directory corp1
+    COMMAND ${CMAKE_COMMAND} -E make_directory corp2
+    COMMAND fuzz_measurement corp2 -max_total_time=300 -max_len=512
+            -dict=${CMAKE_CURRENT_SOURCE_DIR}/fuzz_dictionary.txt
+            -exact_artifact_path=units_fail_measurement_artifact.txt
+	COMMAND ${CMAKE_COMMAND} -E make_directory corp1
     COMMAND fuzz_from_string corp1 -max_total_time=300 -max_len=512
             -dict=${CMAKE_CURRENT_SOURCE_DIR}/fuzz_dictionary.txt
             -exact_artifact_path=units_fail_string_artifact.txt
-    COMMAND ${CMAKE_COMMAND} -E make_directory corp2
-    COMMAND fuzz_with_flags corp2 -max_total_time=300 -max_len=512
-            -dict=${CMAKE_CURRENT_SOURCE_DIR}/fuzz_dictionary.txt
-            -exact_artifact_path=units_fail_flags_artifact.txt
 )

FuzzTargets/Dockerfile

@@ -9,7 +9,7 @@ RUN cmake ../units -DUNITS_BUILD_FUZZ_TARGETS=ON -DCMAKE_CXX_STANDARD=17 -DUNITS
 
 WORKDIR /root/develop/fuzz_targets
 
-RUN cp ../build/FuzzTargets/fuzz_from_string . && cp ../units/FuzzTargets/*.txt . && cp ../units/FuzzTargets/*.sh .
+RUN cp ../build/FuzzTargets/fuzz_measurement . && cp ../units/FuzzTargets/*.txt . && cp ../units/FuzzTargets/*.sh .
 
 FROM ubuntu:18.04
 

FuzzTargets/fuzz_dictionary.txt

@@ -13,6 +13,8 @@
 "s.s"
 "per-unit"
 "\xfe\xcb\xd7\xd8"
+"twenty-seven"
+"one hundred twenty-two"
 "\xff\xff\xff\xff"
 " per "
 "d\x8aleou"
@@ -21,7 +23,6 @@
 "internationaltable"
 "^(1)"
 "\x93\xb6\xc1\xff"
-"putdire"
 "th"
 "@("
 "{}"
@@ -111,6 +112,7 @@
 "thousand"
 "million"
 "billion"
+"hundred"
 "trillion"
 "tenth"
 "apothecary"

FuzzTargets/fuzz_script.sh

@@ -4,13 +4,13 @@ mkdir -p /fuzz/corpus
 mkdir -p /fuzz/logs
 
 # run the first fuzzing sequence
-./fuzz_from_string /fuzz/corpus -max_len=512 -dict=/root/develop/fuzz_targets/fuzz_dictionary.txt -rss_limit_mb=512 -max_total_time=1405 -timeout=20 -artifact_prefix=/fuzz/ &> fuzzlog.txt
+./fuzz_measurement /fuzz/corpus -max_len=512 -dict=/root/develop/fuzz_targets/fuzz_dictionary.txt -rss_limit_mb=512 -max_total_time=1405 -timeout=20 -artifact_prefix=/fuzz/ &> fuzzlog.txt
 
 #if we haven't failed merge the corpus so it is smaller
 if [ $? -eq 0 ]
 then
   mkdir -p new_corpus
-  ./fuzz_from_string new_corpus /fuzz/corpus -merge=1 -max_len=512 -max_total_time=120 &>"/fuzz/logs/fuzz_merge$(date +"%s").log"
+  ./fuzz_measurement new_corpus /fuzz/corpus -merge=1 -max_len=512 -max_total_time=120 &>"/fuzz/logs/fuzz_merge$(date +"%s").log"
   if [ $? -eq 0 ]
   then
     rm /fuzz/corpus/*

FuzzTargets/fuzz_target_measurement_from_string.cpp

@@ -0,0 +1,52 @@
+/*
+Copyright (c) 2019-2020,
+Lawrence Livermore National Security, LLC;
+See the top-level NOTICE for additional details. All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+*/
+
+#include "units/units.hpp"
+#include <cstring>
+#include <exception>
+#include <string>
+
+static bool cflag = units::disableCustomCommodities();
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size)
+{
+    if (Size == 0) {
+        return 0;
+    }
+    std::string test1(reinterpret_cast<const char*>(Data), Size);
+
+    auto meas1 = units::measurement_from_string(test1);
+    if (isnormal(meas1)) {
+        auto str = to_string(meas1);
+        auto meas2 = units::measurement_from_string(str);
+        if (!meas2.units().has_same_base(meas1.units()) && !isnormal(meas2)) {
+            throw(6u);
+        }
+        bool match = (meas1 == meas2);
+        if (!match) {
+            auto mc1 = units::measurement_cast(meas1);
+            auto mc2 = units::measurement_cast(meas2);
+            match = (mc1 == mc2);
+
+            if (!match && isnormal(root(meas2, 2))) {
+                match = (root(mc2, 2) == root(mc1, 2));
+            }
+            if (!match && isnormal(root(meas2, 3))) {
+                match = (root(mc2, 3) == root(mc1, 3));
+            }
+        }
+        if (!match) {
+            if (meas1.units() == meas2.units()) {
+                throw(std::invalid_argument("measurement and conversion don't match but units do"));
+            }
+            throw(std::invalid_argument(
+                "measurement and conversion don't match, units do not match"));
+        }
+    }
+    // its::clearCustomCommodities();
+    return 0; // Non-zero return values are reserved for future use.
+}

docs/introduction/testing.rst

@@ -12,7 +12,7 @@ The units library has a series of units tests that are executed as part of the C
 
 1.  examples_test  A simple executable that loads up some different types of measurements and does a few checks directly,  it is mainly to test linking and has some useful features for helping with the code coverage measures.
 2.  fuzz_issue_tests  tests a set of past fuzzing failures, including, errors, glitches, timeouts, round trip failures, and some round trip failures with particular flags.
-3.  test_all_unit_base **DON'T RUN THIS TEST** it will take a very long time it does an exhaustive test of all possible unit bases to make sure the string conversion round trip works. I haven't actually executed it all yet.
+3.  test_all_unit_base **DO NOT RUN THIS TEST** it will take a very long time it does an exhaustive test of all possible unit bases to make sure the string conversion round trip works. I haven't actually executed it all yet.
 4.  test_commodities  Run test using the commodity related functions and operations on precise_unit's
 5.  test_conversions1 a series of tests about specific conversions, such as temperature, SI prefixes, extended SI units, and some other general operations about conversions
 6.  test_conversions2 run through a series of test units and conversion from one of the converter websites,  there are number of files that get used that contain known conversions
@@ -34,10 +34,10 @@ CI systems
 
 Travis-CI
 -----------
-1. Clang 5.0 build using C++11, and C++14 using header only
+1. Clang 5.0 build using C\++11, and C\++14 using header only
 2. Clang format style check
 3. Clang 3.5 C++11,
-4. GCC 7 C++11, C++14, C++17, Code coverage
+4. GCC 7 C\++11, C\++14, C\++17, Code coverage
 5. GCC 4.8 C++11
 
 Azure
@@ -59,12 +59,12 @@ Circle-CI
 1.  Clang 9,  Thread Sanitizer
 2.  Clang 9,  Address, undefined behavior sanitizer
 3.  Clang 9,  Memory Sanitizer
-4.  Clang 8,  Fuzzing library -- run a couple of defined fuzzing tests from scratch to check for any anomalous situations. It has been a while since this found anything but when started there were quite a few issues.  The Fuzzers run the from_string operations then convert back to a string and back again and check for mismatches or other issues with malformed strings.
+4.  Clang 8,  Fuzzing library -- run a couple of defined fuzzing tests from scratch to check for any anomalous situations. There are currently two fuzzers, the first test the units_from_string, and the second tests the measurement_from string.  It first converts the fuzzing sequence, then if it is a valid sequence, converts it to a string, then converts that string back to a measurement or unit and makes sure the two measurements or units are identical.  Any string sequence which doesn't work is captured and tested.  
 
 Codacy
 ---------
 C++ static analysis and checks
 
 Codecov
 ----------
-Try to maintain the library at 100% coverage. Currently showing at 100% coverage but a few missing operations that just don't show up on code coverage reports
+Try to maintain the library at 100% coverage. Missing header only constexpr functions do not show up on the report, so periodic manual scans are done to make sure we have full coverage.  

test/files/fuzz_issues/meas_fail1

@@ -0,0 +1 @@
+*r'

test/files/fuzz_issues/meas_fail2

@@ -0,0 +1 @@
+���'

test/files/fuzz_issues/meas_fail5

@@ -0,0 +1 @@
+-2)0

test/files/fuzz_issues/meas_fail6

@@ -0,0 +1 @@
+cindex}�

test/files/fuzz_issues/meas_fail7

@@ -0,0 +1 @@
+-��

test/files/fuzz_issues/meas_fail8

@@ -0,0 +1 @@
+/�]/�/�/�

test/fuzz_issue_tests.cpp

@@ -313,3 +313,33 @@ TEST_P(rtripflagProblems, rtripflagFiles)
 }
 
 INSTANTIATE_TEST_SUITE_P(rtripflagFiles, rtripflagProblems, ::testing::Range(1, 7));
+
+class measProblems : public ::testing::TestWithParam<int> {
+};
+
+TEST_P(measProblems, measFiles)
+{
+    auto cdata = loadFailureFile("meas_fail", GetParam());
+    auto m1 = measurement_from_string(cdata);
+    if (isnormal(m1)) {
+        auto str = to_string(m1);
+        auto m2 = measurement_from_string(str);
+        ASSERT_TRUE(m2.units().has_same_base(m1.units()) || isnormal(m2));
+        if (m2 == m1) {
+            EXPECT_EQ(m2, m1);
+            EXPECT_EQ(measurement_cast(m2), measurement_cast(m1));
+            EXPECT_FALSE(measurement_cast(m2) != measurement_cast(m1));
+        } else if (isnormal(root(m2, 2))) {
+            EXPECT_EQ(root(measurement_cast(m2), 2), root(measurement_cast(m1), 2));
+            EXPECT_FALSE(root(measurement_cast(m2), 2) != root(measurement_cast(m1), 2));
+        } else if (isnormal(root(m2, 3))) {
+            EXPECT_EQ(root(measurement_cast(m2), 3), root(measurement_cast(m1), 3));
+            EXPECT_FALSE(root(measurement_cast(m2), 3) != root(measurement_cast(m1), 3));
+        } else {
+            EXPECT_TRUE(measurement_cast(m2) == measurement_cast(m1));
+            EXPECT_FALSE(measurement_cast(m2) != measurement_cast(m1));
+        }
+    }
+}
+
+INSTANTIATE_TEST_SUITE_P(measFiles, measProblems, ::testing::Range(6, 7));

test/test_measurement.cpp

@@ -152,6 +152,19 @@ TEST(Measurement, powroot)
 
     measurement m4(16.0, m.pow(2));
     EXPECT_EQ(sqrt(m4), measurement(4.0, m));
+
+    measurement mneg(-0.25, m.pow(6));
+    EXPECT_FALSE(is_valid(root(mneg, 2)));
+    EXPECT_FALSE(is_valid(root(mneg, -2)));
+    EXPECT_FALSE(is_valid(root(mneg, 4)));
+    EXPECT_FALSE(is_valid(root(mneg, -4)));
+    EXPECT_FALSE(is_valid(root(mneg, 6)));
+    EXPECT_FALSE(is_valid(root(mneg, -6)));
+    EXPECT_TRUE(is_valid(root(mneg, 0)));
+    EXPECT_TRUE(is_valid(root(mneg, 1)));
+    EXPECT_TRUE(is_valid(root(mneg, -1)));
+    EXPECT_TRUE(is_valid(root(mneg, 3)));
+    EXPECT_TRUE(is_valid(root(mneg, -3)));
 #endif
 }
 

test/test_measurement_strings.cpp

@@ -78,4 +78,15 @@ TEST(MeasurementToString, test)
     // from google tests
     EXPECT_EQ(str1, "10 g/L");
     EXPECT_EQ(str2, "2.7 puMW");
+}
+
+TEST(MeasurementToString, unit_withNumbers)
+{
+    measurement ounit = 10.0 * unit(0.712412, kg.pow(2));
+    precise_measurement ounitp = 10.0 * precise_unit(0.712412, precise::kg.pow(2));
+    auto str1 = to_string(ounit);
+    auto str2 = to_string(ounitp);
+    // from google tests
+    EXPECT_TRUE(str1.compare(0, 11, "10 (0.71241") == 0);
+    EXPECT_TRUE(str2.compare(0, 11, "10 (0.71241") == 0);
 }

test/test_unit_strings.cpp

@@ -205,6 +205,16 @@ TEST(stringToUnits, Simple)
     EXPECT_EQ(precise::m, unit_from_string("meter"));
 }
 
+TEST(stringToUnits, to_default_unit)
+{
+    EXPECT_EQ(precise::defunit, unit_from_string("*"));
+    EXPECT_EQ(precise::defunit, unit_from_string("**"));
+    EXPECT_EQ(precise::defunit, unit_from_string("}"));
+    EXPECT_EQ(precise::defunit, unit_from_string("}()"));
+    EXPECT_EQ(precise::defunit, unit_from_string("***"));
+    EXPECT_EQ(precise::defunit, unit_from_string("*******"));
+}
+
 TEST(stringToUnits, Power)
 {
     EXPECT_EQ(precise::m.pow(2), unit_from_string("m^2"));