new features: fixes for #1, #2, #3, #4 and #5.

m-mcgowan · Aug 17, 2014 · 579d5da · 579d5da
1 parent cee8730
commit 579d5da
Show file tree

Hide file tree

Showing 4 changed files with 294 additions and 32 deletions.
diff --git a/README.md b/README.md
@@ -1,3 +1,110 @@
-This is a port of the [Arduino Unit](https://github.com/mmurdoch/arduinounit) library to the spark. Please see that repo for library usage, or check out the examples.
+This is a port of the [Arduino Unit](https://github.com/mmurdoch/arduinounit) library to the spark. 
+Please see that repo for library usage, or check out the examples.
+
+Additions in the spark version:
+
+- when running, the RGB led indicates the overall test health
+- configuration at runtime of tests to include or exclude, via `cmd(include=pattern)`
+- waits to run tests until either 't' is received over serial, or the Cloud `cmd` function is called with 'start'
+- test stats (passed/failed/skipped/count) available as cloud variables 
+- test running state (waiting, running, complete) available as a variable, 
+and changes to running state are published as events.
+
+These features are documented below.
+
+RGB Led
+-------
+
+When the test suite app executes, it initially enters the waiting state. 
+The RGB led shows the usual breathing cyan as the spark is connected to the cloud.
+
+When the test suite is later started, the RGB led shows a solid color to reflect
+the current test health:
+
+ - green: all tests (so far) have passed
+ - orange: some tests skipped but otherwise all passed
+ - red: some tests failed
+
+When the test suite has completed running, the LED status remains so you can leave the spark
+running and come back when it's done to see the result.
+(In future, the LED may blink while the tests are running, and then return to solid when all tests are done, 
+to make it easy to see when the tests are still running or not.)
+
+
+Including/Excluding Tests
+-------------------------
+
+Tests can be filtered at runtime using include/exclude globs. The include/exclude
+feature is described in the [Arduino unit documentation](https://github.com/mmurdoch/arduinounit#selecting-tests).
+
+To exclude tests ending "*_slow":
+
+ `spark call mycore cmd exclude=*_slow`
+
+To run only tests starting with 'abc':
+
+ `spark call mycore cmd exclude=*`
+ `spark call mycore cmd include=abc*`
+
+
+Starting the Test Suite
+-----------------------
+If the test code hasn't requested the test suite starts immediately, the test suite
+will wait until:
+
+- a 't' is sent over Serial
+- the `cmd(start)` function is called.
+
+E.g. to start the tests via the cloud:
+
+ `spark call mycore cmd start`
+
+
+Enter DFU Mode
+--------------
+As part of a locally operating automated tset suite, it can be useful to 
+put the core in DFU mode for quick flashing of another app.
+
+This is done with the command
+
+ `spark call mycore cmd enterDFU`
+
+On receiving the command, the core will reset and enter DFU mode (flashing yellow LED.)
+
+Variables for Test Statistics
+-----------------------------
+
+The test suite exposes variables to the cloud to allow monitoring of the test health:
+
+- count (int32): the number of tests to be run (available from startup)
+- state (int32): the state of the test runner
+ - 1: waiting to start running tests
+ - 2: running - busy executing tests
+ - 3: complete - all tests executed
+- passed (int32): the number of tests passed so far.
+- failed (int32): the number of tests failed so far.
+- skipped (int32): the number of tests skipped so far.
+
+These variables are updated as the test suite executes. 
+
+Events
+------
+
+The test suite publishes events as the test runner state changes. This allows
+external apps to monitor progress, e.g. wait for the test suite to complete.
+
+The event `state` is published whenever the running state of the test suite changes. 
+The event can have these values:
+
+- waiting: waiting to start
+- running: executing tests
+- complete: all tests executed
+
+(NB: it would be easy in code to add an event each time a test has been executed, however
+this could easily generate too many events causing more important events, such as the
+running state change 'complete', to be dropped.)
+
+
+
 
 
diff --git a/firmware/examples/example.cpp b/firmware/examples/example.cpp
@@ -28,24 +28,4 @@ test(HelloWorld) {
 	assertNotEqual("Hello", "World");
 }
 
-void setup()
-{
-	Serial.begin(9600);
-}
-
-void loop()
-{
-	static bool run = false;
-    if (Serial.available()) {
-        char c = Serial.read();
-        if (c=='t') {
-            if (!run) {
-                Serial.println("Running tests");
-                run = true;
-            }
-        }
-    }
-    if (run) {
-        Test::run();
-    }
-}
+UNIT_TEST_APP()
diff --git a/firmware/unit-test.h b/firmware/unit-test.h
@@ -31,6 +31,85 @@
 #define PROGMEM
 #define PSTR
 
+enum RunnerState {
+    INIT,
+    WAITING,
+    RUNNING,
+    COMPLETE
+};
+
+class SparkTestRunner {
+
+private:
+    int _state;
+
+public:
+    SparkTestRunner() : _state(INIT) {
+
+    }
+
+    void begin();        
+
+    bool isStarted() {
+        return _state>=RUNNING;
+    }
+
+    void start() {
+        if (!isStarted())
+            setState(RUNNING);
+    }
+
+    const char* nameForState(RunnerState state) {
+        switch (state) {
+            case INIT: return "init";
+            case WAITING: return "waiting";
+            case RUNNING: return "running";
+            case COMPLETE: return "complete";
+            default:
+                return "";
+        }
+    }
+
+    int testStatusColor();
+
+    void updateLEDStatus() {
+        int rgb = testStatusColor();
+        RGB.control(true);
+        //RGB.color(rgb);
+        LED_SetSignalingColor(rgb);
+        LED_On(LED_RGB);
+    }
+
+    RunnerState state() const { return (RunnerState)_state; }
+    void setState(RunnerState newState) {
+        if (newState!=_state) {            
+            _state = newState;
+            const char* stateName = nameForState((RunnerState)_state);
+            if (isStarted())
+                updateLEDStatus();
+            Spark.publish("state", stateName);
+        }
+    }
+
+    void testDone() {        
+        updateLEDStatus();
+        delay(500);
+    }
+};
+
+SparkTestRunner _runner;
+
+
+
+#define UNIT_TEST_SETUP() \
+    void setup() { unit_test_setup(); }
+
+#define UNIT_TEST_LOOP() \
+    void loop() { unit_test_loop(); }
+
+#define UNIT_TEST_APP() \
+    UNIT_TEST_SETUP(); UNIT_TEST_LOOP();
+
 
 /*
 Copyright (c) 2014 Matt Paine
@@ -776,6 +855,8 @@ Variables you might want to adjust:
 */
 class Test
 {
+    friend class SparkTestRunner;
+
  private:
   // allows for both ram/progmem based names
   class TestString : public Printable {
@@ -795,10 +876,10 @@ class Test
   Test *next;
 
   // static statistics for tests
-  static uint16_t passed;
-  static uint16_t failed;
-  static uint16_t skipped;
-  static uint16_t count;
+  static uint32_t passed;
+  static uint32_t failed;
+  static uint32_t skipped;
+  static uint32_t count;
 
   void resolve();
   void remove();
@@ -1265,10 +1346,10 @@ bool Test::TestString::matches(const char *pattern) const {
 Test* Test::root = 0;
 Test* Test::current = 0;
 
-uint16_t Test::count = 0;
-uint16_t Test::passed = 0;
-uint16_t Test::failed = 0;
-uint16_t Test::skipped = 0;
+uint32_t Test::count = 0;
+uint32_t Test::passed = 0;
+uint32_t Test::failed = 0;
+uint32_t Test::skipped = 0;
 uint8_t Test::max_verbosity = TEST_VERBOSITY_ALL;
 uint8_t Test::min_verbosity = TEST_VERBOSITY_TESTS_SUMMARY;
 
@@ -1285,6 +1366,8 @@ void Test::resolve()
     if (pass) ++Test::passed;
     if (fail) ++Test::failed;
     if (skip) ++Test::skipped;
+
+    _runner.testDone();
 
 #if TEST_VERBOSITY_EXISTS(TESTS_SKIPPED) || TEST_VERBOSITY_EXISTS(TESTS_PASSED) || TEST_VERBOSITY_EXISTS(TESTS_FAILED)
 
@@ -1322,7 +1405,7 @@ void Test::resolve()
     out->print(F(" skipped, out of "));
     out->print(count);
     out->println(F(" test(s)."));
-  }
+  }  
 #endif
 }
 
@@ -1365,6 +1448,8 @@ void Test::setup() {};
 
 void Test::run()
 {
+  _runner.setState(root ? RUNNING : COMPLETE);
+
   for (Test **p = &root; (*p) != 0; ) {
     current = *p;
 
@@ -1383,6 +1468,7 @@ void Test::run()
     } else {
       p=&((*p)->next);
     }
+    break;  // allow main loop to execute
   }
 }
 
@@ -1453,4 +1539,93 @@ bool isMore<const char*>(const char* const &a, const char* const &b)
   return (strcmp(a,b) > 0);
 }
 
+/**
+ * A convenience method to setup serial.
+ */
+void unit_test_setup()
+{
+    Serial.begin(9600);
+    _runner.begin();
+}
+
+bool requestStart = false;
+bool _enterDFU = false;
+
+bool isStartRequested(bool runImmediately) {
+    if (runImmediately || requestStart)
+        return true;
+    if (Serial.available()) {
+        char c = Serial.read();
+        if (c=='t') {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+void enterDFU() {
+    FLASH_OTA_Update_SysFlag = 0x0000;
+    Save_SystemFlags();
+    BKP_WriteBackupRegister(BKP_DR10, 0x0000);
+    USB_Cable_Config(DISABLE);
+    NVIC_SystemReset();
+}
+
+/*
+ * A convenience method to run tests as part of the main loop after a character
+ * is received over serial.
+ **/
+void unit_test_loop(bool runImmediately=false)
+{       
+    if (_enterDFU)
+        enterDFU();
+
+    if (!_runner.isStarted() && isStartRequested(runImmediately)) {
+        Serial.println("Running tests");
+        _runner.start();
+    }
+
+    if (_runner.isStarted()) {
+        Test::run();
+    }
+}
+
+int SparkTestRunner::testStatusColor() {
+    if (Test::failed>0)
+        return RGB_COLOR_RED;
+    else if (Test::skipped>0)
+        return RGB_COLOR_ORANGE;
+    else
+        return RGB_COLOR_GREEN;
+}
+
+int testCmd(String arg) {
+    int result = -1;
+    if (arg.equals("start")) {
+        requestStart = true;
+        result = 0;
+    }
+    else if (arg.startsWith("exclude=")) {
+        String pattern = arg.substring(8);
+        Test::exclude(pattern.c_str());
+    }
+    else if (arg.startsWith("include=")) {
+        String pattern = arg.substring(8);
+        Test::include(pattern.c_str());
+    }
+    else if (arg.equals("enterDFU")) {
+        _enterDFU = true;
+    }
+    return result;
+}
 
+void SparkTestRunner::begin() {    
+    Spark.variable("passed", &Test::passed, INT);
+    Spark.variable("failed", &Test::failed, INT);
+    Spark.variable("skipped", &Test::skipped, INT);
+    Spark.variable("count", &Test::count, INT);
+    Spark.variable("state", &_state, INT);
+    Spark.function("cmd", testCmd);
+    setState(WAITING);
+}