Update how we do our simulator builds to try to avoid a race when bui…

…lding the arduino core (#1371)

* Current Docker Desktop no longer accepts that trailing colon

* Make the firmwaredump plugin not explode when compiled against the simulator

This makes it easier to smoke test more stuff in the simulator

* Add a cache warmer to our test suite.

This will be run before any other tests when running simulator tests.
In theory, this might help prevent race conditions during simulator
testing in CI from causing test failures

* Remove an extra internal / from a path

* Build the cache warmer test before any other (including everything in the plugins directory)

Update the "tests failed" output to be a little bit more friendly

* There was a typo in a directory name that only bit us when building on
case-sensitive filesystems. (This is an older plugin not used in current code)

bin/run-docker

@@ -54,7 +54,7 @@ if [ -z "$DOCKER_LIVE_KALEIDOSCOPE_DIR" ]; then
 	_KALEIDOSCOPE_MOUNT="--mount type=tmpfs,destination=/kaleidoscope:exec"
 else
 	echo "Kaleidoscope is mounted read/write inside docker"
-	_KALEIDOSCOPE_MOUNT="-v $(pwd):/kaleidoscope:"
+	_KALEIDOSCOPE_MOUNT="-v $(pwd):/kaleidoscope"
 fi
 
 

plugins/Kaleidoscope-FirmwareDump/src/kaleidoscope/plugin/FirmwareDump.h

@@ -18,7 +18,7 @@
 #pragma once
 
 #ifdef KALEIDOSCOPE_VIRTUAL_BUILD
-#error "Firmware Dump is not available for virtual builds"
+#warning "Firmware Dump is not available for virtual builds"
 #else
 
 #ifdef __AVR__

plugins/Kaleidoscope-LEDEffect-BootAnimation/src/kaleidoscope/plugin/LEDEffect-BootAnimation.cpp

@@ -15,7 +15,7 @@
  * this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "Kaleidoscope/plugin/LEDEffect-BootAnimation.h"
+#include "Kaleidoscope-LEDEffect-BootAnimation.h"
 
 #include <Arduino.h>  // for PROGMEM, pgm_read_byte
 #include <stdint.h>   // for uint16_t, uint8_t

tests/.gitignore

@@ -1,3 +1,4 @@
 **/bin/
 **/lib/
 **/obj/
+_cache-warmer/warm-cache/generated-all-plugins.h

tests/Makefile

@@ -28,7 +28,7 @@ LIB_DIR		:= ${build_dir}/lib
 
 libcommon_a	:= ${top_dir}/_build/lib/libcommon.a
 
-TEST_PATH 	?= ./
+TEST_PATH 	?= .
 
 export FQBN	?= keyboardio:virtual:model01
 
@@ -46,29 +46,65 @@ include $(top_dir)/etc/makefiles/arduino-cli.mk
 KALEIDOSCOPE_ETC_DIR ?= $(top_dir)/etc
 
 .PHONY: all
-all: ${libcommon_a} googletest ${TESTS} 
+all: ${libcommon_a} googletest run-all
 	@:
 
 .PHONY: cmake-clean
 cmake-clean:
 	rm -rf "${top_dir}"/testing/googletest/build/*
 
-run-all: ${TESTS}
-	@for test in ${TESTS}; do \
-		${MAKE} -s -f ${top_dir}/testing/makefiles/testcase.mk \
-			-C $${test} \
-			testcase=$${test} run || ERROR=$$?; \
+.PHONY: run-all
+run-all: run-cache-warmer run-other-tests aggregate-test-results
+
+.PHONY: run-cache-warmer
+run-cache-warmer: generate-all-plugins-header $(filter ./_cache-warmer/%,${TESTS})
+
+.PHONY: run-other-tests
+run-other-tests: run-cache-warmer
+	@$(MAKE) $(filter-out ./_cache-warmer/%,${TESTS})
+
+
+${TESTS}: ${libcommon_a} googletest
+	$(QUIET) { \
+		$(MAKE) -s -f ${top_dir}/testing/makefiles/testcase.mk -C $@ testcase=$@ build run; \
+		if [ $$? -ne 0 ]; then \
+			hash=$$(echo "$$@" | shasum | cut -f1 -d' '); \
+			echo "$@ failed" > ${build_dir}/test_failures_$$hash.log; \
+		fi; \
+	}
+
+
+.PHONY: aggregate-test-results
+aggregate-test-results: run-other-tests
+	$(QUIET) TEST_ERRORS=0; \
+	for file in ${build_dir}/test_failures_*.log; do \
+		if [ -f "$$file" ]; then \
+			TEST_ERRORS=$$((TEST_ERRORS + 1)); \
+			cat "$$file" >> ${build_dir}/combined_test_failures.log; \
+			rm -f "$$file"; \
+		fi; \
 	done; \
-	if [ -n $${ERROR} ]; then exit $${ERROR}; fi
+	if [ -f ${build_dir}/combined_test_failures.log ]; then \
+		echo "Test failures: $$TEST_ERRORS"; \
+		cat ${build_dir}/combined_test_failures.log; \
+		rm -f ${build_dir}/combined_test_failures.log; \
+		exit 1; \
+	else \
+		echo "All tests passed."; \
+	fi
+
+
+
+
 
 .PHONY: clean
 clean: cmake-clean
-	@for test in ${TESTS}; do \
+	$(QUIET) for test in ${TESTS}; do \
 		${MAKE} -s -f ${top_dir}/testing/makefiles/testcase.mk \
 			-C $${test} \
 			testcase=$${test} clean; \
 	done
-	rm -rf "${build_dir}"/*
+	$(QUIET)  rm -rf "${build_dir}"/*
 
 .PHONY: googletest
 googletest: ${top_dir}/testing/googletest/build/Makefile
@@ -92,5 +128,19 @@ ${libcommon_a}:
 Makefile:
 	@:
 
-${TESTS}: ${libcommon_a} googletest
-	$(QUIET) ${MAKE} -f ${top_dir}/testing/makefiles/testcase.mk -C $@ testcase=$@ build run
+
+
+.PHONY: generate-all-plugins-header
+generate-all-plugins-header:
+	$(QUIET) echo "// This file is generated automatically. Do not edit." > _cache-warmer/warm-cache/generated-all-plugins.h
+	$(QUIET) for dir in ../plugins/*; do \
+		if [ -d "$$dir" ]; then \
+			header_path="$$dir/$$(basename $$dir).h"; \
+			header_src_path="$$dir/src/$$(basename $$dir).h"; \
+			header="$$(basename $$dir).h"; \
+			if [ -f "$$header_path" -o -f "$$header_src_path" ]; then \
+				echo '#include "'$$header'"' >> _cache-warmer/warm-cache/generated-all-plugins.h; \
+			fi; \
+		fi; \
+	done
+

tests/_cache-warmer/warm-cache/sketch.json

@@ -0,0 +1,6 @@
+{
+  "cpu": {
+    "fqbn": "keyboardio:virtual:model01",
+    "port": ""
+  }
+}

tests/_cache-warmer/warm-cache/sketch.yaml

@@ -0,0 +1 @@
+default_fqbn: keyboardio:virtual:model01

tests/_cache-warmer/warm-cache/test.ktest

@@ -0,0 +1,13 @@
+VERSION 1
+
+KEYSWITCH A 2 1
+KEYSWITCH S 2 2
+
+# ==============================================================================
+NAME Warm the cache
+
+RUN 4 ms
+PRESS A
+RUN 1 cycle
+EXPECT keyboard-report Key_A # Key A should be pressed
+

tests/_cache-warmer/warm-cache/warm-cache.ino

@@ -0,0 +1,46 @@
+/* -*- mode: c++ -*-
+ * Copyright (C) 2020  Keyboard.io, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <Kaleidoscope.h>
+#include "generated-all-plugins.h"
+// *INDENT-OFF*
+KEYMAPS(
+    [0] = KEYMAP_STACKED
+    (
+        ___, ___, ___, ___, ___, ___, ___,
+        ___, ___, ___, ___, ___, ___, ___,
+        ___, Key_A, Key_S, ___, ___, ___,
+        ___, ___, ___, ___, ___, ___, ___,
+        ___, ___, ___, ___,
+        ___,
+
+        ___, ___, ___, ___, ___, ___, ___,
+        ___, ___, ___, ___, ___, ___, ___,
+        ___, ___, ___, ___, ___, ___,
+        ___, ___, ___, ___, ___, ___, ___,
+        ___, ___, ___, ___,
+        ___
+    ),
+)
+// *INDENT-ON*
+
+void setup() {
+  Kaleidoscope.setup();
+}
+
+void loop() {
+  Kaleidoscope.loop();
+}