From ce0597601a4726695bea8d2a29901be362c8a72e Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 21 Jul 2023 15:47:39 +1000 Subject: [PATCH 001/176] Refactor tests to use an event style --- src/PowerPlant.cpp | 3 +- src/PowerPlant.ipp | 3 +- tests/CMakeLists.txt | 7 +- tests/dsl/Always.cpp | 67 ++++++++++------ tests/dsl/ArgumentFission.cpp | 122 +++++++++++------------------ tests/dsl/BlockNoData.cpp | 57 +++++++++----- tests/dsl/CommandLineArguments.cpp | 46 ++++++----- tests/dsl/CustomGet.cpp | 40 +++++++--- tests/dsl/DSLProxy.cpp | 50 ++++++++---- tests/test_util/TestBase.hpp | 65 +++++++++++++++ tests/test_util/diff_string.cpp | 80 +++++++++++++++++++ tests/test_util/diff_string.hpp | 40 ++++++++++ tests/test_util/lcs.hpp | 115 +++++++++++++++++++++++++++ 13 files changed, 521 insertions(+), 174 deletions(-) create mode 100644 tests/test_util/TestBase.hpp create mode 100644 tests/test_util/diff_string.cpp create mode 100644 tests/test_util/diff_string.hpp create mode 100644 tests/test_util/lcs.hpp diff --git a/src/PowerPlant.cpp b/src/PowerPlant.cpp index bd03288e2..961c04d50 100644 --- a/src/PowerPlant.cpp +++ b/src/PowerPlant.cpp @@ -37,8 +37,9 @@ void PowerPlant::start() { // We are now running is_running.store(true); - // Direct emit startup event + // Direct emit startup event and command line arguments emit(std::make_unique()); + emit_shared(dsl::store::DataStore::get()); // Start all of the threads scheduler.start(); diff --git a/src/PowerPlant.ipp b/src/PowerPlant.ipp index 349584b3b..d1b0dce57 100644 --- a/src/PowerPlant.ipp +++ b/src/PowerPlant.ipp @@ -45,9 +45,8 @@ inline PowerPlant::PowerPlant(Configuration config, int argc, const char* argv[] args.emplace_back(argv[i]); } - // We emit this twice, so the data is available for extensions + // We emit this twice, so the data is available for extensions, it will also be emitted in start emit(std::make_unique(args)); - emit(std::make_unique(args)); } template diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f4d7b7112..491af31c0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -29,7 +29,9 @@ if(CATCH_FOUND) add_compile_options(-Wall -Wextra -pedantic -Werror) endif(MSVC) - file(GLOB test_src test.cpp "api/*.cpp" "dsl/*.cpp" "dsl/emit/*.cpp" "log/*.cpp") + add_compile_definitions(CATCH_CONFIG_CONSOLE_WIDTH=120) + + file(GLOB test_src test.cpp "api/*.cpp" "dsl/*.cpp" "dsl/emit/*.cpp" "log/*.cpp" "test_util/*.cpp") # Some tests must be executed as individual binaries file(GLOB individual_tests "${CMAKE_CURRENT_SOURCE_DIR}/individual/*.cpp") @@ -40,6 +42,7 @@ if(CATCH_FOUND) add_executable(${test_name} ${test_src}) target_link_libraries(${test_name} NUClear::nuclear) set_target_properties(${test_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/individual") + target_include_directories(${test_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories( ${test_name} SYSTEM PRIVATE ${CATCH_INCLUDE_DIRS} ${PROJECT_BINARY_DIR}/include "${PROJECT_SOURCE_DIR}/src" @@ -51,6 +54,7 @@ if(CATCH_FOUND) endforeach(test_src) add_executable(test_nuclear ${test_src}) + target_include_directories(test_nuclear PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(test_nuclear NUClear::nuclear) target_include_directories( test_nuclear SYSTEM @@ -60,6 +64,7 @@ if(CATCH_FOUND) add_executable(test_network networktest.cpp) target_link_libraries(test_network NUClear::nuclear) + target_include_directories(test_network PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories( test_network SYSTEM PRIVATE ${CATCH_INCLUDE_DIRS} ${PROJECT_BINARY_DIR}/include "${PROJECT_SOURCE_DIR}/src" diff --git a/tests/dsl/Always.cpp b/tests/dsl/Always.cpp index 293195ae1..8da97425f 100644 --- a/tests/dsl/Always.cpp +++ b/tests/dsl/Always.cpp @@ -19,52 +19,67 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { -struct BlankMessage {}; -int i = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -bool emitted_message = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +struct SimpleMessage {}; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { on().then([this] { - // Run until it's 11 then emit the blank message - if (i > 10) { - if (!emitted_message) { - emitted_message = true; - emit(std::make_unique()); - } - } - else { + if (i < 10) { + events.push_back("Always " + std::to_string(i)); ++i; } + else if (i == 10) { + emit(std::make_unique()); + } }); - on>().then([this] { - if (i == 11) { - ++i; - powerplant.shutdown(); - } + on>().then([this] { + events.push_back("Always with SimpleMessage " + std::to_string(i)); + + // We need to shutdown manually as the default pool will always be idle + powerplant.shutdown(); }); } + + /// Counter for the number of times we have run + int i = 0; }; } // namespace -TEST_CASE("Testing on functionality (permanent run)", "[api][always]") { +TEST_CASE("The Always DSL keyword runs continuously when it can", "[api][always]") { NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); + plant.install(); + plant.start(); - // We are installing with an initial log level of debug - plant.install(); - - plant.emit(std::make_unique(5)); + std::vector expected = { + "Always 0", + "Always 1", + "Always 2", + "Always 3", + "Always 4", + "Always 5", + "Always 6", + "Always 7", + "Always 8", + "Always 9", + "Always with SimpleMessage 10", + }; - plant.start(); + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); - REQUIRE(emitted_message); - REQUIRE(i == 12); + // Check the events fired in order and only those events + REQUIRE(events == expected); } diff --git a/tests/dsl/ArgumentFission.cpp b/tests/dsl/ArgumentFission.cpp index f358d0886..675eac8b5 100644 --- a/tests/dsl/ArgumentFission.cpp +++ b/tests/dsl/ArgumentFission.cpp @@ -15,86 +15,58 @@ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ - #include #include #include +#include "test_util/TestBase.hpp" + namespace { -struct BindExtensionTest1 { - static int val1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - static double val2; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +struct BindExtensionTest1 { template - static inline int bind(const std::shared_ptr& /*unused*/, int v1, double v2) { - - val1 = v1; - val2 = v2; - + static inline int bind(const std::shared_ptr& /*unused*/, int v1, bool v2) { + events.push_back("Bind1 with " + std::to_string(v1) + " and " + (v2 ? "true" : "false") + " called"); return 5; } }; -int BindExtensionTest1::val1 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -double BindExtensionTest1::val2 = 0.0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - struct BindExtensionTest2 { - - static std::string val1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - static std::chrono::nanoseconds val2; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - template - static inline double bind(const std::shared_ptr& /*reaction*/, - std::string v1, - std::chrono::nanoseconds v2) { - - val1 = std::move(v1); - val2 = v2; - - return 7.2; + static inline bool bind(const std::shared_ptr& /*reaction*/, + std::string v1, + std::chrono::nanoseconds v2) { + events.push_back("Bind2 with " + v1 + " and " + std::to_string(v2.count()) + " called"); + return true; } }; -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -std::string BindExtensionTest2::val1 = {}; -// NOLINTNEXTLINE(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) -std::chrono::nanoseconds BindExtensionTest2::val2 = std::chrono::nanoseconds(0); - struct BindExtensionTest3 { - - static int val1; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - static int val2; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - static int val3; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - template - static inline NUClear::threading::ReactionHandle - bind(const std::shared_ptr& /*reaction*/, int v1, int v2, int v3) { - - val1 = v1; - val2 = v2; - val3 = v3; - - return {nullptr}; + static inline std::string bind(const std::shared_ptr& /*reaction*/, + int v1, + int v2, + int v3) { + events.push_back("Bind3 with " + std::to_string(v1) + ", " + std::to_string(v2) + " and " + std::to_string(v3) + + " called"); + return "return from Bind3"; } }; -int BindExtensionTest3::val1 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int BindExtensionTest3::val2 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int BindExtensionTest3::val3 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -struct ShutdownFlag {}; - -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - int a = 0; - double b = 0.0; + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { + int a = 0; + bool b = 0.0; + std::string c; - // Run all three of our extension tests - std::tie(std::ignore, a, b, std::ignore) = + // Bind all three functions to test fission + std::tie(std::ignore, a, b, c) = on(5, - 7.9, + false, "Hello", std::chrono::seconds(2), 9, @@ -102,27 +74,9 @@ class TestReactor : public NUClear::Reactor { 11) .then([] {}); - // Check the returns from the bind - REQUIRE(a == 5); - REQUIRE(b == 7.2); - - REQUIRE(BindExtensionTest1::val1 == 5); - REQUIRE(BindExtensionTest1::val2 == 7.9); - - REQUIRE(BindExtensionTest2::val1 == "Hello"); - REQUIRE(BindExtensionTest2::val2.count() == std::chrono::nanoseconds(2 * std::nano::den).count()); - - REQUIRE(BindExtensionTest3::val1 == 9); - REQUIRE(BindExtensionTest3::val2 == 10); - REQUIRE(BindExtensionTest3::val3 == 11); - - // Run a test when there are blanks in the list before filled elements - on, BindExtensionTest1>(2, 3.3).then([] {}); - - on>().then([this] { - // We are finished the test - powerplant.shutdown(); - }); + events.push_back("Bind1 returned " + std::to_string(a)); + events.push_back(std::string("Bind2 returned ") + (b ? "true" : "false")); + events.push_back("Bind3 returned " + c); } }; } // namespace @@ -133,8 +87,20 @@ TEST_CASE("Testing distributing arguments to multiple bind functions (NUClear Fi config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); + plant.start(); - plant.emit(std::make_unique()); + std::vector expected = { + "Bind1 with 5 and false called", + "Bind2 with Hello and 2000000000 called", + "Bind3 with 9, 10 and 11 called", + "Bind1 returned 5", + "Bind2 returned true", + "Bind3 returned return from Bind3", + }; - plant.start(); + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } diff --git a/tests/dsl/BlockNoData.cpp b/tests/dsl/BlockNoData.cpp index 59b86401b..858e20f61 100644 --- a/tests/dsl/BlockNoData.cpp +++ b/tests/dsl/BlockNoData.cpp @@ -19,49 +19,66 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { -struct SimpleMessage {}; + +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) struct MessageA {}; struct MessageB {}; -MessageA* a = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { + on>().then([this] { + events.push_back("MessageA triggered"); + events.push_back("Emitting MessageB"); + emit(std::make_unique()); + }); - on>().then([this] { - auto data = std::make_unique(); - a = data.get(); + on, With>().then([] { // + events.push_back("MessageA with MessageB triggered"); + }); - // Emit required data - emit(data); + on, With>().then([] { // + events.push_back("MessageB with MessageA triggered"); + }); - // Since the data was emitted before the shutdown call it will be processed before total shutdown - powerplant.shutdown(); + on>>().then([this] { + events.push_back("Emitting MessageA"); + emit(std::make_unique()); }); - on, With>().then( - [](const MessageA&, const MessageB&) { FAIL("B was never emitted so this should not be possible"); }); + on().then([this] { + // Emit some messages with data + emit(std::make_unique>()); + }); } }; } // namespace -TEST_CASE("Testing that when a trigger does not have it's data satisfied it does not run", "[api][nodata]") { +TEST_CASE("Testing that when an on statement does not have it's data satisfied it does not run", "[api][nodata]") { NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); + plant.start(); - auto message = std::make_unique(); - - plant.emit(message); + std::vector expected = { + "Emitting MessageA", + "MessageA triggered", + "Emitting MessageB", + "MessageB with MessageA triggered", + }; - plant.start(); + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); - REQUIRE(a != nullptr); + // Check the events fired in order and only those events + REQUIRE(events == expected); } diff --git a/tests/dsl/CommandLineArguments.cpp b/tests/dsl/CommandLineArguments.cpp index 1a9f7955b..7ef287ccd 100644 --- a/tests/dsl/CommandLineArguments.cpp +++ b/tests/dsl/CommandLineArguments.cpp @@ -18,27 +18,29 @@ #include #include +#include + +#include "test_util/TestBase.hpp" namespace { -struct ShutdownNowPlx {}; +// Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +class TestReactor : public test_util::TestBase { +private: + using CommandLineArguments = NUClear::message::CommandLineArguments; -class TestReactor : public NUClear::Reactor { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - on>().then( - [this](const NUClear::message::CommandLineArguments& args) { - REQUIRE(args[0] == "Hello"); - REQUIRE(args[1] == "World"); - - // We can't call shutdown here because - // we haven't started yet. That's because - // emits from Scope::INITIALIZE are not - // considered fully "initialized" - emit(std::make_unique()); - }); - - on>().then([this] { powerplant.shutdown(); }); + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { + + on>().then([](const CommandLineArguments& args) { + std::stringstream output; + for (auto& arg : args) { + output << arg << " "; + } + events.push_back("CommandLineArguments: " + output.str()); + }); } }; } // namespace @@ -46,11 +48,17 @@ class TestReactor : public NUClear::Reactor { TEST_CASE("Testing the Command Line argument capturing", "[api][command_line_arguments]") { const int argc = 2; const char* argv[] = {"Hello", "World"}; // NOLINT(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) - NUClear::PowerPlant::Configuration config; config.thread_count = 1; - NUClear::PowerPlant plant(config, argc, reinterpret_cast(argv)); + NUClear::PowerPlant plant(config, argc, argv); plant.install(); - plant.start(); + + std::vector expected = {"CommandLineArguments: Hello World "}; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } diff --git a/tests/dsl/CustomGet.cpp b/tests/dsl/CustomGet.cpp index a0b48a3c0..bbf5304fa 100644 --- a/tests/dsl/CustomGet.cpp +++ b/tests/dsl/CustomGet.cpp @@ -19,29 +19,34 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { -struct CustomGet : public NUClear::dsl::operation::TypeBind { +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +struct CustomGet : public NUClear::dsl::operation::TypeBind { template - static inline std::shared_ptr get(NUClear::threading::Reaction& /*unused*/) { - return std::make_shared(5); + static inline std::shared_ptr get(NUClear::threading::Reaction& /*unused*/) { + return std::make_shared("Data from a custom getter"); } }; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - on().then([this](const int& x) { - REQUIRE(x == 5); - - powerplant.shutdown(); + on().then([](const std::string& x) { // + events.push_back("CustomGet Triggered"); + events.push_back(x); }); on().then([this] { - // Emit from message 4 to 1 - emit(std::make_unique(10)); + // Emit a CustomGet instance to trigger the reaction + events.push_back("Emitting CustomGet"); + emit(std::make_unique()); }); } }; @@ -53,6 +58,17 @@ TEST_CASE("Test a custom reactor that returns a type that needs dereferencing", config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + std::vector expected = { + "Emitting CustomGet", + "CustomGet Triggered", + "Data from a custom getter", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } diff --git a/tests/dsl/DSLProxy.cpp b/tests/dsl/DSLProxy.cpp index 188f4bbd4..b6db4556f 100644 --- a/tests/dsl/DSLProxy.cpp +++ b/tests/dsl/DSLProxy.cpp @@ -19,13 +19,23 @@ #include #include +#include "test_util/TestBase.hpp" + +namespace { +struct CustomMessage1 {}; +struct CustomMessage2 { + CustomMessage2(int value) : value(value) {} + int value; +}; +} // namespace + namespace NUClear { namespace dsl { namespace operation { template <> - struct DSLProxy - : public NUClear::dsl::operation::TypeBind - , public NUClear::dsl::operation::CacheGet + struct DSLProxy + : public NUClear::dsl::operation::TypeBind + , public NUClear::dsl::operation::CacheGet , public NUClear::dsl::word::Single {}; } // namespace operation } // namespace dsl @@ -33,24 +43,25 @@ namespace dsl { namespace { -class TestReactor : public NUClear::Reactor { -public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - on().then([this](const double& d) { - // The message we received should have test == 10 - REQUIRE(d == 4.4); +class TestReactor : public test_util::TestBase { +public: + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - // We are finished the test - powerplant.shutdown(); + on().then([](const CustomMessage2& d) { + events.push_back("CustomMessage1 Triggered with " + std::to_string(d.value)); }); on().then([this]() { // Emit a double we can get - emit(std::make_unique(4.4)); + events.push_back("Emitting CustomMessage2"); + emit(std::make_unique(123456)); - // Emit an integer to trigger the reaction - emit(std::make_unique()); + // Emit a custom message 1 to trigger the reaction + events.push_back("Emitting CustomMessage1"); + emit(std::make_unique()); }); } }; @@ -62,6 +73,15 @@ TEST_CASE("Testing that the DSL proxy works as expected for binding unmodifyable config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + std::vector expected = {"Emitting CustomMessage2", + "Emitting CustomMessage1", + "CustomMessage1 Triggered with 123456"}; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } diff --git a/tests/test_util/TestBase.hpp b/tests/test_util/TestBase.hpp new file mode 100644 index 000000000..bb2ec25f5 --- /dev/null +++ b/tests/test_util/TestBase.hpp @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef TEST_UTIL_TESTBASE_HPP +#define TEST_UTIL_TESTBASE_HPP + +#include +#include +#include + +#include "test_util/diff_string.hpp" + +namespace test_util { + +template +class TestBase : public NUClear::Reactor { +public: + /** + * @brief Struct to use to emit each step of the test, by doing each step in a separate reaction with low priority, + * it will ensure that everything has finished changing before the next step is run + * + * @tparam i the number of the step + */ + template + struct Step {}; + + /** + * @brief Struct to handle shutting down the powerplant when the system is idle (i.e. the unit test(s) are finished) + */ + struct ShutdownOnIdle {}; + + explicit TestBase(std::unique_ptr environment, const bool& shutdown_on_idle = true) + : Reactor(std::move(environment)) { + + // Shutdown if the system is idle + on, Priority::IDLE>().then([this] { powerplant.shutdown(); }); + on().then([this, shutdown_on_idle] { + if (shutdown_on_idle) { + emit(std::make_unique()); + } + }); + + // Timeout if the test doesn't complete in time + on>().then([this] { powerplant.shutdown(); }); + } +}; + +} // namespace test_util + +#endif // TEST_UTIL_TESTBASE_HPP diff --git a/tests/test_util/diff_string.cpp b/tests/test_util/diff_string.cpp new file mode 100644 index 000000000..8e90c850e --- /dev/null +++ b/tests/test_util/diff_string.cpp @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "diff_string.hpp" + +#include "lcs.hpp" + +namespace test_util { + +std::string diff_string(const std::vector& expected, const std::vector& actual) { + + // Find the longest string in each side or "Expected" and "Actual" if those are the longest + auto len = [](const std::string& a, const std::string& b) { return a.size() < b.size(); }; + auto max_a_it = std::max_element(expected.begin(), expected.end(), len); + auto max_b_it = std::max_element(actual.begin(), actual.end(), len); + int max_a = std::max(int(std::strlen("Expected")), max_a_it != expected.end() ? int(max_a_it->size()) : 0); + int max_b = std::max(int(std::strlen("Actual")), max_b_it != actual.end() ? int(max_b_it->size()) : 0); + + // Start with a header + std::string output = std::string("Expected") + std::string(max_a - 8, ' ') + " | " + std::string("Actual") + + std::string(max_b - 6, ' ') + "\n"; + + // Print a divider characters for a divider + output += std::string(9 + max_a + max_b, '-') + "\n"; + + auto matches = lcs(expected, actual); + auto& match_a = matches.first; + auto& match_b = matches.second; + int i_a = 0; + int i_b = 0; + while (i_a < int(expected.size()) && i_b < int(actual.size())) { + if (match_a[i_a]) { + if (match_b[i_b]) { + output += expected[i_a] + std::string(max_a - int(expected[i_a].size()), ' ') + " <-> " + + actual[i_b] + std::string(max_b - int(actual[i_b].size()), ' ') + "\n"; + i_a++; + i_b++; + } + else { + output += std::string(max_a, ' ') + " <-> " + actual[i_b] + + std::string(max_b - int(actual[i_b].size()), ' ') + "\n"; + i_b++; + } + } + else { + output += expected[i_a] + std::string(max_a - int(expected[i_a].size()), ' ') + " <-> " + + std::string(max_b, ' ') + "\n"; + i_a++; + } + } + while (i_a < int(expected.size())) { + output += expected[i_a] + std::string(max_a - int(expected[i_a].size()), ' ') + " <-> " + + std::string(max_b, ' ') + "\n"; + i_a++; + } + while (i_b < int(actual.size())) { + output += std::string(max_a, ' ') + " <-> " + actual[i_b] + + std::string(max_b - int(actual[i_b].size()), ' ') + "\n"; + i_b++; + } + + return output; +} + +} // namespace test_util diff --git a/tests/test_util/diff_string.hpp b/tests/test_util/diff_string.hpp new file mode 100644 index 000000000..a789f39d8 --- /dev/null +++ b/tests/test_util/diff_string.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef TEST_UTIL_DIFF_STRING_HPP +#define TEST_UTIL_DIFF_STRING_HPP + +#include +#include + +namespace test_util { + +/** + * Using an LCS algorithm prints out the two sets of string (expected and actual) side by side to show the + * differences + * + * @param expected the expected series of events + * @param actual the actual series of events + * + * @return a multiline string showing a human output of the difference + */ +std::string diff_string(const std::vector& expected, const std::vector& actual); + +} // namespace test_util + +#endif // TEST_UTIL_DIFF_STRING_HPP diff --git a/tests/test_util/lcs.hpp b/tests/test_util/lcs.hpp new file mode 100644 index 000000000..c9ade2129 --- /dev/null +++ b/tests/test_util/lcs.hpp @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef TEST_UTIL_LCS_HPP +#define TEST_UTIL_LCS_HPP + +#include +#include +#include + +namespace test_util { + +/** + * Longest common subsequence algorithm that returns which elements from a and b form the common subsequence. + * + * This algorithm compares two lists and finds the longest subsequence you can make that is included in both of the + * lists. It returns a pair of bool vectors that indicate which elements are a part of the subsequence. + * + * @tparam T the type of the elements to be compared + * + * @param a the first list to compare + * @param b the second list to compare + * + * @return two vectors of bools indicating which elements participate in the longest common subexpression + */ +template +std::pair, std::vector> lcs(const std::vector& a, const std::vector& b) { + + // Start with nothing matching + std::vector match_a(a.size(), false); + std::vector match_b(b.size(), false); + + // Nothing matches if one is empty + if (a.empty() || b.empty()) { + return std::make_pair(match_a, match_b); + } + + // Directions matrix for dynamic algorithm + // 0x1 = diagonal, 0x2 = left, 0x4 = top + std::vector> directions(a.size(), std::vector(b.size(), 0)); + + const int insert_weight = 3; + std::vector last_weights(a.size(), 0); + std::vector curr_weights(a.size(), 0); + for (int i = 0, weight = insert_weight; i < int(a.size()); ++i, weight += insert_weight) { + last_weights[i] = weight; + } + + for (int y = 0; y < int(b.size()); ++y) { + for (int x = 0; x < int(a.size()); ++x) { + // Calculate the weights + int weight_from_left = x == 0 ? (y + 2) * insert_weight : curr_weights[x - 1] + insert_weight; + int weight_from_top = last_weights[x] + insert_weight; + int weight_from_diagonal = + a[x] == b[y] ? (x == 0 ? (y + 1) * insert_weight : last_weights[x - 1]) : 0x7FFFFFFF; + + // Find the smallest weight + int min_weight = std::min(std::min(weight_from_left, weight_from_top), weight_from_diagonal); + curr_weights[x] = min_weight; + + int direction = (min_weight == weight_from_diagonal ? 0x01 : 0x0) // + | (min_weight == weight_from_left ? 0x02 : 0x0) // + | (min_weight == weight_from_top ? 0x04 : 0x0); // + + directions[x][y] = direction; + } + + // Swap the weights + std::swap(last_weights, curr_weights); + } + + // Rewrite directions(y,x) to match_a[i] and match_b[i]. + int x = int(a.size()) - 1; + int y = int(b.size()) - 1; + int i_a = x; + int i_b = y; + while (x >= 0 && y >= 0) { + int direction = directions[x][y]; + if (direction & 0x01) { + match_a[i_a--] = true; + match_b[i_b--] = true; + --x; + --y; + } + else if (direction & 0x02) { + --i_a; + --x; + } + else { // (direction & 0x04) + --i_b; + --y; + } + } + + return std::make_pair(match_a, match_b); +} + +} // namespace test_util + +#endif // TEST_UTIL_LCS_HPP From c72cafe15e6e20ce06756e4a496ad37eb669ad3e Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 21 Jul 2023 15:58:10 +1000 Subject: [PATCH 002/176] Refactor some more tests --- tests/dsl/FlagMessage.cpp | 69 +++++++++++++++++++++------------------ tests/dsl/Once.cpp | 66 ++++++++++++++++++++++++------------- 2 files changed, 82 insertions(+), 53 deletions(-) diff --git a/tests/dsl/FlagMessage.cpp b/tests/dsl/FlagMessage.cpp index db333ce87..2765575ed 100644 --- a/tests/dsl/FlagMessage.cpp +++ b/tests/dsl/FlagMessage.cpp @@ -19,50 +19,47 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { + +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + struct SimpleMessage {}; struct MessageA {}; struct MessageB {}; -MessageA* a = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -MessageB* b = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - - - on>().then([this] { - auto data = std::make_unique(); - a = data.get(); + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - // Emit the first half of the requred data - emit(data); - }); on>().then([this] { - // Check a has been emitted - REQUIRE(a != nullptr); - - auto data = std::make_unique(); - b = data.get(); + events.push_back("MessageA triggered"); + events.push_back("Emitting MessageB"); + emit(std::make_unique()); + }); - // Emit the 2nd half - emit(data); + on>().then([] { // + events.push_back("MessageB triggered"); + }); - // We can shutdown now, the other reactions will process before termination - powerplant.shutdown(); + // This should never run + on, With>().then([](const MessageA&, const MessageB&) { // + events.push_back("MessageA with MessageB triggered"); }); - on>().then([] { - // Check b has been emitted - REQUIRE(b != nullptr); + on>>().then([this] { + events.push_back("Step<1> triggered"); + events.push_back("Emitting MessageA"); + emit(std::make_unique()); }); - // We make this high priority to ensure it runs first (will check for more errors) - on, With, Priority::HIGH>().then([](const MessageA&, const MessageB&) { - FAIL("A was never emitted after B so this should not be possible"); + on().then([this] { + events.push_back("Emitting Step<1>"); + emit(std::make_unique>()); }); } }; @@ -75,10 +72,20 @@ TEST_CASE("Testing emitting types that are flag types (Have no contents)", "[api config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); + plant.start(); - auto message = std::make_unique(); + std::vector expected = { + "Emitting Step<1>", + "Step<1> triggered", + "Emitting MessageA", + "MessageA triggered", + "Emitting MessageB", + "MessageB triggered", + }; - plant.emit(message); + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); - plant.start(); + // Check the events fired in order and only those events + REQUIRE(events == expected); } diff --git a/tests/dsl/Once.cpp b/tests/dsl/Once.cpp index 08d6b0930..82f79505c 100644 --- a/tests/dsl/Once.cpp +++ b/tests/dsl/Once.cpp @@ -17,53 +17,75 @@ */ #include -#include #include +#include "test_util/TestBase.hpp" + namespace { -struct SimpleMessage {}; -struct StartMessage {}; +// Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int i = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int j = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +struct SimpleMessage { + SimpleMessage(int run) : run(run) {} + int run = 0; +}; class TestReactor : public NUClear::Reactor { public: TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { // Make this priority high so it will always run first if it is able - on, Priority::HIGH, Once>().then([] { - // Increment the counter, - ++i; - // Function has finished, then should unbind. + on, Priority::HIGH, Once>().then([](const SimpleMessage& msg) { // + events.push_back("Once Trigger executed " + std::to_string(msg.run)); }); - on>().then([this] { - ++j; - // Run until it's 11 then shutdown - if (j > 10) { - powerplant.shutdown(); + on>().then([this](const SimpleMessage& msg) { + events.push_back("Normal Trigger Executed " + std::to_string(msg.run)); + // Keep running until we have run 10 times + if (msg.run < 10) { + events.push_back("Emitting " + std::to_string(msg.run + 1)); + emit(std::make_unique(msg.run + 1)); } else { - powerplant.emit(std::make_unique()); - powerplant.emit(std::make_unique()); + powerplant.shutdown(); } }); + + on().then([this] { + events.push_back("Startup Trigger Executed"); + emit(std::make_unique(0)); + }); } }; } // namespace -TEST_CASE("Testing on functionality", "[api][once]") { +TEST_CASE("Reactions with the Once DSL keyword only execute once", "[api][once]") { NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); - - // We are installing with an initial log level of debug - plant.install(); - plant.emit(std::make_unique()); + plant.install(); plant.start(); - REQUIRE(i == 1); + std::vector expected = { + "Startup Trigger Executed", "Once Trigger executed 0", + "Normal Trigger Executed 0", "Emitting 1", + "Normal Trigger Executed 1", "Emitting 2", + "Normal Trigger Executed 2", "Emitting 3", + "Normal Trigger Executed 3", "Emitting 4", + "Normal Trigger Executed 4", "Emitting 5", + "Normal Trigger Executed 5", "Emitting 6", + "Normal Trigger Executed 6", "Emitting 7", + "Normal Trigger Executed 7", "Emitting 8", + "Normal Trigger Executed 8", "Emitting 9", + "Normal Trigger Executed 9", "Emitting 10", + "Normal Trigger Executed 10", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From b602864334a3a629f502d1a4721caa1d76871cbc Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 21 Jul 2023 15:59:23 +1000 Subject: [PATCH 003/176] Refactor trigger --- tests/dsl/Trigger.cpp | 46 ++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/tests/dsl/Trigger.cpp b/tests/dsl/Trigger.cpp index 191cee331..e22c0fe1f 100644 --- a/tests/dsl/Trigger.cpp +++ b/tests/dsl/Trigger.cpp @@ -19,37 +19,61 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { + +/// @brief A vector of events that have happened +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + struct SimpleMessage { + SimpleMessage(int data) : data(data) {} int data; }; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - on>().then([this](const SimpleMessage& message) { - // The message we received should have test == 10 - REQUIRE(message.data == 10); + on>().then([](const SimpleMessage& message) { // + events.push_back("Trigger " + std::to_string(message.data)); + }); - // We are finished the test - this->powerplant.shutdown(); + on().then([this] { + // Emit some messages with data + for (int i = 0; i < 10; ++i) { + emit(std::make_unique(i)); + } }); } }; } // namespace -TEST_CASE("A very basic test for Emit and On", "[api][trigger]") { +TEST_CASE("Test that Trigger statements get the correct data", "[api][trigger]") { NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); + plant.start(); - auto message = std::make_unique(SimpleMessage{10}); + std::vector expected = { + "Trigger 0", + "Trigger 1", + "Trigger 2", + "Trigger 3", + "Trigger 4", + "Trigger 5", + "Trigger 6", + "Trigger 7", + "Trigger 8", + "Trigger 9", + }; - plant.emit(message); + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); - plant.start(); + // Check the events fired in order and only those events + REQUIRE(events == expected); } From cdbb213a5aef446f9ec1d7ccbe7d14fa25762ed8 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 21 Jul 2023 16:06:43 +1000 Subject: [PATCH 004/176] Refactor last --- tests/dsl/Last.cpp | 70 +++++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/tests/dsl/Last.cpp b/tests/dsl/Last.cpp index 927bfaf9c..fae89bfd8 100644 --- a/tests/dsl/Last.cpp +++ b/tests/dsl/Last.cpp @@ -19,53 +19,40 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { -struct TestMessage { - int value; +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +struct TestMessage { TestMessage(int v) : value(v){}; + int value; }; -int emit_counter = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int recv_counter = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { on>>().then([this](std::list> messages) { - // We got another one - ++recv_counter; - - // Send out another before we test - emit(std::make_unique(++emit_counter)); - - // Finish when we get to 10 - if (messages.front()->value >= 10) { - powerplant.shutdown(); + std::stringstream ss; + for (auto& m : messages) { + ss << m->value << " "; } - else { - // Our list must be less than 5 long - REQUIRE(messages.size() <= 5); - - // If our size is less than 5 it should be the size of the front element - if (messages.size() < 5) { - REQUIRE(int(messages.size()) == messages.back()->value); - } + events.push_back(ss.str()); - // Check that our numbers are increasing - int i = messages.front()->value; - for (auto& m : messages) { - REQUIRE(m->value == i); - ++i; - } + // Finish when we get to 10 + if (messages.back()->value < 10) { + // Send out another message + emit(std::make_unique(messages.back()->value + 1)); } }); - on().then([this] { emit(std::make_unique(++emit_counter)); }); + on().then([this] { emit(std::make_unique(0)); }); } }; + } // namespace TEST_CASE("Testing the last n feature", "[api][last]") { @@ -74,6 +61,25 @@ TEST_CASE("Testing the last n feature", "[api][last]") { config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + std::vector expected = { + "0 ", + "0 1 ", + "0 1 2 ", + "0 1 2 3 ", + "0 1 2 3 4 ", + "1 2 3 4 5 ", + "2 3 4 5 6 ", + "3 4 5 6 7 ", + "4 5 6 7 8 ", + "5 6 7 8 9 ", + "6 7 8 9 10 ", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From 0d091fd28474c104f3b5912eda801eddd38f49b7 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 21 Jul 2023 16:23:16 +1000 Subject: [PATCH 005/176] More refactored tests --- tests/dsl/MainThread.cpp | 51 ++++++++++++++++++++++++---------- tests/dsl/MissingArguments.cpp | 45 ++++++++++++++++++++++-------- tests/dsl/Startup.cpp | 38 +++++++++++++++++-------- 3 files changed, 97 insertions(+), 37 deletions(-) diff --git a/tests/dsl/MainThread.cpp b/tests/dsl/MainThread.cpp index b7f1d1ec6..9cc002efc 100644 --- a/tests/dsl/MainThread.cpp +++ b/tests/dsl/MainThread.cpp @@ -19,43 +19,66 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { -class TestReactor : public NUClear::Reactor { +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +struct MessageA {}; +struct MessageB {}; + +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { // Run a task without MainThread to make sure it isn't on the main thread - on>().then("Non-MainThread reaction", [this] { - // We shouldn't be on the main thread - REQUIRE(NUClear::util::main_thread_id != std::this_thread::get_id()); + on>().then("Non-MainThread reaction", [this] { + events.push_back(std::string("MessageA triggered ") + + (NUClear::util::main_thread_id == std::this_thread::get_id() ? "on main thread" + : "on non-main thread")); - emit(std::make_unique(1.1)); + events.push_back("Emitting MessageB"); + emit(std::make_unique()); }); // Run a task with MainThread and ensure that it is on the main thread - on, MainThread>().then("MainThread reaction", [this] { - // Shutdown first so the test will end even if the next check fails - powerplant.shutdown(); + on, MainThread>().then("MainThread reaction", [this] { + events.push_back(std::string("MessageB triggered ") + + (NUClear::util::main_thread_id == std::this_thread::get_id() ? "on main thread" + : "on non-main thread")); - // We should be on the main thread - REQUIRE(NUClear::util::main_thread_id == std::this_thread::get_id()); + // Since we are a multithreaded test with MainThread we need to shutdown the test ourselves + powerplant.shutdown(); }); on().then([this]() { // Emit an integer to trigger the reaction - emit(std::make_unique()); + events.push_back("Emitting MessageA"); + emit(std::make_unique()); }); } }; } // namespace TEST_CASE("Testing that the MainThread keyword runs tasks on the main thread", "[api][dsl][main_thread]") { - NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + std::vector expected = { + "Emitting MessageA", + "MessageA triggered on non-main thread", + "Emitting MessageB", + "MessageB triggered on main thread", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } diff --git a/tests/dsl/MissingArguments.cpp b/tests/dsl/MissingArguments.cpp index e05513db5..cecb20a02 100644 --- a/tests/dsl/MissingArguments.cpp +++ b/tests/dsl/MissingArguments.cpp @@ -19,32 +19,39 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + template struct Message { int val; Message(int val) : val(val){}; }; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { on>, With>, With>, With>>().then( - [this](const Message<2>& m2, const Message<4>& m4) { - REQUIRE(m2.val == 2 + 4); - REQUIRE(m4.val == 4 + 4); - - powerplant.shutdown(); + [](const Message<2>& m2, const Message<4>& m4) { + events.push_back("Message<2>: " + std::to_string(m2.val)); + events.push_back("Message<4>: " + std::to_string(m4.val)); }); on().then([this] { // Emit from message 4 to 1 - emit(std::make_unique>(8)); - emit(std::make_unique>(7)); - emit(std::make_unique>(6)); - emit(std::make_unique>(5)); + events.push_back("Emitting Message<4>"); + emit(std::make_unique>(4 * 4)); + events.push_back("Emitting Message<3>"); + emit(std::make_unique>(3 * 3)); + events.push_back("Emitting Message<2>"); + emit(std::make_unique>(2 * 2)); + events.push_back("Emitting Message<1>"); + emit(std::make_unique>(1 * 1)); }); } }; @@ -56,6 +63,20 @@ TEST_CASE("Testing that when arguments missing from the call it can still run", config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + std::vector expected = { + "Emitting Message<4>", + "Emitting Message<3>", + "Emitting Message<2>", + "Emitting Message<1>", + "Message<2>: 4", + "Message<4>: 16", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } diff --git a/tests/dsl/Startup.cpp b/tests/dsl/Startup.cpp index cd5e158a7..94fcac898 100644 --- a/tests/dsl/Startup.cpp +++ b/tests/dsl/Startup.cpp @@ -19,35 +19,51 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { + +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + struct SimpleMessage { SimpleMessage(int data) : data(data) {} int data{0}; }; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - on>().then([this](const SimpleMessage& message) { - // The message we received should have test == 10 - REQUIRE(message.data == 10); - - // We are finished the test - powerplant.shutdown(); + on>().then([](const SimpleMessage& message) { // + events.push_back("SimpleMessage triggered with " + std::to_string(message.data)); }); - on().then([this]() { emit(std::make_unique(10)); }); + on().then([this]() { + events.push_back("Startup triggered"); + events.push_back("Emitting SimpleMessage"); + emit(std::make_unique(10)); + }); } }; } // namespace TEST_CASE("Testing the startup event is emitted at the start of the program", "[api][startup]") { - NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + std::vector expected = { + "Startup triggered", + "Emitting SimpleMessage", + "SimpleMessage triggered with 10", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From 05c8192215eabac0850bba0ef18b5edbef6855ae Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 21 Jul 2023 16:31:32 +1000 Subject: [PATCH 006/176] Missing header for strlen --- tests/test_util/diff_string.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_util/diff_string.cpp b/tests/test_util/diff_string.cpp index 8e90c850e..97fc9d235 100644 --- a/tests/test_util/diff_string.cpp +++ b/tests/test_util/diff_string.cpp @@ -18,6 +18,8 @@ #include "diff_string.hpp" +#include + #include "lcs.hpp" namespace test_util { From 584d88bd79630ecddf814503197bae9f4bb3da20 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 21 Jul 2023 16:33:56 +1000 Subject: [PATCH 007/176] clang-tidy --- tests/dsl/ArgumentFission.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/dsl/ArgumentFission.cpp b/tests/dsl/ArgumentFission.cpp index 675eac8b5..1d9a6cb03 100644 --- a/tests/dsl/ArgumentFission.cpp +++ b/tests/dsl/ArgumentFission.cpp @@ -28,7 +28,9 @@ std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-gl struct BindExtensionTest1 { template - static inline int bind(const std::shared_ptr& /*unused*/, int v1, bool v2) { + static inline int bind(const std::shared_ptr& /*unused*/, + const int& v1, + const bool& v2) { events.push_back("Bind1 with " + std::to_string(v1) + " and " + (v2 ? "true" : "false") + " called"); return 5; } @@ -37,8 +39,8 @@ struct BindExtensionTest1 { struct BindExtensionTest2 { template static inline bool bind(const std::shared_ptr& /*reaction*/, - std::string v1, - std::chrono::nanoseconds v2) { + const std::string& v1, + const std::chrono::nanoseconds& v2) { events.push_back("Bind2 with " + v1 + " and " + std::to_string(v2.count()) + " called"); return true; } @@ -47,9 +49,9 @@ struct BindExtensionTest2 { struct BindExtensionTest3 { template static inline std::string bind(const std::shared_ptr& /*reaction*/, - int v1, - int v2, - int v3) { + const int& v1, + const int& v2, + const int& v3) { events.push_back("Bind3 with " + std::to_string(v1) + ", " + std::to_string(v2) + " and " + std::to_string(v3) + " called"); return "return from Bind3"; @@ -60,7 +62,7 @@ class TestReactor : public test_util::TestBase { public: TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { int a = 0; - bool b = 0.0; + bool b = false; std::string c; // Bind all three functions to test fission From bdfc9b0f8969bdcd0ba9724a9e647d1c64350ce2 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 26 Jul 2023 10:53:50 +1000 Subject: [PATCH 008/176] IO test --- tests/dsl/IO.cpp | 78 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index 165b16c8f..2660b910a 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -19,6 +19,8 @@ #include #include +#include "test_util/TestBase.hpp" + // Windows can't do this test as it doesn't have file descriptors #ifndef _WIN32 @@ -28,49 +30,52 @@ namespace { +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + class TestReactor : public NUClear::Reactor { public: TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { std::array fds{-1, -1}; - if (pipe(fds.data()) < 0) { - FAIL("We couldn't make the pipe for the test"); + if (::pipe(fds.data()) < 0) { + events.push_back("Pipe creation failed"); + return; } in = fds[0]; out = fds[1]; + events.push_back("Pipe created"); - on(in, IO::READ).then([this](const IO::Event& e) { - // Read from our FD - unsigned char val{0}; - const ssize_t bytes = ::read(e.fd, &val, 1); + on(in.get(), IO::READ).then([this](const IO::Event& e) { + // Read from our fd + char c{0}; + auto bytes = ::read(e.fd, &c, 1); - // Check the data is correct - REQUIRE((e.events & IO::READ) != 0); - REQUIRE(bytes == 1); - REQUIRE(val == 0xDE); + events.push_back("Read " + std::to_string(bytes) + " bytes (" + c + ") from pipe"); - // Shutdown - powerplant.shutdown(); + if (c == 'o') { + powerplant.shutdown(); + } }); - writer = on(out, IO::WRITE).then([this](const IO::Event& e) { + writer = on(out.get(), IO::WRITE).then([this](const IO::Event& e) { // Send data into our fd - const unsigned char val = 0xDE; - const ssize_t bytes = ::write(e.fd, &val, 1); + const char c = "Hello"[char_no++]; + const ssize_t sent = ::write(e.fd, &c, 1); - // Check that our data was sent - REQUIRE((e.events & IO::WRITE) != 0); - REQUIRE(bytes == 1); + events.push_back("Wrote " + std::to_string(sent) + " bytes (" + c + ") to pipe"); - // Unbind ourselves - writer.unbind(); + if (char_no == 5) { + writer.unbind(); + } }); } - int in{-1}; - int out{-1}; + NUClear::util::FileDescriptor in{}; + NUClear::util::FileDescriptor out{}; + int char_no{0}; ReactionHandle writer{}; }; } // namespace @@ -81,8 +86,33 @@ TEST_CASE("Testing the IO extension", "[api][io]") { config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + std::vector expected = { + "Pipe created", + "Wrote 1 bytes (H) to pipe", + "Read 1 bytes (H) from pipe", + "Wrote 1 bytes (e) to pipe", + "Read 1 bytes (e) from pipe", + "Wrote 1 bytes (l) to pipe", + "Read 1 bytes (l) from pipe", + "Wrote 1 bytes (l) to pipe", + "Read 1 bytes (l) from pipe", + "Wrote 1 bytes (o) to pipe", + "Read 1 bytes (o) from pipe", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); +} + +#else // _WIN32 + +TEST_CASE("Testing the IO extension", "[api][io]") { + SKIP("IO extension is not supported on Windows") } -#endif +#endif // _WIN32 From 3c078551548acf6963971c82951105da86607e32 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 26 Jul 2023 14:41:00 +1000 Subject: [PATCH 009/176] Reaction handle --- tests/api/ReactionHandle.cpp | 54 ++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/tests/api/ReactionHandle.cpp b/tests/api/ReactionHandle.cpp index bfb08a611..a80912d8e 100644 --- a/tests/api/ReactionHandle.cpp +++ b/tests/api/ReactionHandle.cpp @@ -19,37 +19,67 @@ #include #include +#include "test_util/TestBase.hpp" + // Anonymous namespace to keep everything file local namespace { -template -struct Message {}; +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +struct Message { + Message(int i) : i(i) {} + int i; +}; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { // Make an always disabled reaction - ReactionHandle a = - on>, Priority::HIGH>().then([] { FAIL("This reaction is disabled always"); }); + a = on, Priority::HIGH>().then([](const Message& msg) { // + events.push_back("Executed disabled reaction " + std::to_string(msg.i)); + }); a.disable(); - const ReactionHandle b = on>>().then([this] { powerplant.shutdown(); }); + // Make a reaction that we toggle on and off + b = on, Priority::HIGH>().then([this](const Message& msg) { // + events.push_back("Executed toggled reaction " + std::to_string(msg.i)); + b.disable(); + emit(std::make_unique(1)); + }); + + on>().then([](const Message& msg) { // + events.push_back("Executed enabled reaction " + std::to_string(msg.i)); + }); // Start our test - on().then([this] { emit(std::make_unique>()); }); + on().then([this] { // + emit(std::make_unique(0)); + }); } + + ReactionHandle a{}; + ReactionHandle b{}; }; } // namespace TEST_CASE("Testing reaction handle functionality", "[api][reactionhandle]") { - NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); - - // We are installing with an initial log level of debug plant.install(); - plant.start(); + + std::vector expected = { + "Executed toggled reaction 0", + "Executed enabled reaction 0", + "Executed enabled reaction 1", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From cd1fa6f2dc98cb2bfeebab4bbf6b3830d4d2f67e Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 26 Jul 2023 15:07:03 +1000 Subject: [PATCH 010/176] Optional test --- tests/dsl/Optional.cpp | 86 +++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 52 deletions(-) diff --git a/tests/dsl/Optional.cpp b/tests/dsl/Optional.cpp index c5ce9fa85..5577d9d3b 100644 --- a/tests/dsl/Optional.cpp +++ b/tests/dsl/Optional.cpp @@ -19,71 +19,45 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { -int trigger1 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int trigger2 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int trigger3 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int trigger4 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) struct MessageA {}; struct MessageB {}; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - on, With>().then([](const MessageA&, const MessageB&) { - ++trigger1; - FAIL("This should never run as MessageB is never emitted"); + on, With>().then([](const MessageA&, const MessageB&) { // + events.push_back("Executed reaction with A and B"); }); - on, Optional>>().then( - [this](const MessageA&, const std::shared_ptr& b) { - ++trigger2; - - switch (trigger2) { - case 1: - // On our trigger, b should not exist - REQUIRE(!b); - break; - default: FAIL("Trigger 2 was triggered more than once"); - } - - // Emit B to start the second set - emit(std::make_unique()); - }); + on, Optional>>().then([this](const std::shared_ptr& b) { + events.push_back(std::string("Executed reaction with A and optional B with B") + (b ? "+" : "-")); + // Emit B to start the second set + events.push_back("Emitting B"); + emit(std::make_unique()); + }); - on, With>().then([] { - // This should run once - ++trigger3; + on, With>().then([] { // + events.push_back("Executed reaction with B and A"); }); // Double trigger test (to ensure that it can handle multiple DSL words on, Trigger>>().then( - [this](const std::shared_ptr& a, const std::shared_ptr& b) { - ++trigger4; - switch (trigger4) { - case 1: - // Check that A exists and B does not - REQUIRE(a); - REQUIRE(!b); - break; - case 2: - // Check that both exist - REQUIRE(a); - REQUIRE(b); - - // We should be done now - powerplant.shutdown(); - break; - - default: FAIL("Trigger 4 should only be triggered twice"); break; - } + [](const std::shared_ptr& a, const std::shared_ptr& b) { // + events.push_back(std::string("Executed reaction with optional A and B with A") + (a ? "+" : "-") + + " and B" + (b ? "+" : "-")); }); on().then([this] { // Emit only message A + events.push_back("Emitting A"); emit(std::make_unique()); }); } @@ -96,12 +70,20 @@ TEST_CASE("Testing that optional is able to let data through even if it's invali config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); - // Check that it was all as expected - REQUIRE(trigger1 == 0); - REQUIRE(trigger2 == 1); - REQUIRE(trigger3 == 1); - REQUIRE(trigger4 == 2); + std::vector expected = { + "Emitting A", + "Executed reaction with A and optional B with B-", + "Emitting B", + "Executed reaction with optional A and B with A+ and B-", + "Executed reaction with B and A", + "Executed reaction with optional A and B with A+ and B+", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From ce46a531a4c363b368ae12354aa4f2ee4e9fe996 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 27 Jul 2023 11:07:16 +1000 Subject: [PATCH 011/176] Replace single test with more robust buffer test --- tests/dsl/Buffer.cpp | 172 +++++++++++++++++++++++++++++++++++++++++++ tests/dsl/Single.cpp | 104 -------------------------- 2 files changed, 172 insertions(+), 104 deletions(-) create mode 100644 tests/dsl/Buffer.cpp delete mode 100644 tests/dsl/Single.cpp diff --git a/tests/dsl/Buffer.cpp b/tests/dsl/Buffer.cpp new file mode 100644 index 000000000..21d8392fa --- /dev/null +++ b/tests/dsl/Buffer.cpp @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2017 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include "test_util/TestBase.hpp" + +namespace { + +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +struct Message { + Message(int i) : i(i) {} + int i; +}; + +class TestReactor : public test_util::TestBase { +public: + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { + + on>().then([](const Message& msg) { // + events.push_back("Trigger reaction " + std::to_string(msg.i)); + }); + on, Single>().then([](const Message& msg) { // + events.push_back("Single reaction " + std::to_string(msg.i)); + }); + on, Buffer<2>>().then([](const Message& msg) { // + events.push_back("Buffer<2> reaction " + std::to_string(msg.i)); + }); + on, Buffer<3>>().then([](const Message& msg) { // + events.push_back("Buffer<3> reaction " + std::to_string(msg.i)); + }); + on, Buffer<4>>().then([](const Message& msg) { // + events.push_back("Buffer<4> reaction " + std::to_string(msg.i)); + }); + + on>, Priority::LOW>().then([this] { + events.push_back("Step 1"); + emit(std::make_unique(1)); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Step 2"); + emit(std::make_unique(2)); + emit(std::make_unique(3)); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Step 3"); + emit(std::make_unique(4)); + emit(std::make_unique(5)); + emit(std::make_unique(6)); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Step 4"); + emit(std::make_unique(7)); + emit(std::make_unique(8)); + emit(std::make_unique(9)); + emit(std::make_unique(10)); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Step 5"); + emit(std::make_unique(11)); + emit(std::make_unique(12)); + emit(std::make_unique(13)); + emit(std::make_unique(14)); + emit(std::make_unique(15)); + }); + + on().then("Startup", [this]() { + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + }); + } +}; +} // namespace + +TEST_CASE("Test that Buffer and Single limit the number of concurrent executions", "[api][precondition][single]") { + + NUClear::PowerPlant::Configuration config; + config.thread_count = 1; + NUClear::PowerPlant plant(config); + plant.install(); + plant.start(); + + std::vector expected = { + "Step 1", + "Trigger reaction 1", + "Single reaction 1", + "Buffer<2> reaction 1", + "Buffer<3> reaction 1", + "Buffer<4> reaction 1", + "Step 2", + "Trigger reaction 2", + "Single reaction 2", + "Buffer<2> reaction 2", + "Buffer<3> reaction 2", + "Buffer<4> reaction 2", + "Trigger reaction 3", + "Buffer<2> reaction 3", + "Buffer<3> reaction 3", + "Buffer<4> reaction 3", + "Step 3", + "Trigger reaction 4", + "Single reaction 4", + "Buffer<2> reaction 4", + "Buffer<3> reaction 4", + "Buffer<4> reaction 4", + "Trigger reaction 5", + "Buffer<2> reaction 5", + "Buffer<3> reaction 5", + "Buffer<4> reaction 5", + "Trigger reaction 6", + "Buffer<3> reaction 6", + "Buffer<4> reaction 6", + "Step 4", + "Trigger reaction 7", + "Single reaction 7", + "Buffer<2> reaction 7", + "Buffer<3> reaction 7", + "Buffer<4> reaction 7", + "Trigger reaction 8", + "Buffer<2> reaction 8", + "Buffer<3> reaction 8", + "Buffer<4> reaction 8", + "Trigger reaction 9", + "Buffer<3> reaction 9", + "Buffer<4> reaction 9", + "Trigger reaction 10", + "Buffer<4> reaction 10", + "Step 5", + "Trigger reaction 11", + "Single reaction 11", + "Buffer<2> reaction 11", + "Buffer<3> reaction 11", + "Buffer<4> reaction 11", + "Trigger reaction 12", + "Buffer<2> reaction 12", + "Buffer<3> reaction 12", + "Buffer<4> reaction 12", + "Trigger reaction 13", + "Buffer<3> reaction 13", + "Buffer<4> reaction 13", + "Trigger reaction 14", + "Buffer<4> reaction 14", + "Trigger reaction 15", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); +} diff --git a/tests/dsl/Single.cpp b/tests/dsl/Single.cpp deleted file mode 100644 index 17522e1e0..000000000 --- a/tests/dsl/Single.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -namespace { - -struct MessageCount { - MessageCount() = default; - - std::atomic message1{0}; - std::atomic message2{0}; - std::atomic message3{0}; -}; - -MessageCount message_count; // NOLINT(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) - -struct SimpleMessage1 { - int data{0}; -}; - -struct SimpleMessage2 { - int data{0}; -}; - -struct SimpleMessage3 { - int data{0}; -}; - -class TestReactor : public NUClear::Reactor { -public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - - on, Single>().then("SimpleMessage1", [this](const SimpleMessage1&) { - // Increment our run count - ++message_count.message1; - - // Emit a message 2 - emit(std::make_unique()); - - // Wait for 10 ms - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - - // Emit a message 3 - emit(std::make_unique()); - - // Emit another message 2 - emit(std::make_unique()); - - // We are finished the test - powerplant.shutdown(); - }); - - on, Single>().then("SimpleMessage2", - [](const SimpleMessage2&) { ++message_count.message2; }); - - on, With, Single>().then( - "SimpleMessage2 With SimpleMessage3", - [](const SimpleMessage2&, const SimpleMessage3&) { ++message_count.message3; }); - - on().then("Startup", [this]() { - // Emit two events, only one should run - emit(std::make_unique()); - emit(std::make_unique()); - }); - } -}; -} // namespace - -TEST_CASE("Test that single prevents a second call while one is executing", "[api][precondition][single]") { - - NUClear::PowerPlant::Configuration config; - // Unless there are at least 2 threads here single makes no sense ;) - config.thread_count = 2; - NUClear::PowerPlant plant(config); - plant.install(); - - plant.start(); - - // Require that only 1 run has happened on message 1 - REQUIRE(message_count.message1 == 1); - - // Require that 2 runs have happened on message 2 - REQUIRE(message_count.message2 == 2); - - // Require that only 1 run has happened on message 3 - REQUIRE(message_count.message3 == 1); -} From 441103b542c3aef86be2e1a2f5446fd2efa5905a Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 27 Jul 2023 11:09:24 +1000 Subject: [PATCH 012/176] clang-tidy --- tests/dsl/CommandLineArguments.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dsl/CommandLineArguments.cpp b/tests/dsl/CommandLineArguments.cpp index 7ef287ccd..e3ab6b1a3 100644 --- a/tests/dsl/CommandLineArguments.cpp +++ b/tests/dsl/CommandLineArguments.cpp @@ -36,7 +36,7 @@ class TestReactor : public test_util::TestBase { on>().then([](const CommandLineArguments& args) { std::stringstream output; - for (auto& arg : args) { + for (const auto& arg : args) { output << arg << " "; } events.push_back("CommandLineArguments: " + output.str()); From 7f310fdde421ed38ebcbdeb0fafbb5c3092400bc Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 27 Jul 2023 11:13:06 +1000 Subject: [PATCH 013/176] Fix IO test as read/write could be out of order --- tests/dsl/IO.cpp | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index 2660b910a..3fb82d9ab 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -30,8 +30,10 @@ namespace { -/// @brief Events that occur during the test -std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +/// @brief Events that occur during the test reading +std::vector read_events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +/// @brief Events that occur during the test writing +std::vector write_events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) class TestReactor : public NUClear::Reactor { public: @@ -40,20 +42,17 @@ class TestReactor : public NUClear::Reactor { std::array fds{-1, -1}; if (::pipe(fds.data()) < 0) { - events.push_back("Pipe creation failed"); return; } - in = fds[0]; out = fds[1]; - events.push_back("Pipe created"); on(in.get(), IO::READ).then([this](const IO::Event& e) { // Read from our fd char c{0}; auto bytes = ::read(e.fd, &c, 1); - events.push_back("Read " + std::to_string(bytes) + " bytes (" + c + ") from pipe"); + read_events.push_back("Read " + std::to_string(bytes) + " bytes (" + c + ") from pipe"); if (c == 'o') { powerplant.shutdown(); @@ -65,7 +64,7 @@ class TestReactor : public NUClear::Reactor { const char c = "Hello"[char_no++]; const ssize_t sent = ::write(e.fd, &c, 1); - events.push_back("Wrote " + std::to_string(sent) + " bytes (" + c + ") to pipe"); + write_events.push_back("Wrote " + std::to_string(sent) + " bytes (" + c + ") to pipe"); if (char_no == 5) { writer.unbind(); @@ -88,31 +87,31 @@ TEST_CASE("Testing the IO extension", "[api][io]") { plant.install(); plant.start(); - std::vector expected = { - "Pipe created", - "Wrote 1 bytes (H) to pipe", + std::vector read_expected = { "Read 1 bytes (H) from pipe", - "Wrote 1 bytes (e) to pipe", "Read 1 bytes (e) from pipe", - "Wrote 1 bytes (l) to pipe", "Read 1 bytes (l) from pipe", - "Wrote 1 bytes (l) to pipe", "Read 1 bytes (l) from pipe", - "Wrote 1 bytes (o) to pipe", "Read 1 bytes (o) from pipe", }; // Make an info print the diff in an easy to read way if we fail - INFO(test_util::diff_string(expected, events)); + INFO("Read Events\n" << test_util::diff_string(read_expected, read_events)); - // Check the events fired in order and only those events - REQUIRE(events == expected); -} + std::vector write_expected{ + "Wrote 1 bytes (H) to pipe", + "Wrote 1 bytes (e) to pipe", + "Wrote 1 bytes (l) to pipe", + "Wrote 1 bytes (l) to pipe", + "Wrote 1 bytes (o) to pipe", + }; -#else // _WIN32 + // Make an info print the diff in an easy to read way if we fail + INFO("Write Events\n" << test_util::diff_string(write_expected, write_events)); -TEST_CASE("Testing the IO extension", "[api][io]") { - SKIP("IO extension is not supported on Windows") + // Check the events fired in order and only those events + REQUIRE(read_events == read_expected); + REQUIRE(write_events == write_expected); } #endif // _WIN32 From 2967a187036bb8a2a427f32597d3c32f4f91acd3 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 27 Jul 2023 11:30:37 +1000 Subject: [PATCH 014/176] Sync order test is pretty good already --- tests/dsl/{SingleSync.cpp => SyncOrder.cpp} | 31 +++++++++++---------- 1 file changed, 16 insertions(+), 15 deletions(-) rename tests/dsl/{SingleSync.cpp => SyncOrder.cpp} (69%) diff --git a/tests/dsl/SingleSync.cpp b/tests/dsl/SyncOrder.cpp similarity index 69% rename from tests/dsl/SingleSync.cpp rename to tests/dsl/SyncOrder.cpp index 696dd37c2..15a457101 100644 --- a/tests/dsl/SingleSync.cpp +++ b/tests/dsl/SyncOrder.cpp @@ -21,8 +21,12 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { +constexpr int N_EVENTS = 1000; + struct Message { int val; Message(int val) : val(val){}; @@ -30,39 +34,36 @@ struct Message { struct ShutdownOnIdle {}; -std::vector values; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - on, Sync>().then("SyncReaction", [](const Message& m) { - values.push_back("Received value " + std::to_string(m.val)); + on, Sync>().then([](const Message& m) { // + events.push_back("Received value " + std::to_string(m.val)); }); - on, Priority::IDLE>().then("ShutdownOnIdle", [this] { powerplant.shutdown(); }); - on().then("Startup", [this] { - values.clear(); - for (int i = 0; i < 1000; ++i) { + for (int i = 0; i < N_EVENTS; ++i) { emit(std::make_unique(i)); } - emit(std::make_unique()); }); } }; } // namespace -TEST_CASE("Testing that the Sync priority queue word works correctly", "[api][sync][priority]") { +TEST_CASE("Sync events execute in order", "[api][sync][priority]") { NUClear::PowerPlant::Configuration config; - config.thread_count = 2; + config.thread_count = 4; NUClear::PowerPlant plant(config); plant.install(); plant.start(); - REQUIRE(values.size() == 1000); - for (int i = 0; i < 1000; ++i) { - CHECK(values[i] == "Received value " + std::to_string(i)); + REQUIRE(events.size() == N_EVENTS); + for (int i = 0; i < N_EVENTS; ++i) { + CHECK(events[i] == "Received value " + std::to_string(i)); } } From a24577fc5f8ac6b31d4664f5b2232f26d9a71080 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 10:29:59 +1000 Subject: [PATCH 015/176] Priority test --- tests/dsl/Priority.cpp | 127 +++++++++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 48 deletions(-) diff --git a/tests/dsl/Priority.cpp b/tests/dsl/Priority.cpp index 7a16e1f49..c68a71afb 100644 --- a/tests/dsl/Priority.cpp +++ b/tests/dsl/Priority.cpp @@ -19,48 +19,64 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { -struct Message1 {}; -struct Message2 {}; -struct Message3 {}; +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -bool low = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -bool med = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -bool high = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +template +struct Message {}; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - - on, Priority::HIGH>().then("High", [] { - // We should be the first to run - REQUIRE(!low); - REQUIRE(!med); - REQUIRE(!high); - - high = true; - }); - - on, Priority::NORMAL>().then("Normal", [] { - // We should be the second to run - REQUIRE(!low); - REQUIRE(!med); - REQUIRE(high); - - med = true; - }); - - on, Priority::LOW>().then("Low", [this] { - // We should be the final one to run - REQUIRE(!low); - REQUIRE(med); - REQUIRE(high); - - low = true; - - // We're done - powerplant.shutdown(); + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { + + // Declare in the order you'd expect them to fire + on>, Priority::REALTIME>().then([] { events.push_back("Realtime Message<1>"); }); + on>, Priority::HIGH>().then("High", [] { events.push_back("High Message<1>"); }); + on>>().then([] { events.push_back("Default Message<1>"); }); + on>, Priority::NORMAL>().then("Normal", [] { events.push_back("Normal Message<1>"); }); + on>, Priority::LOW>().then("Low", [] { events.push_back("Low Message<1>"); }); + on>, Priority::IDLE>().then([] { events.push_back("Idle Message<1>"); }); + + // Declare in the opposite order to what you'd expect them to fire + on>, Priority::IDLE>().then([] { events.push_back("Idle Message<2>"); }); + on>, Priority::LOW>().then([] { events.push_back("Low Message<2>"); }); + on>, Priority::NORMAL>().then([] { events.push_back("Normal Message<2>"); }); + on>>().then([] { events.push_back("Default Message<2>"); }); + on>, Priority::HIGH>().then([] { events.push_back("High Message<2>"); }); + on>, Priority::REALTIME>().then([] { events.push_back("Realtime Message<2>"); }); + + // Declare in a random order + std::array order = {0, 1, 2, 3, 4}; + std::shuffle(order.begin(), order.end(), std::mt19937(std::random_device()())); + for (const auto& i : order) { + switch (i) { + case 0: + on>, Priority::REALTIME>().then([] { events.push_back("Realtime Message<3>"); }); + break; + case 1: + on>, Priority::HIGH>().then([] { events.push_back("High Message<3>"); }); + break; + case 2: + on>, Priority::NORMAL>().then([] { events.push_back("Normal Message<3>"); }); + on>>().then([] { events.push_back("Default Message<3>"); }); + break; + case 3: + on>, Priority::LOW>().then([] { events.push_back("Low Message<3>"); }); + break; + case 4: + on>, Priority::IDLE>().then([] { events.push_back("Idle Message<3>"); }); + break; + } + } + + on().then([this] { + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); }); } }; @@ -73,17 +89,32 @@ TEST_CASE("Tests that priority orders the tasks appropriately", "[api][priority] config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - - // Emit message 2, then 1 then 3 (totally wrong order) - // Should require the priority queue to sort it out - plant.emit(std::make_unique()); - plant.emit(std::make_unique()); - plant.emit(std::make_unique()); - plant.start(); - // Make sure everything ran - REQUIRE(low); - REQUIRE(med); - REQUIRE(high); + std::vector expected = { + "Realtime Message<1>", + "Realtime Message<2>", + "Realtime Message<3>", + "High Message<1>", + "High Message<2>", + "High Message<3>", + "Default Message<1>", + "Normal Message<1>", + "Normal Message<2>", + "Default Message<2>", + "Normal Message<3>", + "Default Message<3>", + "Low Message<1>", + "Low Message<2>", + "Low Message<3>", + "Idle Message<1>", + "Idle Message<2>", + "Idle Message<3>", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From 3acb34396804e61a8f7ea73bce757eb582f93493 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 10:34:53 +1000 Subject: [PATCH 016/176] Shutdown test --- tests/dsl/Shutdown.cpp | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/tests/dsl/Shutdown.cpp b/tests/dsl/Shutdown.cpp index 1a028bd36..9481e14c7 100644 --- a/tests/dsl/Shutdown.cpp +++ b/tests/dsl/Shutdown.cpp @@ -19,23 +19,31 @@ #include #include +#include "test_util/TestBase.hpp" + // Anonymous namespace to keep everything file local namespace { -volatile bool did_shutdown = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -struct SimpleMessage {}; +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { + + on().then([] { // + events.push_back("Shutdown task executed"); + }); - on>().then([this] { - // Shutdown so we can test shutting down + on>>().then([this] { + events.push_back("Requesting shutdown"); powerplant.shutdown(); }); - on().then([] { did_shutdown = true; }); + on().then([this] { + events.push_back("Starting test"); + emit(std::make_unique>()); + }); } }; } // namespace @@ -46,10 +54,17 @@ TEST_CASE("A test that a shutdown message is emitted when the system shuts down" config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); + plant.start(); - plant.emit(std::make_unique()); + std::vector expected = { + "Starting test", + "Requesting shutdown", + "Shutdown task executed", + }; - plant.start(); + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); - REQUIRE(did_shutdown); + // Check the events fired in order and only those events + REQUIRE(events == expected); } From fdcbd0b1794c41fea7768ee9e040c9350f68785c Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 10:45:02 +1000 Subject: [PATCH 017/176] Raw function test --- tests/dsl/RawFunction.cpp | 97 +++++++++++++++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 9 deletions(-) diff --git a/tests/dsl/RawFunction.cpp b/tests/dsl/RawFunction.cpp index 7ae93d204..ff6397aee 100644 --- a/tests/dsl/RawFunction.cpp +++ b/tests/dsl/RawFunction.cpp @@ -19,22 +19,83 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { -bool ran = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +struct Message { + Message(std::string data) : data(std::move(data)) {} + std::string data; +}; + +struct Data { + Data(std::string data) : data(std::move(data)) {} + std::string data; +}; -double do_amazing_thing() { - ran = true; +/** + * @brief Test a raw function that takes no arguments and has a return type. + * The return type should be ignored and this function should run without issue. + * + * @return double + */ +double raw_function_test_no_args() { + events.push_back("Raw function no args"); return 5.0; } -class TestReactor : public NUClear::Reactor { +/** + * @brief Raw function that takes one argument (the left side of the trigger) + * + * @param msg the message + */ +void raw_function_test_left_arg(const Message& msg) { + events.push_back("Raw function left arg: " + msg.data); +} + +/** + * @brief Raw function that takes one argument (the right side of the trigger) + * + * @param data the data + */ +void raw_function_test_right_arg(const Data& data) { + events.push_back("Raw function right arg: " + data.data); +} + +/** + * @brief Raw function that takes both arguments + * + * @param msg the message + * @param data the data + */ +void raw_function_test_both_args(const Message& msg, const Data& data) { + events.push_back("Raw function both args: " + msg.data + " " + data.data); +} + + +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { + + on, Trigger>().then(raw_function_test_no_args); + on, Trigger>().then(raw_function_test_left_arg); + on, Trigger>().then(raw_function_test_right_arg); + on, Trigger>().then(raw_function_test_both_args); - on().then(do_amazing_thing); + on>>().then([this] { emit(std::make_unique("D1")); }); + on>>().then([this] { emit(std::make_unique("M2")); }); + on>>().then([this] { emit(std::make_unique("D3")); }); + on>>().then([this] { emit(std::make_unique("M4")); }); - on().then([this] { powerplant.shutdown(); }); + on().then([this] { + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + }); } }; } // namespace @@ -45,8 +106,26 @@ TEST_CASE("Test reaction can take a raw function instead of just a lambda", "[ap config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); - REQUIRE(ran); + std::vector expected = { + "Raw function no args", + "Raw function left arg: M2", + "Raw function right arg: D1", + "Raw function both args: M2 D1", + "Raw function no args", + "Raw function left arg: M2", + "Raw function right arg: D3", + "Raw function both args: M2 D3", + "Raw function no args", + "Raw function left arg: M4", + "Raw function right arg: D3", + "Raw function both args: M4 D3", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From 11463a68131fa28b588d4fa53842c86e2982dbdb Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 11:26:25 +1000 Subject: [PATCH 018/176] Combine and improve the every test --- tests/dsl/Every.cpp | 99 ++++++++++++++++++++++------------------- tests/dsl/Every_Per.cpp | 89 ------------------------------------ 2 files changed, 52 insertions(+), 136 deletions(-) delete mode 100644 tests/dsl/Every_Per.cpp diff --git a/tests/dsl/Every.cpp b/tests/dsl/Every.cpp index 50cfb2bbe..a04203a75 100644 --- a/tests/dsl/Every.cpp +++ b/tests/dsl/Every.cpp @@ -20,70 +20,75 @@ #include #include -namespace { - -class TestReactor : public NUClear::Reactor { -public: - // Store our times - std::vector times{}; +#include "test_util/TestBase.hpp" - static constexpr size_t NUM_LOG_ITEMS = 1000; +namespace { - static constexpr size_t WAIT_LENGTH_MILLIS = 1; +// Store our times +std::vector times{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +std::vector per_times{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +std::vector dynamic_times{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { +class TestReactor : public test_util::TestBase { - // Trigger every 10 milliseconds - on>().then([this] { - // Start logging our times each time an emit happens - times.push_back(NUClear::clock::now()); +public: + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { - // Once we have enough items then we can do our statistics - if (times.size() == NUM_LOG_ITEMS) { + // Trigger on 3 different types of every + on>>().then([]() { times.push_back(NUClear::clock::now()); }); + on>().then([]() { per_times.push_back(NUClear::clock::now()); }); + on>(std::chrono::milliseconds(1)).then([]() { dynamic_times.push_back(NUClear::clock::now()); }); - // Build up our difference vector - std::vector diff; + // Gather data for some amount of time + on>().then([this] { powerplant.shutdown(); }); + } +}; +} // namespace - for (size_t i = 0; i < times.size() - 1; ++i) { - const std::chrono::nanoseconds delta = times[i + 1] - times[i]; +void test_results(const std::vector& times) { - // Store our difference in seconds - diff.push_back(double(delta.count()) / double(std::nano::den)); - } + // Build up our difference vector + std::vector diff; + for (size_t i = 0; i < times.size() - 1; ++i) { + double delta = std::chrono::duration_cast>(times[i + 1] - times[i]).count(); - // Normalize our differences to jitter - for (double& d : diff) { - d -= double(WAIT_LENGTH_MILLIS) / 1000.0; - } + // Calculate the difference between the expected and actual time + diff.push_back(delta - 1e-3); + } - // Calculate our mean, range, and stddev for the set - const double sum = std::accumulate(std::begin(diff), std::end(diff), 0.0); - const double mean = sum / double(diff.size()); - const double variance = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0); - const double stddev = std::sqrt(variance / double(diff.size())); + // Calculate our mean, range, and stddev for the set + const double sum = std::accumulate(std::begin(diff), std::end(diff), 0.0); + const double mean = sum / double(diff.size()); + const double variance = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0); + const double stddev = std::sqrt(variance / double(diff.size())); - // As time goes on the average wait should be 0 (we accept less then 0.5ms for this test) - REQUIRE(fabs(mean) < 0.0005); + // As time goes on the average wait should be close to 0 + INFO("Average error in timing: " << mean << "±" << stddev); - // Require that 95% (ish) of all results are fast enough - REQUIRE(fabs(mean + stddev * 2) < 0.008); - } - // Once we have more then enough items then we shutdown the powerplant - else if (times.size() > NUM_LOG_ITEMS) { - // We are finished the test - this->powerplant.shutdown(); - } - }); - } -}; -} // namespace + REQUIRE(std::abs(mean) < 0.0005); + REQUIRE(std::abs(mean + stddev * 2) < 0.008); +} -TEST_CASE("Testing the Every<> Smart Type", "[api][every][period]") { +TEST_CASE("Testing the Every<> DSL word", "[api][every][per]") { NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + { + INFO("Testing Every"); + test_results(times); + } + + { + INFO("Testing Every Per"); + test_results(per_times); + } + + { + INFO("Testing Dynamic Every every"); + test_results(dynamic_times); + } } diff --git a/tests/dsl/Every_Per.cpp b/tests/dsl/Every_Per.cpp deleted file mode 100644 index 90e2c35f9..000000000 --- a/tests/dsl/Every_Per.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include - -namespace { - -class TestReactorPer : public NUClear::Reactor { -public: - // Store our times - std::vector times{}; - - static constexpr size_t NUM_LOG_ITEMS = 1000; - - static constexpr size_t CYCLES_PER_SECOND = 1000; - - TestReactorPer(std::unique_ptr environment) : Reactor(std::move(environment)) { - - // Trigger every 10 milliseconds - on>>().then([this]() { - // Start logging our times each time an emit happens - times.push_back(NUClear::clock::now()); - - // Once we have enough items then we can do our statistics - if (times.size() == NUM_LOG_ITEMS) { - - // Build up our difference vector - std::vector diff; - - for (size_t i = 0; i < times.size() - 1; ++i) { - const std::chrono::nanoseconds delta = times[i + 1] - times[i]; - - // Store our difference in seconds - diff.push_back(double(delta.count()) / double(std::nano::den)); - } - - // Normalize our differences to jitter - for (double& d : diff) { - d -= 1.0 / double(CYCLES_PER_SECOND); - } - - // Calculate our mean, range, and stddev for the set - const double sum = std::accumulate(std::begin(diff), std::end(diff), 0.0); - const double mean = sum / double(diff.size()); - const double variance = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0); - const double stddev = std::sqrt(variance / double(diff.size())); - - // As time goes on the average wait should be 0 (we accept less then 0.5ms for this test) - REQUIRE(fabs(mean) < 0.0005); - - // Require that 95% (ish) of all results are fast enough - REQUIRE(fabs(mean + stddev * 2) < 0.008); - } - // Once we have more then enough items then we shutdown the powerplant - else if (times.size() > NUM_LOG_ITEMS) { - // We are finished the test - this->powerplant.shutdown(); - } - }); - } -}; -} // namespace - -TEST_CASE("Testing the Every<> Smart Type using Per", "[api][every][per]") { - - NUClear::PowerPlant::Configuration config; - config.thread_count = 1; - NUClear::PowerPlant plant(config); - plant.install(); - - plant.start(); -} From d9580daea3a88d05ae117f183b278de0bcd3f658 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 11:30:05 +1000 Subject: [PATCH 019/176] Swap argument fission test to do an implicit conversion --- tests/dsl/ArgumentFission.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/dsl/ArgumentFission.cpp b/tests/dsl/ArgumentFission.cpp index 1d9a6cb03..5c8279845 100644 --- a/tests/dsl/ArgumentFission.cpp +++ b/tests/dsl/ArgumentFission.cpp @@ -51,9 +51,9 @@ struct BindExtensionTest3 { static inline std::string bind(const std::shared_ptr& /*reaction*/, const int& v1, const int& v2, - const int& v3) { - events.push_back("Bind3 with " + std::to_string(v1) + ", " + std::to_string(v2) + " and " + std::to_string(v3) - + " called"); + const std::chrono::nanoseconds& v3) { + events.push_back("Bind3 with " + std::to_string(v1) + ", " + std::to_string(v2) + " and " + + std::to_string(v3.count()) + " called"); return "return from Bind3"; } }; @@ -73,7 +73,7 @@ class TestReactor : public test_util::TestBase { std::chrono::seconds(2), 9, 10, - 11) + std::chrono::seconds(11)) .then([] {}); events.push_back("Bind1 returned " + std::to_string(a)); @@ -94,7 +94,7 @@ TEST_CASE("Testing distributing arguments to multiple bind functions (NUClear Fi std::vector expected = { "Bind1 with 5 and false called", "Bind2 with Hello and 2000000000 called", - "Bind3 with 9, 10 and 11 called", + "Bind3 with 9, 10 and 11000000000 called", "Bind1 returned 5", "Bind2 returned true", "Bind3 returned return from Bind3", From a59b120e44891639f08b0acba2600b5a4e869966 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 11:44:47 +1000 Subject: [PATCH 020/176] Fix the bug with dynamic every --- src/dsl/word/Every.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dsl/word/Every.hpp b/src/dsl/word/Every.hpp index f670029c3..0c4555c74 100644 --- a/src/dsl/word/Every.hpp +++ b/src/dsl/word/Every.hpp @@ -76,11 +76,11 @@ namespace dsl { * seconds, minutes, hours). Note that you can also define your own unit: See * http://en.cppreference.com/w/cpp/chrono/duration */ - template + template struct Every; - template <> - struct Every<0, NUClear::clock::duration> { + template + struct Every<0, period> { template static inline void bind(const std::shared_ptr& reaction, From 65cafe4e673f61652003b539621f3410135cbaa4 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 12:16:37 +1000 Subject: [PATCH 021/176] Fix shadow --- tests/dsl/Every.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/dsl/Every.cpp b/tests/dsl/Every.cpp index a04203a75..9c4049efe 100644 --- a/tests/dsl/Every.cpp +++ b/tests/dsl/Every.cpp @@ -25,7 +25,7 @@ namespace { // Store our times -std::vector times{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +std::vector every_times{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) std::vector per_times{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) std::vector dynamic_times{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) @@ -35,7 +35,7 @@ class TestReactor : public test_util::TestBase { TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { // Trigger on 3 different types of every - on>>().then([]() { times.push_back(NUClear::clock::now()); }); + on>>().then([]() { every_times.push_back(NUClear::clock::now()); }); on>().then([]() { per_times.push_back(NUClear::clock::now()); }); on>(std::chrono::milliseconds(1)).then([]() { dynamic_times.push_back(NUClear::clock::now()); }); @@ -79,7 +79,7 @@ TEST_CASE("Testing the Every<> DSL word", "[api][every][per]") { { INFO("Testing Every"); - test_results(times); + test_results(every_times); } { From 2ebb0634e6ca28ee58a875a5f90defd8dc38e783 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 12:26:15 +1000 Subject: [PATCH 022/176] Sync test --- tests/dsl/Sync.cpp | 89 ++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/tests/dsl/Sync.cpp b/tests/dsl/Sync.cpp index d9b473c6b..5c1493b54 100644 --- a/tests/dsl/Sync.cpp +++ b/tests/dsl/Sync.cpp @@ -19,103 +19,106 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + template struct Message { - int val; - Message(int val) : val(val){}; + Message(std::string data) : data(std::move(data)){}; + std::string data; }; - -std::atomic semaphore(0); // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int finished = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { on>, Sync>().then([this](const Message<0>& m) { - // Increment our semaphore - ++semaphore; + events.push_back("Sync A " + m.data); // Sleep for some time to be safe std::this_thread::sleep_for(std::chrono::milliseconds(5)); - // Check we got the right message - REQUIRE(m.val == 123); - - // Require our semaphore is 1 - REQUIRE(semaphore == 1); - // Emit a message 1 here, it should not run yet - emit(std::make_unique>(10)); + events.push_back("Sync A emitting"); + emit(std::make_unique>("From Sync A")); // Sleep for some time again std::this_thread::sleep_for(std::chrono::milliseconds(5)); - // Decrement our semaphore - --semaphore; + events.push_back("Sync A " + m.data + " finished"); }); on>, Sync>().then([this](const Message<0>& m) { - // Increment our semaphore - ++semaphore; + events.push_back("Sync B " + m.data); // Sleep for some time to be safe std::this_thread::sleep_for(std::chrono::milliseconds(5)); - // Check we got the right message - REQUIRE(m.val == 123); - - // Require our semaphore is 1 - REQUIRE(semaphore == 1); - // Emit a message 1 here, it should not run yet - emit(std::make_unique>(10)); + events.push_back("Sync B emitting"); + emit(std::make_unique>("From Sync B")); // Sleep for some time again std::this_thread::sleep_for(std::chrono::milliseconds(5)); - // Decrement our semaphore - --semaphore; + events.push_back("Sync B " + m.data + " finished"); }); on>, Sync>().then([this](const Message<1>& m) { - // Increment our semaphore - ++semaphore; + events.push_back("Sync C " + m.data); // Sleep for some time to be safe std::this_thread::sleep_for(std::chrono::milliseconds(5)); - // Check we got the right message - REQUIRE(m.val == 10); - - // Require our semaphore is 1 - REQUIRE(semaphore == 1); + // Emit a message 1 here, it should not run yet + events.push_back("Sync C waiting"); // Sleep for some time again std::this_thread::sleep_for(std::chrono::milliseconds(5)); - // Decrement our semaphore - --semaphore; + events.push_back("Sync C " + m.data + " finished"); - if (++finished == 2) { + if (m.data == "From Sync B") { powerplant.shutdown(); } }); - on().then([this] { emit(std::make_unique>(123)); }); + on().then([this] { // + emit(std::make_unique>("From Startup")); + }); } }; } // namespace TEST_CASE("Testing that the Sync word works correctly", "[api][sync]") { - NUClear::PowerPlant::Configuration config; config.thread_count = 4; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + std::vector expected = { + "Sync A From Startup", + "Sync A emitting", + "Sync A From Startup finished", + "Sync B From Startup", + "Sync B emitting", + "Sync B From Startup finished", + "Sync C From Sync A", + "Sync C waiting", + "Sync C From Sync A finished", + "Sync C From Sync B", + "Sync C waiting", + "Sync C From Sync B finished", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From deff1efdd2b2ff1d2648d39ad1cb8c584943ee47 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 13:03:20 +1000 Subject: [PATCH 023/176] Watchdog test --- tests/dsl/Watchdog.cpp | 167 ++++++++++++++++++----------------------- 1 file changed, 74 insertions(+), 93 deletions(-) diff --git a/tests/dsl/Watchdog.cpp b/tests/dsl/Watchdog.cpp index e82e1c0b4..a4c368baf 100644 --- a/tests/dsl/Watchdog.cpp +++ b/tests/dsl/Watchdog.cpp @@ -21,97 +21,74 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { -NUClear::clock::time_point start; // NOLINT(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) -NUClear::clock::time_point end; // NOLINT(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) -NUClear::clock::time_point end_a; // NOLINT(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) -NUClear::clock::time_point end_b; // NOLINT(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) -bool a_ended = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -bool b_ended = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -int count = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -#ifdef _WIN32 -// The precision of timing on Windows (with the current NUClear timing method) is not great. -// This defines the intervals larger to avoid the problems at smaller intervals -// TODO(Josephus or Trent): use a higher precision timing method on Windows (look into nanosleep, -// in addition to the condition lock and spin lock used in ChronoController.hpp) -constexpr int WATCHDOG_TIMEOUT = 30; -constexpr int EVERY_INTERVAL = 5; -#else -constexpr int WATCHDOG_TIMEOUT = 10; -constexpr int EVERY_INTERVAL = 5; -#endif - -class TestReactor : public NUClear::Reactor { -public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - start = NUClear::clock::now(); - count = 0; +template +struct Flag {}; - // Trigger the watchdog after WATCHDOG_TIMEOUT milliseconds - on>().then([this] { - end = NUClear::clock::now(); +class TestReactor : public test_util::TestBase { +public: + TestReactor(std::unique_ptr environment) + : TestBase(std::move(environment), false), start(NUClear::clock::now()) { - // When our watchdog eventually triggers, shutdown + on, 50, std::chrono::milliseconds>>().then([this] { + events.push_back("Watchdog 1 triggered @ " + rounded_time()); powerplant.shutdown(); }); - // Service the watchdog every EVERY_INTERVAL milliseconds, 20 times. Then let it expire to trigger and end the - // test. - on>().then([this] { - // service the watchdog - if (++count < 20) { - emit(ServiceWatchdog()); + on, 40, std::chrono::milliseconds>>().then([this] { + if (flag2++ < 3) { + events.push_back("Watchdog 2 triggered @ " + rounded_time()); + emit(ServiceWatchdog>()); } }); - } -}; -class TestReactorRuntimeArg : public NUClear::Reactor { -public: - TestReactorRuntimeArg(std::unique_ptr environment) : Reactor(std::move(environment)) { - - start = NUClear::clock::now(); - count = 0; - - // Trigger the watchdog after WATCHDOG_TIMEOUT milliseconds - on>(std::string("test a")) - .then([this] { - end_a = NUClear::clock::now(); - a_ended = true; - - // When our watchdog eventually triggers, shutdown - if (b_ended) { - powerplant.shutdown(); - } - }); - - // Trigger the watchdog after WATCHDOG_TIMEOUT milliseconds - on>(std::string("test b")) - .then([this] { - end_b = NUClear::clock::now(); - b_ended = true; - - // When our watchdog eventually triggers, shutdown - if (a_ended) { - powerplant.shutdown(); - } - }); - - // Service the watchdog every EVERY_INTERVAL milliseconds, 20 times. Then let it expire to trigger and end the - // test. - on>().then([this] { - // service the watchdog - if (++count < 20) { - emit(ServiceWatchdog(std::string("test a"))); - emit(ServiceWatchdog(std::string("test b"))); + // Watchdog with subtypes + on, 30, std::chrono::milliseconds>>('a').then([this] { + if (flag3a++ < 3) { + events.push_back("Watchdog 3A triggered @ " + rounded_time()); + emit(ServiceWatchdog>()); + emit(ServiceWatchdog>()); } }); + on, 20, std::chrono::milliseconds>>('b').then([this] { + if (flag3b++ < 3) { + events.push_back("Watchdog 3B triggered @ " + rounded_time()); + emit(ServiceWatchdog>()); + emit(ServiceWatchdog>()); + emit(ServiceWatchdog>('a')); + } + }); + + on, 10, std::chrono::milliseconds>>().then([this] { + if (flag4++ < 3) { + events.push_back("Watchdog 4 triggered @ " + rounded_time()); + emit(ServiceWatchdog>()); + emit(ServiceWatchdog>()); + emit(ServiceWatchdog>('a')); + emit(ServiceWatchdog>('b')); + } + }); + } + + std::string rounded_time() { + double diff = std::chrono::duration_cast>(NUClear::clock::now() - start).count(); + // Round to 100ths of a second + return std::to_string(int(std::round(diff * 100))); } + + NUClear::clock::time_point start; + int flag2{0}; + int flag3a{0}; + int flag3b{0}; + int flag4{0}; }; + } // namespace TEST_CASE("Testing the Watchdog Smart Type", "[api][watchdog]") { @@ -120,23 +97,27 @@ TEST_CASE("Testing the Watchdog Smart Type", "[api][watchdog]") { config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - - plant.start(); - - // Require that at least the minimum time interval to have run all Everys has passed - REQUIRE(end - start > std::chrono::milliseconds(20 * EVERY_INTERVAL)); -} - -TEST_CASE("Testing the Watchdog Smart Type with a sub type", "[api][watchdog][sub_type]") { - - NUClear::PowerPlant::Configuration config; - config.thread_count = 1; - NUClear::PowerPlant plant(config); - plant.install(); - plant.start(); - // Require that at least the minimum time interval to have run all Everys has passed - REQUIRE(end_a - start > std::chrono::milliseconds(20 * EVERY_INTERVAL)); - REQUIRE(end_b - start > std::chrono::milliseconds(20 * EVERY_INTERVAL)); + std::vector expected = { + "Watchdog 4 triggered @ 1", + "Watchdog 4 triggered @ 2", + "Watchdog 4 triggered @ 3", + "Watchdog 3B triggered @ 5", + "Watchdog 3B triggered @ 7", + "Watchdog 3B triggered @ 9", + "Watchdog 3A triggered @ 12", + "Watchdog 3A triggered @ 16", + "Watchdog 3A triggered @ 19", + "Watchdog 2 triggered @ 23", + "Watchdog 2 triggered @ 27", + "Watchdog 2 triggered @ 31", + "Watchdog 1 triggered @ 36", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From 919e41d6e88bc70c79a7ce0347dd936e58023747 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 13:03:48 +1000 Subject: [PATCH 024/176] remove weird comment --- src/util/FileDescriptor.hpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/util/FileDescriptor.hpp b/src/util/FileDescriptor.hpp index df5502cf9..4f7310dcc 100644 --- a/src/util/FileDescriptor.hpp +++ b/src/util/FileDescriptor.hpp @@ -26,18 +26,11 @@ namespace util { /** * @brief An RAII file descriptor. + * * @details This class represents an RAII file descriptor. - * It will close the file descriptor it holds on - * destruction. + * It will close the file descriptor it holds on destruction. */ class FileDescriptor { - - /** - * @brief Constructs a new RAII file descriptor. - * - * @param fd [description] - */ - public: FileDescriptor(); FileDescriptor(const fd_t& fd); From 46cd107b4701f9a002b8438f33ef7dd25b7b2cc8 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 13:14:51 +1000 Subject: [PATCH 025/176] Floor is safer than round --- tests/dsl/Watchdog.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/dsl/Watchdog.cpp b/tests/dsl/Watchdog.cpp index a4c368baf..cdbc7fbaa 100644 --- a/tests/dsl/Watchdog.cpp +++ b/tests/dsl/Watchdog.cpp @@ -37,13 +37,13 @@ class TestReactor : public test_util::TestBase { : TestBase(std::move(environment), false), start(NUClear::clock::now()) { on, 50, std::chrono::milliseconds>>().then([this] { - events.push_back("Watchdog 1 triggered @ " + rounded_time()); + events.push_back("Watchdog 1 triggered @ " + floored_time()); powerplant.shutdown(); }); on, 40, std::chrono::milliseconds>>().then([this] { if (flag2++ < 3) { - events.push_back("Watchdog 2 triggered @ " + rounded_time()); + events.push_back("Watchdog 2 triggered @ " + floored_time()); emit(ServiceWatchdog>()); } }); @@ -51,14 +51,14 @@ class TestReactor : public test_util::TestBase { // Watchdog with subtypes on, 30, std::chrono::milliseconds>>('a').then([this] { if (flag3a++ < 3) { - events.push_back("Watchdog 3A triggered @ " + rounded_time()); + events.push_back("Watchdog 3A triggered @ " + floored_time()); emit(ServiceWatchdog>()); emit(ServiceWatchdog>()); } }); on, 20, std::chrono::milliseconds>>('b').then([this] { if (flag3b++ < 3) { - events.push_back("Watchdog 3B triggered @ " + rounded_time()); + events.push_back("Watchdog 3B triggered @ " + floored_time()); emit(ServiceWatchdog>()); emit(ServiceWatchdog>()); emit(ServiceWatchdog>('a')); @@ -67,7 +67,7 @@ class TestReactor : public test_util::TestBase { on, 10, std::chrono::milliseconds>>().then([this] { if (flag4++ < 3) { - events.push_back("Watchdog 4 triggered @ " + rounded_time()); + events.push_back("Watchdog 4 triggered @ " + floored_time()); emit(ServiceWatchdog>()); emit(ServiceWatchdog>()); emit(ServiceWatchdog>('a')); @@ -76,10 +76,10 @@ class TestReactor : public test_util::TestBase { }); } - std::string rounded_time() { + std::string floored_time() { double diff = std::chrono::duration_cast>(NUClear::clock::now() - start).count(); // Round to 100ths of a second - return std::to_string(int(std::round(diff * 100))); + return std::to_string(int(std::floor(diff * 100))); } NUClear::clock::time_point start; @@ -107,12 +107,12 @@ TEST_CASE("Testing the Watchdog Smart Type", "[api][watchdog]") { "Watchdog 3B triggered @ 7", "Watchdog 3B triggered @ 9", "Watchdog 3A triggered @ 12", - "Watchdog 3A triggered @ 16", - "Watchdog 3A triggered @ 19", - "Watchdog 2 triggered @ 23", - "Watchdog 2 triggered @ 27", - "Watchdog 2 triggered @ 31", - "Watchdog 1 triggered @ 36", + "Watchdog 3A triggered @ 15", + "Watchdog 3A triggered @ 18", + "Watchdog 2 triggered @ 22", + "Watchdog 2 triggered @ 26", + "Watchdog 2 triggered @ 30", + "Watchdog 1 triggered @ 35", }; // Make an info print the diff in an easy to read way if we fail From 0cd84def522fb3324779c1cf7bb163b3519f633a Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 15:04:11 +1000 Subject: [PATCH 026/176] Update custom clock test --- tests/individual/CustomClock.cpp | 51 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/individual/CustomClock.cpp b/tests/individual/CustomClock.cpp index 17aab7369..3044c84c4 100644 --- a/tests/individual/CustomClock.cpp +++ b/tests/individual/CustomClock.cpp @@ -23,6 +23,8 @@ #define NUCLEAR_CUSTOM_CLOCK #include +#include "test_util/TestBase.hpp" + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); @@ -42,19 +44,20 @@ template struct Message {}; // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -std::vector times; -constexpr int n_time = 100; +std::vector> times; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - - // Running every this slowed down clock should execute slower - on>().then([this] { - times.push_back(std::chrono::steady_clock::now()); - if (times.size() > n_time) { - powerplant.shutdown(); - } + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { + + // Collect steady clock times as well as NUClear clock times + on>().then([] { // + times.emplace_back(std::chrono::steady_clock::now(), NUClear::clock::now()); + }); + + // Collect until the watchdog times out + on>().then([this] { // + powerplant.shutdown(); }); } }; @@ -65,24 +68,22 @@ TEST_CASE("Testing custom clock works correctly", "[api][custom_clock]") { NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); - - // We are installing with an initial log level of debug plant.install(); - plant.start(); - // Build up our difference vector - double total = 0.0; - for (size_t i = 0; i < times.size() - 1; ++i) { - total += (double((times[i + 1] - times[i]).count()) / double(std::nano::den)); + // Calculate the average ratio delta time for steady and custom clocks + double steady_total = 0; + double custom_total = 0; + + for (int i = 0; i + 1 < int(times.size()); ++i) { + using namespace std::chrono; + steady_total += duration_cast>(times[i + 1].first - times[i].first).count(); + custom_total += duration_cast>(times[i + 1].second - times[i].second).count(); } -#ifdef _WIN32 - const double timing_epsilon = 1e-2; -#else - const double timing_epsilon = 1e-3; -#endif + // The ratio should be about 0.5 + REQUIRE((custom_total / steady_total) == Approx(0.5)); - // The total should be about 2.0 - REQUIRE(total == Approx(2.0).epsilon(timing_epsilon)); + // The amount of time that passed should be (n - 1) * 2 * 10ms + REQUIRE(steady_total == Approx(2.0 * (times.size() - 1) * 1e-2)); } From 62cb8f05caa82427db324120d3e5435a8d3ca7d4 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 15:04:47 +1000 Subject: [PATCH 027/176] Finished time is closer to the correct time --- tests/individual/BaseClock.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/individual/BaseClock.cpp b/tests/individual/BaseClock.cpp index eeca152eb..cc707b7ce 100644 --- a/tests/individual/BaseClock.cpp +++ b/tests/individual/BaseClock.cpp @@ -28,6 +28,7 @@ #include #include "message/ReactionStatistics.hpp" +#include "test_util/TestBase.hpp" // Anonymous namespace to keep everything file local namespace { @@ -51,7 +52,7 @@ class TestReactor : public NUClear::Reactor { on>().then( [this](const NUClear::message::ReactionStatistics& stats) { const std::lock_guard lock(times_mutex); - times.push_back(std::make_pair(stats.emitted, std::chrono::system_clock::now())); + times.push_back(std::make_pair(stats.finished, std::chrono::system_clock::now())); if (times.size() > n_time) { powerplant.shutdown(); } From fc3664f4182c31443a9d019108684b4d97f77b95 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 15:19:33 +1000 Subject: [PATCH 028/176] Split off the DSL ordering test and add a proper With test --- tests/dsl/DSLOrdering.cpp | 89 ++++++++++++++++++++++++++++++++++++++ tests/dsl/With.cpp | 90 ++++++++++++++++++++++++++++----------- 2 files changed, 155 insertions(+), 24 deletions(-) create mode 100644 tests/dsl/DSLOrdering.cpp diff --git a/tests/dsl/DSLOrdering.cpp b/tests/dsl/DSLOrdering.cpp new file mode 100644 index 000000000..94959a8b6 --- /dev/null +++ b/tests/dsl/DSLOrdering.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2017 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include "test_util/TestBase.hpp" + +namespace { + +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +template +struct Message { + Message(std::string data) : data(std::move(data)) {} + std::string data; +}; + +class TestReactor : public test_util::TestBase { +public: + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { + // Check that the lists are combined, and that the function args are in order + on>, Trigger>, With>>().then( + [](const Message<1>& a, const Message<3>& c, const Message<2>& b) { + events.push_back("A:" + a.data + " B:" + b.data + " C:" + c.data); + }); + + // Make sure we can pass an empty function in here + on>, With, Message<2>>>().then([] { events.push_back("Empty function"); }); + + on>, Priority::LOW>().then([this] { + events.push_back("Emitting 1"); + emit(std::make_unique>("1")); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Emitting 2"); + emit(std::make_unique>("2")); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Emitting 3"); + emit(std::make_unique>("3")); + }); + + on().then([this] { + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + }); + } +}; +} // namespace + +TEST_CASE("Testing poorly ordered on arguments", "[api][dsl][order][with]") { + + NUClear::PowerPlant::Configuration config; + config.thread_count = 1; + NUClear::PowerPlant plant(config); + plant.install(); + plant.start(); + + std::vector expected = { + "Emitting 1", + "Emitting 2", + "Emitting 3", + "A:1 B:2 C:3", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); +} diff --git a/tests/dsl/With.cpp b/tests/dsl/With.cpp index d51b4077f..6fcc94b1b 100644 --- a/tests/dsl/With.cpp +++ b/tests/dsl/With.cpp @@ -19,45 +19,87 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { -struct DifferentOrderingMessage1 { - int a; -}; -struct DifferentOrderingMessage2 { - int a; +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +struct Message { + Message(std::string data) : data(std::move(data)) {} + std::string data; }; -struct DifferentOrderingMessage3 { - int a; +struct Data { + Data(std::string data) : data(std::move(data)) {} + std::string data; }; -class DifferentOrderingReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - DifferentOrderingReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { // Check that the lists are combined, and that the function args are in order - on, Trigger, With>().then( - [this](const DifferentOrderingMessage1&, - const DifferentOrderingMessage3&, - const DifferentOrderingMessage2&) { this->powerplant.shutdown(); }); - - // Make sure we can pass an empty function in here - on, With>().then( - [] {}); + on, With>().then([](const Message& m, const Data& d) { // + events.push_back("Message: " + m.data + " Data: " + d.data); + }); + + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Data 1"); + emit(std::make_unique("D1")); + }); + + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Data 2"); + emit(std::make_unique("D2")); + }); + + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Message 1"); + emit(std::make_unique("M1")); + }); + + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Data 3"); + emit(std::make_unique("D3")); + }); + + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Message 2"); + emit(std::make_unique("M2")); + }); + + on().then([this] { + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + }); } }; } // namespace -TEST_CASE("Testing poorly ordered on arguments", "[api][with]") { +TEST_CASE("Testing the with dsl keyword", "[api][with]") { NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); - plant.install(); + plant.install(); + plant.start(); + + std::vector expected = { + "Emitting Data 1", + "Emitting Data 2", + "Emitting Message 1", + "Message: M1 Data: D2", + "Emitting Data 3", + "Emitting Message 2", + "Message: M2 Data: D3", + }; - plant.emit(std::make_unique()); - plant.emit(std::make_unique()); - plant.emit(std::make_unique()); + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); - REQUIRE_NOTHROW(plant.start()); - REQUIRE_FALSE(plant.running()); + // Check the events fired in order and only those events + REQUIRE(events == expected); } From e2be9938579bedec102fe78c7d0629fd182f0ddf Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 15:20:05 +1000 Subject: [PATCH 029/176] Fix some tests --- tests/dsl/BlockNoData.cpp | 2 +- tests/dsl/DSLProxy.cpp | 8 +++++--- tests/dsl/FlagMessage.cpp | 2 +- tests/dsl/RawFunction.cpp | 8 ++++---- tests/dsl/Shutdown.cpp | 2 +- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/dsl/BlockNoData.cpp b/tests/dsl/BlockNoData.cpp index 858e20f61..5ed21751e 100644 --- a/tests/dsl/BlockNoData.cpp +++ b/tests/dsl/BlockNoData.cpp @@ -47,7 +47,7 @@ class TestReactor : public test_util::TestBase { events.push_back("MessageB with MessageA triggered"); }); - on>>().then([this] { + on>, Priority::LOW>().then([this] { events.push_back("Emitting MessageA"); emit(std::make_unique()); }); diff --git a/tests/dsl/DSLProxy.cpp b/tests/dsl/DSLProxy.cpp index b6db4556f..d665a317e 100644 --- a/tests/dsl/DSLProxy.cpp +++ b/tests/dsl/DSLProxy.cpp @@ -75,9 +75,11 @@ TEST_CASE("Testing that the DSL proxy works as expected for binding unmodifyable plant.install(); plant.start(); - std::vector expected = {"Emitting CustomMessage2", - "Emitting CustomMessage1", - "CustomMessage1 Triggered with 123456"}; + std::vector expected = { + "Emitting CustomMessage2", + "Emitting CustomMessage1", + "CustomMessage1 Triggered with 123456", + }; // Make an info print the diff in an easy to read way if we fail INFO(test_util::diff_string(expected, events)); diff --git a/tests/dsl/FlagMessage.cpp b/tests/dsl/FlagMessage.cpp index 2765575ed..9ad588436 100644 --- a/tests/dsl/FlagMessage.cpp +++ b/tests/dsl/FlagMessage.cpp @@ -51,7 +51,7 @@ class TestReactor : public test_util::TestBase { events.push_back("MessageA with MessageB triggered"); }); - on>>().then([this] { + on>, Priority::LOW>().then([this] { events.push_back("Step<1> triggered"); events.push_back("Emitting MessageA"); emit(std::make_unique()); diff --git a/tests/dsl/RawFunction.cpp b/tests/dsl/RawFunction.cpp index ff6397aee..21f68d471 100644 --- a/tests/dsl/RawFunction.cpp +++ b/tests/dsl/RawFunction.cpp @@ -85,10 +85,10 @@ class TestReactor : public test_util::TestBase { on, Trigger>().then(raw_function_test_right_arg); on, Trigger>().then(raw_function_test_both_args); - on>>().then([this] { emit(std::make_unique("D1")); }); - on>>().then([this] { emit(std::make_unique("M2")); }); - on>>().then([this] { emit(std::make_unique("D3")); }); - on>>().then([this] { emit(std::make_unique("M4")); }); + on>, Priority::LOW>().then([this] { emit(std::make_unique("D1")); }); + on>, Priority::LOW>().then([this] { emit(std::make_unique("M2")); }); + on>, Priority::LOW>().then([this] { emit(std::make_unique("D3")); }); + on>, Priority::LOW>().then([this] { emit(std::make_unique("M4")); }); on().then([this] { emit(std::make_unique>()); diff --git a/tests/dsl/Shutdown.cpp b/tests/dsl/Shutdown.cpp index 9481e14c7..16ac43011 100644 --- a/tests/dsl/Shutdown.cpp +++ b/tests/dsl/Shutdown.cpp @@ -35,7 +35,7 @@ class TestReactor : public test_util::TestBase { events.push_back("Shutdown task executed"); }); - on>>().then([this] { + on>, Priority::LOW>().then([this] { events.push_back("Requesting shutdown"); powerplant.shutdown(); }); From f05015b9b800c226c93d168bc605f233593adecf Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 15:25:24 +1000 Subject: [PATCH 030/176] Make watchdog behave like every with a stable tick rate --- src/dsl/word/Watchdog.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsl/word/Watchdog.hpp b/src/dsl/word/Watchdog.hpp index c01c21cc5..74aa6a66f 100644 --- a/src/dsl/word/Watchdog.hpp +++ b/src/dsl/word/Watchdog.hpp @@ -277,7 +277,7 @@ namespace dsl { } // Now automatically service the watchdog - time = NUClear::clock::now() + period(ticks); + time = time + period(ticks); } // Change our wait time to our new watchdog time else { From 43ac81096749a61d3b12c5d3366852c200af6942 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 15:49:10 +1000 Subject: [PATCH 031/176] Improve transients test --- tests/dsl/Transient.cpp | 161 ++++++++++++++++++++++++++++ tests/dsl/TransientMultiTrigger.cpp | 132 ----------------------- 2 files changed, 161 insertions(+), 132 deletions(-) create mode 100644 tests/dsl/Transient.cpp delete mode 100644 tests/dsl/TransientMultiTrigger.cpp diff --git a/tests/dsl/Transient.cpp b/tests/dsl/Transient.cpp new file mode 100644 index 000000000..6008d05bb --- /dev/null +++ b/tests/dsl/Transient.cpp @@ -0,0 +1,161 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2017 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include "test_util/TestBase.hpp" + +namespace { + +struct Message { + Message(std::string msg) : msg(std::move(msg)) {} + std::string msg; +}; + +struct TransientMessage { + TransientMessage(std::string msg = "", bool valid = false) : msg(std::move(msg)), valid(valid) {} + std::string msg; + bool valid; + + operator bool() const { + return valid; + } +}; +} // namespace + +namespace NUClear { +namespace dsl { + namespace trait { + template <> + struct is_transient : public std::true_type {}; + } // namespace trait +} // namespace dsl +} // namespace NUClear + +namespace { + +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +struct TransientGetter : public NUClear::dsl::operation::TypeBind { + + template + static inline TransientMessage get(NUClear::threading::Reaction& r) { + + // Get the real message and return it directly so transient can activate + auto raw = NUClear::dsl::operation::CacheGet::get(r); + if (raw == nullptr) { + return TransientMessage(); + } + return *raw; + } +}; + +class TestReactor : public test_util::TestBase { +public: + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { + + on, TransientGetter>().then( + [](const Message& m, const TransientMessage& t) { events.push_back(m.msg + " : " + t.msg); }); + + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Message 1"); + emit(std::make_unique("S1")); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Transient 1"); + emit(std::make_unique("T1", true)); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Message 2"); + emit(std::make_unique("S2")); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Invalid Transient 2"); + emit(std::make_unique("T2", false)); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Message 3"); + emit(std::make_unique("S3")); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Transient 3"); + emit(std::make_unique("T3", true)); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Transient 4"); + emit(std::make_unique("T4", true)); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Invalid Transient 5"); + emit(std::make_unique("T5", false)); + }); + on>, Priority::LOW>().then([this] { + events.push_back("Emitting Message 4"); + emit(std::make_unique("S4")); + }); + + on().then([this] { + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + emit(std::make_unique>()); + }); + } +}; +} // namespace + +TEST_CASE("Testing whether getters that return transient data can cache between calls", "[api][transient]") { + NUClear::PowerPlant::Configuration config; + config.thread_count = 1; + NUClear::PowerPlant plant(config); + plant.install(); + plant.start(); + + std::vector expected = { + "Emitting Message 1", + "Emitting Transient 1", + "S1 : T1", + "Emitting Message 2", + "S2 : T1", + "Emitting Invalid Transient 2", + "S2 : T1", + "Emitting Message 3", + "S3 : T1", + "Emitting Transient 3", + "S3 : T3", + "Emitting Transient 4", + "S3 : T4", + "Emitting Invalid Transient 5", + "S3 : T4", + "Emitting Message 4", + "S4 : T4", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); +} diff --git a/tests/dsl/TransientMultiTrigger.cpp b/tests/dsl/TransientMultiTrigger.cpp deleted file mode 100644 index 4c8e3789d..000000000 --- a/tests/dsl/TransientMultiTrigger.cpp +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -namespace { - -int value = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -std::vector> value_pairs; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -struct DataType { - int value; - bool good; - - operator bool() const { - return good; - } -}; -} // namespace - -namespace NUClear { -namespace dsl { - namespace trait { - template <> - struct is_transient : public std::true_type {}; - } // namespace trait -} // namespace dsl -} // namespace NUClear - -namespace { -struct SimpleMessage { - SimpleMessage(int value) : value(value){}; - int value; -}; - -struct TransientTypeGetter : public NUClear::dsl::operation::TypeBind { - - template - static inline DataType get(NUClear::threading::Reaction& /*unused*/) { - - // We say for this test that our data is valid if it is odd - return DataType{value, value % 2 == 1}; - } -}; - -class TestReactor : public NUClear::Reactor { -public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { - - on>().then( - [](const DataType& d, const SimpleMessage& m) { value_pairs.push_back(std::make_pair(m.value, d.value)); }); - - on().then([this] { - // Our data starts off as invalid - value = 0; - - // This should not start a run as our data is invalid - emit(std::make_unique(10)); - - // Change our value to 1, our transient data is now valid - value = 1; - - // This should execute our function resulting in the pair 10,1 - emit(std::make_unique(0)); - - // This should make our transient data invalid again - value = 2; - - // This should execute our function resulting in the pair 20,1 - emit(std::make_unique(20)); - - // This should update to a new good value - value = 5; - - // This should execute our function resulting in the pair 30,5 - emit(std::make_unique(30)); - - // This should execute our function resulting in the pair 30,5 - emit(std::make_unique(0)); - - // Value is now bad again - value = 10; - - // This should execute our function resulting in the pair 30,5 - emit(std::make_unique(0)); - // TODO(trent): technically the thing that triggered this resulted in invalid data but used old data, do we - // want to stop this? - // TODO(trent): This would result in two states, invalid data, and non existant data - - // We are finished the test - powerplant.shutdown(); - }); - } -}; -} // namespace - -TEST_CASE("Testing whether getters that return transient data can cache between calls", "[api][transient]") { - - NUClear::PowerPlant::Configuration config; - config.thread_count = 1; - NUClear::PowerPlant plant(config); - plant.install(); - - plant.start(); - - // Now we validate the list (which may be in a different order due to NUClear scheduling) - std::sort(std::begin(value_pairs), std::end(value_pairs)); - - // Check that it was all as expected - REQUIRE(value_pairs.size() == 5); - REQUIRE(value_pairs[0] == std::make_pair(10, 1)); - REQUIRE(value_pairs[1] == std::make_pair(20, 1)); - REQUIRE(value_pairs[2] == std::make_pair(30, 5)); - REQUIRE(value_pairs[3] == std::make_pair(30, 5)); - REQUIRE(value_pairs[4] == std::make_pair(30, 5)); -} From 715b5c7d75e4d3d2e3aa121bc4add5fd30b6a7f3 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 3 Aug 2023 16:23:54 +1000 Subject: [PATCH 032/176] Reaction statistics test --- tests/api/ReactionStatistics.cpp | 96 ++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 43 deletions(-) diff --git a/tests/api/ReactionStatistics.cpp b/tests/api/ReactionStatistics.cpp index fb7e14e64..21ac9538a 100644 --- a/tests/api/ReactionStatistics.cpp +++ b/tests/api/ReactionStatistics.cpp @@ -19,73 +19,67 @@ #include #include +#include "test_util/TestBase.hpp" + // Anonymous namespace to keep everything file local namespace { +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + template struct Message {}; - -struct LoopMsg {}; - -bool seen_message0 = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -bool seen_message_startup = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +struct LoopMessage {}; using NUClear::message::ReactionStatistics; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - on>().then("Reaction Stats Handler 2", [this](const ReactionStatistics&) { - // This reaction is just here to cause a potential looping of reaction statistics handlers - emit(std::make_unique()); + // This reaction is here to emit something from a ReactionStatistics trigger + // This shouldn't cause reaction statistics of their own otherwise everything would explode + on>().then("Loop Statistics", [this](const ReactionStatistics&) { + emit(std::make_unique()); }); + on>().then("No Statistics", [] {}); - on>().then("NoStats", [] { - // This guy is triggered by someone triggering on reaction statistics, don't run - }); - - on>().then("Reaction Stats Handler", [this](const ReactionStatistics& stats) { - // If we are seeing ourself, fail - REQUIRE(stats.identifiers.name != "Reaction Stats Handler"); - - // If we are seeing the other reaction statistics handler, fail - REQUIRE(stats.identifiers.name != "Reaction Stats Handler 2"); - - // If we are seeing the other reaction statistics handler, fail - REQUIRE(stats.identifiers.name != "NoStats"); - // Flag if we have seen the message handler - if (stats.identifiers.name == "Message Handler") { - seen_message0 = true; - } - // Flag if we have seen the startup handler - else if (stats.identifiers.name == "Startup Handler") { - seen_message_startup = true; + on>().then("Reaction Stats Handler", [](const ReactionStatistics& stats) { + // Other reactions statistics run on this because of built in NUClear reactors (e.g. chrono controller etc) + // We want to filter those out so only our own stats are shown + if (stats.identifiers.name.empty() + || stats.identifiers.reactor != NUClear::util::demangle(typeid(TestReactor).name())) { + return; } + events.push_back("Stats for " + stats.identifiers.name + " from " + stats.identifiers.reactor); + events.push_back(stats.identifiers.dsl); // Ensure exceptions are passed through correctly in the exception handler if (stats.exception) { - REQUIRE(stats.identifiers.name == "Exception Handler"); try { std::rethrow_exception(stats.exception); } catch (const std::exception& e) { - REQUIRE(seen_message0); - REQUIRE(seen_message_startup); - REQUIRE(std::string(e.what()) == std::string("Exceptions happened")); - - // We are done - powerplant.shutdown(); + events.push_back("Exception received: \"" + std::string(e.what()) + "\""); } } }); - on>>().then("Message Handler", [this] { emit(std::make_unique>()); }); + on>>().then("Exception Handler", [] { + events.push_back("Running Exception Handler"); + throw std::runtime_error("Text in an exception"); + }); - on>>().then("Exception Handler", [] { throw std::runtime_error("Exceptions happened"); }); + on>>().then("Message Handler", [this] { + events.push_back("Running Message Handler"); + emit(std::make_unique>()); + }); - on().then("Startup Handler", [this] { emit(std::make_unique>()); }); + on().then("Startup Handler", [this] { + events.push_back("Running Startup Handler"); + emit(std::make_unique>()); + }); } }; } // namespace @@ -95,9 +89,25 @@ TEST_CASE("Testing reaction statistics functionality", "[api][reactionstatistics NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); - - // We are installing with an initial log level of debug plant.install(); - plant.start(); + + std::vector expected = { + "Running Startup Handler", + "Stats for Startup Handler from (anonymous namespace)::TestReactor", + "NUClear::Reactor::on", + "Running Message Handler", + "Stats for Message Handler from (anonymous namespace)::TestReactor", + "NUClear::Reactor::on>>", + "Running Exception Handler", + "Stats for Exception Handler from (anonymous namespace)::TestReactor", + "NUClear::Reactor::on>>", + "Exception received: \"Text in an exception\"", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From 4670b6056293f55763ffe3e82d038b8cedd39d4f Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 16:08:38 +1000 Subject: [PATCH 033/176] revamp the TCP test --- src/dsl/word/TCP.hpp | 8 +- src/extension/IOController_Posix.hpp | 6 + src/util/FileDescriptor.cpp | 42 ++++--- src/util/FileDescriptor.hpp | 44 ++++--- tests/dsl/TCP.cpp | 181 +++++++++++---------------- 5 files changed, 141 insertions(+), 140 deletions(-) diff --git a/src/dsl/word/TCP.hpp b/src/dsl/word/TCP.hpp index 2b5a569ad..2c97974dd 100644 --- a/src/dsl/word/TCP.hpp +++ b/src/dsl/word/TCP.hpp @@ -84,7 +84,8 @@ namespace dsl { in_port_t port = 0) { // Make our socket - util::FileDescriptor fd = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + util::FileDescriptor fd(::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), + [](fd_t fd) { ::shutdown(fd, SHUT_RDWR); }); if (fd < 0) { throw std::system_error(network_errno, @@ -127,7 +128,10 @@ namespace dsl { reaction->unbinders.push_back([](const threading::Reaction& r) { r.reactor.emit(std::make_unique>(r.id)); }); - reaction->unbinders.push_back([cfd](const threading::Reaction&) { close(cfd); }); + reaction->unbinders.push_back([cfd](const threading::Reaction&) { + ::shutdown(cfd, SHUT_RDWR); + ::close(cfd); + }); auto io_config = std::make_unique(IOConfiguration{fd.release(), IO::READ, reaction}); diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index 4b2df200b..b6c99edf1 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -218,6 +218,12 @@ namespace extension { // TODO(trent): If we had a close, or error stop listening? } } + + if ((fd.revents & IO::CLOSE) != 0) { + // Remove all the reactions that wanted this fd and flag the list as dirty + reactions.erase(range.first, range.second); + dirty = true; + } } } diff --git a/src/util/FileDescriptor.cpp b/src/util/FileDescriptor.cpp index 6067eb06d..d624d4442 100644 --- a/src/util/FileDescriptor.cpp +++ b/src/util/FileDescriptor.cpp @@ -2,15 +2,28 @@ #include +#include "platform.hpp" + namespace NUClear { namespace util { - FileDescriptor::FileDescriptor() = default; - FileDescriptor::FileDescriptor(const fd_t& fd) : fd(fd) {} + FileDescriptor::FileDescriptor() = default; + FileDescriptor::FileDescriptor(const int& fd_, std::function cleanup_) + : fd(fd_), cleanup(std::move(cleanup_)) {} FileDescriptor::~FileDescriptor() { - close_fd(); + close(); + } + + void FileDescriptor::close() { + if (valid()) { + if (cleanup) { + cleanup(fd); + } + ::close(fd); + fd = INVALID_SOCKET; + } } FileDescriptor::FileDescriptor(FileDescriptor&& rhs) noexcept : fd{rhs.fd} { @@ -21,38 +34,35 @@ namespace util { FileDescriptor& FileDescriptor::operator=(FileDescriptor&& rhs) noexcept { if (this != &rhs) { - close_fd(); - fd = rhs.get(); - rhs.fd = INVALID_SOCKET; + close(); + fd = std::exchange(rhs.fd, INVALID_SOCKET); } return *this; } // No Lint: As we are giving access to a variable which can change state. - // NOLINTNEXTLINE(readability-make-member-function-const) file descriptors can be modified - fd_t FileDescriptor::get() { + // NOLINTNEXTLINE(readability-make-member-function-const) + int FileDescriptor::get() { return fd; } bool FileDescriptor::valid() const { +#ifdef _WIN32 return fd != INVALID_SOCKET; +#else + return ::fcntl(fd, F_GETFL) != -1 || errno != EBADF; +#endif } fd_t FileDescriptor::release() { return std::exchange(fd, INVALID_SOCKET); } - // NOLINTNEXTLINE(readability-make-member-function-const) file descriptors can be modified + // Should not be const as editing the file descriptor would change the state + // NOLINTNEXTLINE(readability-make-member-function-const) FileDescriptor::operator fd_t() { return fd; } - void FileDescriptor::close_fd() { - if (fd != INVALID_SOCKET) { - close(fd); - } - fd = INVALID_SOCKET; - } - } // namespace util } // namespace NUClear diff --git a/src/util/FileDescriptor.hpp b/src/util/FileDescriptor.hpp index 4f7310dcc..1549a6bf0 100644 --- a/src/util/FileDescriptor.hpp +++ b/src/util/FileDescriptor.hpp @@ -19,6 +19,8 @@ #ifndef NUCLEAR_UTIL_FILEDESCRIPTOR_HPP #define NUCLEAR_UTIL_FILEDESCRIPTOR_HPP +#include + #include "platform.hpp" namespace NUClear { @@ -32,8 +34,18 @@ namespace util { */ class FileDescriptor { public: + /** + * @brief Construct a File Descriptor with an invalid file descriptor. + */ FileDescriptor(); - FileDescriptor(const fd_t& fd); + + /** + * @brief Constructs a new RAII file descriptor. + * + * @param fd the file descriptor to hold + * @param cleanup an optional cleanup function to call on close + */ + FileDescriptor(const int& fd_, std::function cleanup_ = nullptr); // Don't allow copy construction or assignment FileDescriptor(const FileDescriptor&) = delete; @@ -44,7 +56,7 @@ namespace util { FileDescriptor& operator=(FileDescriptor&& rhs) noexcept; /** - * @brief Destruct the file descriptor, closes the held FD + * @brief Destruct the file descriptor, closes the held fd */ ~FileDescriptor(); @@ -53,8 +65,9 @@ namespace util { * * @return the file descriptor */ - // NOLINTNEXTLINE(readability-make-member-function-const) file descriptors can be modified - fd_t get(); + // No Lint: As we are giving access to a variable which can change state. + // NOLINTNEXTLINE(readability-make-member-function-const) + int get(); /** * @brief Returns if the currently held file descriptor is valid @@ -65,32 +78,29 @@ namespace util { bool valid() const; /** - * @brief Releases the file descriptor from RAII. - * @details This releases and returns the file descriptor - * it holds. This allows it to no longer be - * subject to closure on deletion. - * After this this FileDescriptor object will - * be invalidated. - * @return the held file descriptor + * @brief Close the currently held file descriptor */ - fd_t release(); + void close(); /** - * @brief Casts this to the held file descriptor + * @brief Release the currently held file descriptor * * @return the file descriptor */ - // NOLINTNEXTLINE(readability-make-member-function-const) file descriptors can be modified - operator fd_t(); + fd_t release(); /** - * @brief Close the current file descriptor, if it's valid + * @brief Implicitly convert this class to a file descriptor + * + * @return the file descriptor */ - void close_fd(); + operator fd_t(); private: /// @brief The held file descriptor fd_t fd{INVALID_SOCKET}; + /// @brief An optional cleanup function to call on close + std::function cleanup; }; } // namespace util diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index c291cfccc..b23b0b062 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -18,160 +18,131 @@ #include #include + +#include "test_util/TestBase.hpp" + namespace { +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + constexpr in_port_t PORT = 40009; -int messages_received = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) const std::string TEST_STRING = "Hello TCP World!"; -struct Message {}; +struct TestConnection { + TestConnection(std::string name, in_port_t port) : name(std::move(name)), port(port) {} + std::string name; + in_port_t port; +}; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + void handle_data(const std::string& name, const IO::Event& event) { + // We have data to read + if ((event.events & IO::READ) != 0) { + + // Read into the buffer + std::array buff{}; + ssize_t len = ::recv(event.fd, buff.data(), socklen_t(TEST_STRING.size()), 0); + if (len == 0) { + events.push_back(name + " closed"); + } + else { + events.push_back(name + " received: " + std::string(buff.data(), len)); + ::send(event.fd, buff.data(), socklen_t(len), 0); + } + } + } + + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { // Bind to a known port on(PORT).then([this](const TCP::Connection& connection) { + events.push_back("Known Port connection accepted"); on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { - // If we read 0 later it means orderly shutdown - ssize_t len = -1; - - // We have data to read - if ((event.events & IO::READ) != 0) { - - std::array buff = {0}; - - // Read into the buffer - len = ::recv(event.fd, buff.data(), static_cast(TEST_STRING.size()), 0); - - // 0 indicates orderly shutdown of the socket - if (len != 0) { - - // Test the data - REQUIRE(len == int(TEST_STRING.size())); - REQUIRE(TEST_STRING == std::string(buff.data())); - ++messages_received; - } - } - - // The connection was closed and the other test finished - if (len == 0 || ((event.events & IO::CLOSE) != 0) || messages_received == 2) { - if (messages_received == 2) { - known_port_fd.close_fd(); - powerplant.shutdown(); - } - } + handle_data("Known Port", event); }); }); // Bind to an unknown port and get the port number - in_port_t bound_port = 0; - std::tie(std::ignore, bound_port, std::ignore) = on().then([this](const TCP::Connection& connection) { + in_port_t ephemeral_port = 0; + std::tie(std::ignore, ephemeral_port, std::ignore) = on().then([this](const TCP::Connection& connection) { + events.push_back("Ephemeral Port connection accepted"); on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { - // If we read 0 later it means orderly shutdown - ssize_t len = -1; - - // We have data to read - if ((event.events & IO::READ) != 0) { - - std::array buff = {0}; - - // Read into the buffer - len = ::recv(event.fd, buff.data(), static_cast(TEST_STRING.size()), 0); - - // 0 indicates orderly shutdown of the socket - if (len != 0) { - // Test the data - REQUIRE(len == int(TEST_STRING.size())); - REQUIRE(TEST_STRING == std::string(buff.data())); - ++messages_received; - } - } - - // The connection was closed and the other test finished - if (len == 0 || ((event.events & IO::CLOSE) != 0) || messages_received == 2) { - if (messages_received == 2) { - bound_port_fd.close_fd(); - powerplant.shutdown(); - } - } + handle_data("Ephemeral Port", event); }); }); // Send a test message to the known port - on>().then([this] { + on, Sync>().then([](const TestConnection& target) { // Open a random socket - known_port_fd = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - - // Our address to our local connection - sockaddr_in address{}; - address.sin_family = AF_INET; - address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - address.sin_port = htons(PORT); + NUClear::util::FileDescriptor fd(::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), + [](int fd) { ::shutdown(fd, SHUT_RDWR); }); - // Connect to ourself - REQUIRE(::connect(known_port_fd, reinterpret_cast(&address), sizeof(address)) == 0); - - // Set linger so we ensure sending all data - linger l{1, 2}; - REQUIRE(::setsockopt(known_port_fd, SOL_SOCKET, SO_LINGER, reinterpret_cast(&l), sizeof(linger)) - == 0); - - // Write on our socket - const ssize_t sent = - ::send(known_port_fd, TEST_STRING.data(), static_cast(TEST_STRING.size()), 0); - - // We must have sent the right amount of data - REQUIRE(sent == int(TEST_STRING.size())); - }); - - // Send a test message to the freely bound port - on>().then([this, bound_port] { - // Open a random socket - bound_port_fd = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (fd.valid() == false) { + throw std::runtime_error("Failed to create socket"); + } // Our address to our local connection sockaddr_in address{}; address.sin_family = AF_INET; address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - address.sin_port = htons(bound_port); + address.sin_port = htons(target.port); // Connect to ourself - REQUIRE(::connect(bound_port_fd, reinterpret_cast(&address), sizeof(address)) == 0); - - // Set linger so we ensure sending all data - linger l{1, 2}; - REQUIRE(::setsockopt(bound_port_fd, SOL_SOCKET, SO_LINGER, reinterpret_cast(&l), sizeof(linger)) - == 0); + if (::connect(fd, reinterpret_cast(&address), sizeof(address)) != 0) { + throw std::runtime_error("Failed to connect to socket"); + } // Write on our socket - const ssize_t sent = - ::send(bound_port_fd, TEST_STRING.data(), static_cast(TEST_STRING.size()), 0); + events.push_back(target.name + " sending"); + auto written = ::send(fd, TEST_STRING.data(), socklen_t(TEST_STRING.size()), 0); - // We must have sent the right amount of data - REQUIRE(sent == int(TEST_STRING.size())); + // Receive the echo + std::array buff{}; + const ssize_t recv = ::recv(fd, buff.data(), socklen_t(TEST_STRING.size()), 0); + events.push_back(target.name + " echoed: " + std::string(buff.data(), recv)); }); - on().then([this] { + on().then([this, ephemeral_port] { // Emit a message just so it will be when everything is running - emit(std::make_unique()); + emit(std::make_unique("Known Port", PORT)); + emit(std::make_unique("Ephemeral Port", ephemeral_port)); }); } private: NUClear::util::FileDescriptor known_port_fd; - NUClear::util::FileDescriptor bound_port_fd; + NUClear::util::FileDescriptor ephemeral_port_fd; }; } // namespace TEST_CASE("Testing listening for TCP connections and receiving data messages", "[api][network][tcp]") { NUClear::PowerPlant::Configuration config; - config.thread_count = 1; + config.thread_count = 4; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + std::vector expected = { + "Known Port sending", + "Known Port connection accepted", + "Known Port received: Hello TCP World!", + "Known Port echoed: Hello TCP World!", + "Known Port closed", + "Ephemeral Port sending", + "Ephemeral Port connection accepted", + "Ephemeral Port received: Hello TCP World!", + "Ephemeral Port echoed: Hello TCP World!", + "Ephemeral Port closed", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From 5bd10068757fb95009b508074cf07baa04b59551 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 16:08:54 +1000 Subject: [PATCH 034/176] error --- tests/dsl/TCP.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index b23b0b062..1e1e5607f 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -98,7 +98,7 @@ class TestReactor : public test_util::TestBase { // Write on our socket events.push_back(target.name + " sending"); - auto written = ::send(fd, TEST_STRING.data(), socklen_t(TEST_STRING.size()), 0); + ::send(fd, TEST_STRING.data(), socklen_t(TEST_STRING.size()), 0); // Receive the echo std::array buff{}; From 10b4f2292eb86a6c4dd323530d539793228eaf7d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 16:35:17 +1000 Subject: [PATCH 035/176] Make ReactionStatistics test more repeatable --- src/Reactor.hpp | 6 +++--- tests/api/ReactionStatistics.cpp | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Reactor.hpp b/src/Reactor.hpp index 9553b699a..b526b0c42 100644 --- a/src/Reactor.hpp +++ b/src/Reactor.hpp @@ -301,9 +301,9 @@ class Reactor { auto then(const std::string& label, Function&& callback, const util::Sequence& /*s*/) { // Regex replace NUClear::dsl::Parse with NUClear::Reactor::on so that it reads more like what is expected - std::string dsl = std::regex_replace(util::demangle(typeid(DSL).name()), - std::regex("NUClear::dsl::Parse<"), - "NUClear::Reactor::on<"); + std::string dsl = util::demangle(typeid(DSL).name()); + dsl = std::regex_replace(dsl, std::regex("NUClear::dsl::Parse<"), "NUClear::Reactor::on<"); + dsl = std::regex_replace(dsl, std::regex(R"(\s+)"), ""); // Generate the identifier threading::ReactionIdentifiers identifiers{label, diff --git a/tests/api/ReactionStatistics.cpp b/tests/api/ReactionStatistics.cpp index 21ac9538a..38b1614de 100644 --- a/tests/api/ReactionStatistics.cpp +++ b/tests/api/ReactionStatistics.cpp @@ -21,8 +21,8 @@ #include "test_util/TestBase.hpp" -// Anonymous namespace to keep everything file local -namespace { +// This namespace is named to make things consistent with the reaction statistics test +namespace stats_test { /// @brief Events that occur during the test std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) @@ -82,32 +82,32 @@ class TestReactor : public test_util::TestBase { }); } }; -} // namespace +} // namespace stats_test TEST_CASE("Testing reaction statistics functionality", "[api][reactionstatistics]") { NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); - plant.install(); + plant.install(); plant.start(); std::vector expected = { "Running Startup Handler", - "Stats for Startup Handler from (anonymous namespace)::TestReactor", + "Stats for Startup Handler from stats_test::TestReactor", "NUClear::Reactor::on", "Running Message Handler", - "Stats for Message Handler from (anonymous namespace)::TestReactor", - "NUClear::Reactor::on>>", + "Stats for Message Handler from stats_test::TestReactor", + "NUClear::Reactor::on>>", "Running Exception Handler", - "Stats for Exception Handler from (anonymous namespace)::TestReactor", - "NUClear::Reactor::on>>", + "Stats for Exception Handler from stats_test::TestReactor", + "NUClear::Reactor::on>>", "Exception received: \"Text in an exception\"", }; // Make an info print the diff in an easy to read way if we fail - INFO(test_util::diff_string(expected, events)); + INFO(test_util::diff_string(expected, stats_test::events)); // Check the events fired in order and only those events - REQUIRE(events == expected); + REQUIRE(stats_test::events == expected); } From 286a25864a9e3a492eef083624e3f87c10fc3d3d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 16:37:10 +1000 Subject: [PATCH 036/176] Include fnctl --- src/util/platform.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util/platform.hpp b/src/util/platform.hpp index 4d8afbcf1..741f623e8 100644 --- a/src/util/platform.hpp +++ b/src/util/platform.hpp @@ -145,6 +145,7 @@ int sendmsg(fd_t fd, msghdr* msg, int flags); // Include real networking stuff #include + #include #include #include #include From f35f6d633334c9180af390a36b42d5279fc14505 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 16:41:12 +1000 Subject: [PATCH 037/176] ok --- src/util/platform.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util/platform.hpp b/src/util/platform.hpp index 741f623e8..95f8597a8 100644 --- a/src/util/platform.hpp +++ b/src/util/platform.hpp @@ -156,6 +156,8 @@ int sendmsg(fd_t fd, msghdr* msg, int flags); #include #include + #include + // Move errno so it can be used in windows #define network_errno errno #define INVALID_SOCKET (-1) From d9cd03ac572ea753081cfa003a45f598370f88c3 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 16:46:53 +1000 Subject: [PATCH 038/176] Windows error --- src/util/FileDescriptor.cpp | 2 +- src/util/FileDescriptor.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/FileDescriptor.cpp b/src/util/FileDescriptor.cpp index d624d4442..741408006 100644 --- a/src/util/FileDescriptor.cpp +++ b/src/util/FileDescriptor.cpp @@ -9,7 +9,7 @@ namespace util { FileDescriptor::FileDescriptor() = default; - FileDescriptor::FileDescriptor(const int& fd_, std::function cleanup_) + FileDescriptor::FileDescriptor(const int& fd_, std::function cleanup_) : fd(fd_), cleanup(std::move(cleanup_)) {} FileDescriptor::~FileDescriptor() { diff --git a/src/util/FileDescriptor.hpp b/src/util/FileDescriptor.hpp index 1549a6bf0..ddea08bf4 100644 --- a/src/util/FileDescriptor.hpp +++ b/src/util/FileDescriptor.hpp @@ -45,7 +45,7 @@ namespace util { * @param fd the file descriptor to hold * @param cleanup an optional cleanup function to call on close */ - FileDescriptor(const int& fd_, std::function cleanup_ = nullptr); + FileDescriptor(const int& fd_, std::function cleanup_ = nullptr); // Don't allow copy construction or assignment FileDescriptor(const FileDescriptor&) = delete; From abcdcfb5eda77f561f33ca80e565eacd48541353 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 16:47:29 +1000 Subject: [PATCH 039/176] clang-tidy --- tests/individual/CustomClock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/individual/CustomClock.cpp b/tests/individual/CustomClock.cpp index 3044c84c4..dcb6aac7c 100644 --- a/tests/individual/CustomClock.cpp +++ b/tests/individual/CustomClock.cpp @@ -76,7 +76,7 @@ TEST_CASE("Testing custom clock works correctly", "[api][custom_clock]") { double custom_total = 0; for (int i = 0; i + 1 < int(times.size()); ++i) { - using namespace std::chrono; + using namespace std::chrono; // NOLINT(google-build-using-namespace) fine in function scope steady_total += duration_cast>(times[i + 1].first - times[i].first).count(); custom_total += duration_cast>(times[i + 1].second - times[i].second).count(); } From 0943208cd0180c4e6eea3488f1b21839253e2ef8 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 17:02:14 +1000 Subject: [PATCH 040/176] /shrug --- src/util/FileDescriptor.cpp | 8 ++++++-- src/util/FileDescriptor.hpp | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/util/FileDescriptor.cpp b/src/util/FileDescriptor.cpp index 741408006..52e5244b4 100644 --- a/src/util/FileDescriptor.cpp +++ b/src/util/FileDescriptor.cpp @@ -21,7 +21,11 @@ namespace util { if (cleanup) { cleanup(fd); } +#ifdef _WIN32 + ::closesocket(fd); +#else ::close(fd); +#endif fd = INVALID_SOCKET; } } @@ -34,7 +38,7 @@ namespace util { FileDescriptor& FileDescriptor::operator=(FileDescriptor&& rhs) noexcept { if (this != &rhs) { - close(); + this->close(); fd = std::exchange(rhs.fd, INVALID_SOCKET); } return *this; @@ -42,7 +46,7 @@ namespace util { // No Lint: As we are giving access to a variable which can change state. // NOLINTNEXTLINE(readability-make-member-function-const) - int FileDescriptor::get() { + fd_t FileDescriptor::get() { return fd; } diff --git a/src/util/FileDescriptor.hpp b/src/util/FileDescriptor.hpp index ddea08bf4..d6c4f3192 100644 --- a/src/util/FileDescriptor.hpp +++ b/src/util/FileDescriptor.hpp @@ -100,7 +100,7 @@ namespace util { /// @brief The held file descriptor fd_t fd{INVALID_SOCKET}; /// @brief An optional cleanup function to call on close - std::function cleanup; + std::function cleanup; }; } // namespace util From a435df9fd027493756c292ee17e6c28fab42dcbb Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 18:19:11 +1000 Subject: [PATCH 041/176] Make test end at the end --- tests/dsl/TCP.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 1e1e5607f..c9529deea 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -37,7 +37,9 @@ struct TestConnection { in_port_t port; }; -class TestReactor : public test_util::TestBase { +struct FinishTest {}; + +class TestReactor : public test_util::TestBase { public: void handle_data(const std::string& name, const IO::Event& event) { // We have data to read @@ -48,6 +50,7 @@ class TestReactor : public test_util::TestBase { ssize_t len = ::recv(event.fd, buff.data(), socklen_t(TEST_STRING.size()), 0); if (len == 0) { events.push_back(name + " closed"); + emit(std::make_unique()); } else { events.push_back(name + " received: " + std::string(buff.data(), len)); @@ -106,6 +109,14 @@ class TestReactor : public test_util::TestBase { events.push_back(target.name + " echoed: " + std::string(buff.data(), recv)); }); + on, Sync>().then([this] { + ++count; + if (count == 2) { + events.push_back("Finishing Test"); + powerplant.shutdown(); + } + }); + on().then([this, ephemeral_port] { // Emit a message just so it will be when everything is running emit(std::make_unique("Known Port", PORT)); @@ -116,13 +127,15 @@ class TestReactor : public test_util::TestBase { private: NUClear::util::FileDescriptor known_port_fd; NUClear::util::FileDescriptor ephemeral_port_fd; + // How many tests were run + int count = 0; }; } // namespace TEST_CASE("Testing listening for TCP connections and receiving data messages", "[api][network][tcp]") { NUClear::PowerPlant::Configuration config; - config.thread_count = 4; + config.thread_count = 2; NUClear::PowerPlant plant(config); plant.install(); plant.start(); @@ -138,6 +151,7 @@ TEST_CASE("Testing listening for TCP connections and receiving data messages", " "Ephemeral Port received: Hello TCP World!", "Ephemeral Port echoed: Hello TCP World!", "Ephemeral Port closed", + "Finishing Test", }; // Make an info print the diff in an easy to read way if we fail From 07af03fde3d90c2e758971d1b4420ccfb8b1d6f4 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 19:06:37 +1000 Subject: [PATCH 042/176] Fix --- src/util/FileDescriptor.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/FileDescriptor.hpp b/src/util/FileDescriptor.hpp index d6c4f3192..8fea44dba 100644 --- a/src/util/FileDescriptor.hpp +++ b/src/util/FileDescriptor.hpp @@ -67,7 +67,7 @@ namespace util { */ // No Lint: As we are giving access to a variable which can change state. // NOLINTNEXTLINE(readability-make-member-function-const) - int get(); + fd_t get(); /** * @brief Returns if the currently held file descriptor is valid From 1b582475269b4c59c4e05152916e6d6bab075830 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 19:07:27 +1000 Subject: [PATCH 043/176] clang-tidy --- tests/dsl/TCP.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index c9529deea..9a1408f3d 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -84,7 +84,7 @@ class TestReactor : public test_util::TestBase { NUClear::util::FileDescriptor fd(::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), [](int fd) { ::shutdown(fd, SHUT_RDWR); }); - if (fd.valid() == false) { + if (!fd.valid()) { throw std::runtime_error("Failed to create socket"); } From f513224706501337d051262389446afa5a7c1170 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 19:40:18 +1000 Subject: [PATCH 044/176] clang-tidy --- tests/dsl/Transient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dsl/Transient.cpp b/tests/dsl/Transient.cpp index 6008d05bb..86555dfe2 100644 --- a/tests/dsl/Transient.cpp +++ b/tests/dsl/Transient.cpp @@ -61,7 +61,7 @@ struct TransientGetter : public NUClear::dsl::operation::TypeBind::get(r); if (raw == nullptr) { - return TransientMessage(); + return {}; } return *raw; } From 929d4a121c55f7db6e31214dfbe7e6104f58b5ec Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 20:13:44 +1000 Subject: [PATCH 045/176] clang-tidy --- tests/dsl/Watchdog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dsl/Watchdog.cpp b/tests/dsl/Watchdog.cpp index cdbc7fbaa..c61be7e38 100644 --- a/tests/dsl/Watchdog.cpp +++ b/tests/dsl/Watchdog.cpp @@ -76,7 +76,7 @@ class TestReactor : public test_util::TestBase { }); } - std::string floored_time() { + static std::string floored_time() const { double diff = std::chrono::duration_cast>(NUClear::clock::now() - start).count(); // Round to 100ths of a second return std::to_string(int(std::floor(diff * 100))); From a5ea9258e83cfcd67b549628471664379c6d0479 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 23:21:39 +1000 Subject: [PATCH 046/176] . --- tests/dsl/Watchdog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dsl/Watchdog.cpp b/tests/dsl/Watchdog.cpp index c61be7e38..541d9641b 100644 --- a/tests/dsl/Watchdog.cpp +++ b/tests/dsl/Watchdog.cpp @@ -76,7 +76,7 @@ class TestReactor : public test_util::TestBase { }); } - static std::string floored_time() const { + static std::string floored_time() { double diff = std::chrono::duration_cast>(NUClear::clock::now() - start).count(); // Round to 100ths of a second return std::to_string(int(std::floor(diff * 100))); From f49212d74c2bc599621389c7ab2df9f9a6f6e336 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 4 Aug 2023 23:22:13 +1000 Subject: [PATCH 047/176] . --- tests/dsl/Watchdog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dsl/Watchdog.cpp b/tests/dsl/Watchdog.cpp index 541d9641b..b64296286 100644 --- a/tests/dsl/Watchdog.cpp +++ b/tests/dsl/Watchdog.cpp @@ -76,7 +76,7 @@ class TestReactor : public test_util::TestBase { }); } - static std::string floored_time() { + std::string floored_time() const { double diff = std::chrono::duration_cast>(NUClear::clock::now() - start).count(); // Round to 100ths of a second return std::to_string(int(std::floor(diff * 100))); From 2b5c7e6e6a912bf8427ae4765099ed6de309bacd Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Sat, 5 Aug 2023 08:48:56 +1000 Subject: [PATCH 048/176] windows --- src/util/platform.hpp | 5 +++++ tests/dsl/TCP.cpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/util/platform.hpp b/src/util/platform.hpp index 95f8597a8..0a98c9dfc 100644 --- a/src/util/platform.hpp +++ b/src/util/platform.hpp @@ -78,6 +78,11 @@ // Whoever thought this was a good idea was a terrible person #undef ERROR + // Make the windows shutdown functions look like the posix ones + #define SHUT_RD SD_RECEIVE + #define SHUT_WR SD_RECEIVE + #define SHUT_RDWR SD_BOTH + #endif // _WIN32 /******************************************* diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 9a1408f3d..ad3998f91 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -82,7 +82,7 @@ class TestReactor : public test_util::TestBase { on, Sync>().then([](const TestConnection& target) { // Open a random socket NUClear::util::FileDescriptor fd(::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), - [](int fd) { ::shutdown(fd, SHUT_RDWR); }); + [](NUClear::fd_t fd) { ::shutdown(fd, SHUT_RDWR); }); if (!fd.valid()) { throw std::runtime_error("Failed to create socket"); From ffbf130ca1bcf05e5e6f1faf55128506ef630899 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Sat, 5 Aug 2023 19:31:12 +1000 Subject: [PATCH 049/176] Windows!!! "shakes fist" --- src/dsl/word/TCP.hpp | 4 ++++ src/util/FileDescriptor.cpp | 2 +- src/util/FileDescriptor.hpp | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/dsl/word/TCP.hpp b/src/dsl/word/TCP.hpp index 2c97974dd..5d69f8ce4 100644 --- a/src/dsl/word/TCP.hpp +++ b/src/dsl/word/TCP.hpp @@ -130,7 +130,11 @@ namespace dsl { }); reaction->unbinders.push_back([cfd](const threading::Reaction&) { ::shutdown(cfd, SHUT_RDWR); +#ifdef _WIN32 + ::closesocket(cfd); +#else ::close(cfd); +#endif }); auto io_config = std::make_unique(IOConfiguration{fd.release(), IO::READ, reaction}); diff --git a/src/util/FileDescriptor.cpp b/src/util/FileDescriptor.cpp index 52e5244b4..9517efb88 100644 --- a/src/util/FileDescriptor.cpp +++ b/src/util/FileDescriptor.cpp @@ -9,7 +9,7 @@ namespace util { FileDescriptor::FileDescriptor() = default; - FileDescriptor::FileDescriptor(const int& fd_, std::function cleanup_) + FileDescriptor::FileDescriptor(const fd_t& fd_, std::function cleanup_) : fd(fd_), cleanup(std::move(cleanup_)) {} FileDescriptor::~FileDescriptor() { diff --git a/src/util/FileDescriptor.hpp b/src/util/FileDescriptor.hpp index 8fea44dba..f209bf15a 100644 --- a/src/util/FileDescriptor.hpp +++ b/src/util/FileDescriptor.hpp @@ -45,7 +45,7 @@ namespace util { * @param fd the file descriptor to hold * @param cleanup an optional cleanup function to call on close */ - FileDescriptor(const int& fd_, std::function cleanup_ = nullptr); + FileDescriptor(const fd_t& fd_, std::function cleanup_ = nullptr); // Don't allow copy construction or assignment FileDescriptor(const FileDescriptor&) = delete; From 7473fd0fd52747e8fab2669611841dd3366a6e7a Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Sat, 5 Aug 2023 19:44:23 +1000 Subject: [PATCH 050/176] windows wants to be difficult --- src/Reactor.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Reactor.hpp b/src/Reactor.hpp index b526b0c42..40789b219 100644 --- a/src/Reactor.hpp +++ b/src/Reactor.hpp @@ -302,7 +302,8 @@ class Reactor { // Regex replace NUClear::dsl::Parse with NUClear::Reactor::on so that it reads more like what is expected std::string dsl = util::demangle(typeid(DSL).name()); - dsl = std::regex_replace(dsl, std::regex("NUClear::dsl::Parse<"), "NUClear::Reactor::on<"); + dsl = std::regex_replace(dsl, std::regex(R"(NUClear::dsl::Parse<)"), "NUClear::Reactor::on<"); + dsl = std::regex_replace(dsl, std::regex(R"(struct\s+)"), ""); dsl = std::regex_replace(dsl, std::regex(R"(\s+)"), ""); // Generate the identifier From 41c91b51d7c1231a21ff3014520a346c9580e491 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 7 Aug 2023 15:55:30 +1000 Subject: [PATCH 051/176] more windows nonsense --- src/Reactor.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Reactor.hpp b/src/Reactor.hpp index 40789b219..422a6f904 100644 --- a/src/Reactor.hpp +++ b/src/Reactor.hpp @@ -304,6 +304,7 @@ class Reactor { std::string dsl = util::demangle(typeid(DSL).name()); dsl = std::regex_replace(dsl, std::regex(R"(NUClear::dsl::Parse<)"), "NUClear::Reactor::on<"); dsl = std::regex_replace(dsl, std::regex(R"(struct\s+)"), ""); + dsl = std::regex_replace(dsl, std::regex(R"(class\s+)"), ""); dsl = std::regex_replace(dsl, std::regex(R"(\s+)"), ""); // Generate the identifier From dcb30444047f6dc2f8e24ebac8a8770a700ab788 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 7 Aug 2023 16:36:28 +1000 Subject: [PATCH 052/176] clang tidy rules --- tests/api/ReactionHandle.cpp | 2 +- tests/api/ReactionStatistics.cpp | 2 +- tests/dsl/Always.cpp | 2 +- tests/dsl/ArgumentFission.cpp | 2 +- tests/dsl/BlockNoData.cpp | 2 +- tests/dsl/Buffer.cpp | 2 +- tests/dsl/CommandLineArguments.cpp | 2 +- tests/dsl/CustomGet.cpp | 2 +- tests/dsl/DSLOrdering.cpp | 2 +- tests/dsl/DSLProxy.cpp | 2 +- tests/dsl/FlagMessage.cpp | 2 +- tests/dsl/Last.cpp | 2 +- tests/dsl/MainThread.cpp | 2 +- tests/dsl/MissingArguments.cpp | 2 +- tests/dsl/Once.cpp | 2 +- tests/dsl/Optional.cpp | 2 +- tests/dsl/Priority.cpp | 2 +- tests/dsl/RawFunction.cpp | 2 +- tests/dsl/Shutdown.cpp | 2 +- tests/dsl/Startup.cpp | 2 +- tests/dsl/Sync.cpp | 2 +- tests/dsl/TCP.cpp | 2 +- tests/dsl/Transient.cpp | 2 +- tests/dsl/Trigger.cpp | 2 +- tests/dsl/Watchdog.cpp | 2 +- tests/dsl/With.cpp | 2 +- tests/test_util/diff_string.cpp | 10 +++++----- tests/test_util/lcs.hpp | 16 ++++++++-------- 28 files changed, 39 insertions(+), 39 deletions(-) diff --git a/tests/api/ReactionHandle.cpp b/tests/api/ReactionHandle.cpp index a80912d8e..eeb412c46 100644 --- a/tests/api/ReactionHandle.cpp +++ b/tests/api/ReactionHandle.cpp @@ -71,7 +71,7 @@ TEST_CASE("Testing reaction handle functionality", "[api][reactionhandle]") { plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Executed toggled reaction 0", "Executed enabled reaction 0", "Executed enabled reaction 1", diff --git a/tests/api/ReactionStatistics.cpp b/tests/api/ReactionStatistics.cpp index 38b1614de..74e4e1c92 100644 --- a/tests/api/ReactionStatistics.cpp +++ b/tests/api/ReactionStatistics.cpp @@ -92,7 +92,7 @@ TEST_CASE("Testing reaction statistics functionality", "[api][reactionstatistics plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Running Startup Handler", "Stats for Startup Handler from stats_test::TestReactor", "NUClear::Reactor::on", diff --git a/tests/dsl/Always.cpp b/tests/dsl/Always.cpp index 8da97425f..f7853c6ea 100644 --- a/tests/dsl/Always.cpp +++ b/tests/dsl/Always.cpp @@ -63,7 +63,7 @@ TEST_CASE("The Always DSL keyword runs continuously when it can", "[api][always] plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Always 0", "Always 1", "Always 2", diff --git a/tests/dsl/ArgumentFission.cpp b/tests/dsl/ArgumentFission.cpp index 5c8279845..6b59acab8 100644 --- a/tests/dsl/ArgumentFission.cpp +++ b/tests/dsl/ArgumentFission.cpp @@ -91,7 +91,7 @@ TEST_CASE("Testing distributing arguments to multiple bind functions (NUClear Fi plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Bind1 with 5 and false called", "Bind2 with Hello and 2000000000 called", "Bind3 with 9, 10 and 11000000000 called", diff --git a/tests/dsl/BlockNoData.cpp b/tests/dsl/BlockNoData.cpp index 5ed21751e..807c96113 100644 --- a/tests/dsl/BlockNoData.cpp +++ b/tests/dsl/BlockNoData.cpp @@ -69,7 +69,7 @@ TEST_CASE("Testing that when an on statement does not have it's data satisfied i plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Emitting MessageA", "MessageA triggered", "Emitting MessageB", diff --git a/tests/dsl/Buffer.cpp b/tests/dsl/Buffer.cpp index 21d8392fa..200a5fc76 100644 --- a/tests/dsl/Buffer.cpp +++ b/tests/dsl/Buffer.cpp @@ -101,7 +101,7 @@ TEST_CASE("Test that Buffer and Single limit the number of concurrent executions plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Step 1", "Trigger reaction 1", "Single reaction 1", diff --git a/tests/dsl/CommandLineArguments.cpp b/tests/dsl/CommandLineArguments.cpp index e3ab6b1a3..0901c1262 100644 --- a/tests/dsl/CommandLineArguments.cpp +++ b/tests/dsl/CommandLineArguments.cpp @@ -54,7 +54,7 @@ TEST_CASE("Testing the Command Line argument capturing", "[api][command_line_arg plant.install(); plant.start(); - std::vector expected = {"CommandLineArguments: Hello World "}; + const std::vector expected = {"CommandLineArguments: Hello World "}; // Make an info print the diff in an easy to read way if we fail INFO(test_util::diff_string(expected, events)); diff --git a/tests/dsl/CustomGet.cpp b/tests/dsl/CustomGet.cpp index bbf5304fa..a588a469c 100644 --- a/tests/dsl/CustomGet.cpp +++ b/tests/dsl/CustomGet.cpp @@ -60,7 +60,7 @@ TEST_CASE("Test a custom reactor that returns a type that needs dereferencing", plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Emitting CustomGet", "CustomGet Triggered", "Data from a custom getter", diff --git a/tests/dsl/DSLOrdering.cpp b/tests/dsl/DSLOrdering.cpp index 94959a8b6..c8bd279cb 100644 --- a/tests/dsl/DSLOrdering.cpp +++ b/tests/dsl/DSLOrdering.cpp @@ -74,7 +74,7 @@ TEST_CASE("Testing poorly ordered on arguments", "[api][dsl][order][with]") { plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Emitting 1", "Emitting 2", "Emitting 3", diff --git a/tests/dsl/DSLProxy.cpp b/tests/dsl/DSLProxy.cpp index d665a317e..f2f3e148c 100644 --- a/tests/dsl/DSLProxy.cpp +++ b/tests/dsl/DSLProxy.cpp @@ -75,7 +75,7 @@ TEST_CASE("Testing that the DSL proxy works as expected for binding unmodifyable plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Emitting CustomMessage2", "Emitting CustomMessage1", "CustomMessage1 Triggered with 123456", diff --git a/tests/dsl/FlagMessage.cpp b/tests/dsl/FlagMessage.cpp index 9ad588436..d9930b7b1 100644 --- a/tests/dsl/FlagMessage.cpp +++ b/tests/dsl/FlagMessage.cpp @@ -74,7 +74,7 @@ TEST_CASE("Testing emitting types that are flag types (Have no contents)", "[api plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Emitting Step<1>", "Step<1> triggered", "Emitting MessageA", diff --git a/tests/dsl/Last.cpp b/tests/dsl/Last.cpp index fae89bfd8..bd951d9d1 100644 --- a/tests/dsl/Last.cpp +++ b/tests/dsl/Last.cpp @@ -63,7 +63,7 @@ TEST_CASE("Testing the last n feature", "[api][last]") { plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "0 ", "0 1 ", "0 1 2 ", diff --git a/tests/dsl/MainThread.cpp b/tests/dsl/MainThread.cpp index 9cc002efc..adb87f803 100644 --- a/tests/dsl/MainThread.cpp +++ b/tests/dsl/MainThread.cpp @@ -69,7 +69,7 @@ TEST_CASE("Testing that the MainThread keyword runs tasks on the main thread", " plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Emitting MessageA", "MessageA triggered on non-main thread", "Emitting MessageB", diff --git a/tests/dsl/MissingArguments.cpp b/tests/dsl/MissingArguments.cpp index cecb20a02..230085156 100644 --- a/tests/dsl/MissingArguments.cpp +++ b/tests/dsl/MissingArguments.cpp @@ -65,7 +65,7 @@ TEST_CASE("Testing that when arguments missing from the call it can still run", plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Emitting Message<4>", "Emitting Message<3>", "Emitting Message<2>", diff --git a/tests/dsl/Once.cpp b/tests/dsl/Once.cpp index 82f79505c..772d8019d 100644 --- a/tests/dsl/Once.cpp +++ b/tests/dsl/Once.cpp @@ -68,7 +68,7 @@ TEST_CASE("Reactions with the Once DSL keyword only execute once", "[api][once]" plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Startup Trigger Executed", "Once Trigger executed 0", "Normal Trigger Executed 0", "Emitting 1", "Normal Trigger Executed 1", "Emitting 2", diff --git a/tests/dsl/Optional.cpp b/tests/dsl/Optional.cpp index 5577d9d3b..4d86e7e45 100644 --- a/tests/dsl/Optional.cpp +++ b/tests/dsl/Optional.cpp @@ -72,7 +72,7 @@ TEST_CASE("Testing that optional is able to let data through even if it's invali plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Emitting A", "Executed reaction with A and optional B with B-", "Emitting B", diff --git a/tests/dsl/Priority.cpp b/tests/dsl/Priority.cpp index c68a71afb..d1a075dac 100644 --- a/tests/dsl/Priority.cpp +++ b/tests/dsl/Priority.cpp @@ -91,7 +91,7 @@ TEST_CASE("Tests that priority orders the tasks appropriately", "[api][priority] plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Realtime Message<1>", "Realtime Message<2>", "Realtime Message<3>", diff --git a/tests/dsl/RawFunction.cpp b/tests/dsl/RawFunction.cpp index 21f68d471..6f12d5cd8 100644 --- a/tests/dsl/RawFunction.cpp +++ b/tests/dsl/RawFunction.cpp @@ -108,7 +108,7 @@ TEST_CASE("Test reaction can take a raw function instead of just a lambda", "[ap plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Raw function no args", "Raw function left arg: M2", "Raw function right arg: D1", diff --git a/tests/dsl/Shutdown.cpp b/tests/dsl/Shutdown.cpp index 16ac43011..11ef295b5 100644 --- a/tests/dsl/Shutdown.cpp +++ b/tests/dsl/Shutdown.cpp @@ -56,7 +56,7 @@ TEST_CASE("A test that a shutdown message is emitted when the system shuts down" plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Starting test", "Requesting shutdown", "Shutdown task executed", diff --git a/tests/dsl/Startup.cpp b/tests/dsl/Startup.cpp index 94fcac898..8915c427f 100644 --- a/tests/dsl/Startup.cpp +++ b/tests/dsl/Startup.cpp @@ -55,7 +55,7 @@ TEST_CASE("Testing the startup event is emitted at the start of the program", "[ plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Startup triggered", "Emitting SimpleMessage", "SimpleMessage triggered with 10", diff --git a/tests/dsl/Sync.cpp b/tests/dsl/Sync.cpp index 5c1493b54..29530e2cb 100644 --- a/tests/dsl/Sync.cpp +++ b/tests/dsl/Sync.cpp @@ -101,7 +101,7 @@ TEST_CASE("Testing that the Sync word works correctly", "[api][sync]") { plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Sync A From Startup", "Sync A emitting", "Sync A From Startup finished", diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index ad3998f91..4eff87313 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -140,7 +140,7 @@ TEST_CASE("Testing listening for TCP connections and receiving data messages", " plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Known Port sending", "Known Port connection accepted", "Known Port received: Hello TCP World!", diff --git a/tests/dsl/Transient.cpp b/tests/dsl/Transient.cpp index 86555dfe2..00066a601 100644 --- a/tests/dsl/Transient.cpp +++ b/tests/dsl/Transient.cpp @@ -133,7 +133,7 @@ TEST_CASE("Testing whether getters that return transient data can cache between plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Emitting Message 1", "Emitting Transient 1", "S1 : T1", diff --git a/tests/dsl/Trigger.cpp b/tests/dsl/Trigger.cpp index e22c0fe1f..ca8eae0fc 100644 --- a/tests/dsl/Trigger.cpp +++ b/tests/dsl/Trigger.cpp @@ -58,7 +58,7 @@ TEST_CASE("Test that Trigger statements get the correct data", "[api][trigger]") plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Trigger 0", "Trigger 1", "Trigger 2", diff --git a/tests/dsl/Watchdog.cpp b/tests/dsl/Watchdog.cpp index b64296286..fec10845d 100644 --- a/tests/dsl/Watchdog.cpp +++ b/tests/dsl/Watchdog.cpp @@ -99,7 +99,7 @@ TEST_CASE("Testing the Watchdog Smart Type", "[api][watchdog]") { plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Watchdog 4 triggered @ 1", "Watchdog 4 triggered @ 2", "Watchdog 4 triggered @ 3", diff --git a/tests/dsl/With.cpp b/tests/dsl/With.cpp index 6fcc94b1b..aaee9c166 100644 --- a/tests/dsl/With.cpp +++ b/tests/dsl/With.cpp @@ -87,7 +87,7 @@ TEST_CASE("Testing the with dsl keyword", "[api][with]") { plant.install(); plant.start(); - std::vector expected = { + const std::vector expected = { "Emitting Data 1", "Emitting Data 2", "Emitting Message 1", diff --git a/tests/test_util/diff_string.cpp b/tests/test_util/diff_string.cpp index 97fc9d235..0eb8a8a0d 100644 --- a/tests/test_util/diff_string.cpp +++ b/tests/test_util/diff_string.cpp @@ -27,11 +27,11 @@ namespace test_util { std::string diff_string(const std::vector& expected, const std::vector& actual) { // Find the longest string in each side or "Expected" and "Actual" if those are the longest - auto len = [](const std::string& a, const std::string& b) { return a.size() < b.size(); }; - auto max_a_it = std::max_element(expected.begin(), expected.end(), len); - auto max_b_it = std::max_element(actual.begin(), actual.end(), len); - int max_a = std::max(int(std::strlen("Expected")), max_a_it != expected.end() ? int(max_a_it->size()) : 0); - int max_b = std::max(int(std::strlen("Actual")), max_b_it != actual.end() ? int(max_b_it->size()) : 0); + auto len = [](const std::string& a, const std::string& b) { return a.size() < b.size(); }; + auto max_a_it = std::max_element(expected.begin(), expected.end(), len); + auto max_b_it = std::max_element(actual.begin(), actual.end(), len); + const int max_a = std::max(int(std::strlen("Expected")), max_a_it != expected.end() ? int(max_a_it->size()) : 0); + const int max_b = std::max(int(std::strlen("Actual")), max_b_it != actual.end() ? int(max_b_it->size()) : 0); // Start with a header std::string output = std::string("Expected") + std::string(max_a - 8, ' ') + " | " + std::string("Actual") diff --git a/tests/test_util/lcs.hpp b/tests/test_util/lcs.hpp index c9ade2129..5eac13c47 100644 --- a/tests/test_util/lcs.hpp +++ b/tests/test_util/lcs.hpp @@ -64,18 +64,18 @@ std::pair, std::vector> lcs(const std::vector& a, con for (int y = 0; y < int(b.size()); ++y) { for (int x = 0; x < int(a.size()); ++x) { // Calculate the weights - int weight_from_left = x == 0 ? (y + 2) * insert_weight : curr_weights[x - 1] + insert_weight; - int weight_from_top = last_weights[x] + insert_weight; - int weight_from_diagonal = + const int weight_from_left = x == 0 ? (y + 2) * insert_weight : curr_weights[x - 1] + insert_weight; + const int weight_from_top = last_weights[x] + insert_weight; + const int weight_from_diagonal = a[x] == b[y] ? (x == 0 ? (y + 1) * insert_weight : last_weights[x - 1]) : 0x7FFFFFFF; // Find the smallest weight - int min_weight = std::min(std::min(weight_from_left, weight_from_top), weight_from_diagonal); - curr_weights[x] = min_weight; + const int min_weight = std::min(std::min(weight_from_left, weight_from_top), weight_from_diagonal); + curr_weights[x] = min_weight; - int direction = (min_weight == weight_from_diagonal ? 0x01 : 0x0) // - | (min_weight == weight_from_left ? 0x02 : 0x0) // - | (min_weight == weight_from_top ? 0x04 : 0x0); // + const int direction = (min_weight == weight_from_diagonal ? 0x01 : 0x0) // + | (min_weight == weight_from_left ? 0x02 : 0x0) // + | (min_weight == weight_from_top ? 0x04 : 0x0); // directions[x][y] = direction; } From 59fb065578e73c9ac04da4ef36aea5fa2f555697 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 7 Aug 2023 16:39:03 +1000 Subject: [PATCH 053/176] clang-tidy --- tests/dsl/Every.cpp | 2 +- tests/dsl/IO.cpp | 4 ++-- tests/dsl/TCP.cpp | 2 +- tests/dsl/Watchdog.cpp | 3 ++- tests/test_util/lcs.hpp | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/dsl/Every.cpp b/tests/dsl/Every.cpp index 9c4049efe..0856d99de 100644 --- a/tests/dsl/Every.cpp +++ b/tests/dsl/Every.cpp @@ -50,7 +50,7 @@ void test_results(const std::vector& times) { // Build up our difference vector std::vector diff; for (size_t i = 0; i < times.size() - 1; ++i) { - double delta = std::chrono::duration_cast>(times[i + 1] - times[i]).count(); + const double delta = std::chrono::duration_cast>(times[i + 1] - times[i]).count(); // Calculate the difference between the expected and actual time diff.push_back(delta - 1e-3); diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index 3fb82d9ab..7fe9dbd4f 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -87,7 +87,7 @@ TEST_CASE("Testing the IO extension", "[api][io]") { plant.install(); plant.start(); - std::vector read_expected = { + const std::vector read_expected = { "Read 1 bytes (H) from pipe", "Read 1 bytes (e) from pipe", "Read 1 bytes (l) from pipe", @@ -98,7 +98,7 @@ TEST_CASE("Testing the IO extension", "[api][io]") { // Make an info print the diff in an easy to read way if we fail INFO("Read Events\n" << test_util::diff_string(read_expected, read_events)); - std::vector write_expected{ + const std::vector write_expected{ "Wrote 1 bytes (H) to pipe", "Wrote 1 bytes (e) to pipe", "Wrote 1 bytes (l) to pipe", diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 4eff87313..c71becd18 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -47,7 +47,7 @@ class TestReactor : public test_util::TestBase { // Read into the buffer std::array buff{}; - ssize_t len = ::recv(event.fd, buff.data(), socklen_t(TEST_STRING.size()), 0); + const ssize_t len = ::recv(event.fd, buff.data(), socklen_t(TEST_STRING.size()), 0); if (len == 0) { events.push_back(name + " closed"); emit(std::make_unique()); diff --git a/tests/dsl/Watchdog.cpp b/tests/dsl/Watchdog.cpp index fec10845d..1d3981a34 100644 --- a/tests/dsl/Watchdog.cpp +++ b/tests/dsl/Watchdog.cpp @@ -77,7 +77,8 @@ class TestReactor : public test_util::TestBase { } std::string floored_time() const { - double diff = std::chrono::duration_cast>(NUClear::clock::now() - start).count(); + using namespace std::chrono; // NOLINT(google-build-using-namespace) fine in function scope + const double diff = duration_cast>(NUClear::clock::now() - start).count(); // Round to 100ths of a second return std::to_string(int(std::floor(diff * 100))); } diff --git a/tests/test_util/lcs.hpp b/tests/test_util/lcs.hpp index 5eac13c47..d42bda4d3 100644 --- a/tests/test_util/lcs.hpp +++ b/tests/test_util/lcs.hpp @@ -90,7 +90,7 @@ std::pair, std::vector> lcs(const std::vector& a, con int i_a = x; int i_b = y; while (x >= 0 && y >= 0) { - int direction = directions[x][y]; + const int& direction = directions[x][y]; if (direction & 0x01) { match_a[i_a--] = true; match_b[i_b--] = true; From 44b35ca72e4b6f2ea832e1cfe99404324dc0b11b Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 11:25:52 +1000 Subject: [PATCH 054/176] Refactor and update the IO/TCP to make it's behaviour more consistent and reliable (windows to come) --- src/dsl/word/IO.hpp | 28 ++- src/dsl/word/TCP.hpp | 14 +- src/dsl/word/UDP.hpp | 45 +--- src/extension/IOController_Posix.hpp | 360 ++++++++++++++++----------- tests/dsl/IO.cpp | 23 +- tests/dsl/TCP.cpp | 33 +-- tests/test_util/TestBase.hpp | 6 +- 7 files changed, 286 insertions(+), 223 deletions(-) diff --git a/src/dsl/word/IO.hpp b/src/dsl/word/IO.hpp index 3b5f83ce9..d96d07b9b 100644 --- a/src/dsl/word/IO.hpp +++ b/src/dsl/word/IO.hpp @@ -32,11 +32,18 @@ namespace dsl { namespace word { struct IOConfiguration { + IOConfiguration(fd_t fd, int events, const std::shared_ptr& reaction) + : fd(fd), events(events), reaction(reaction) {} fd_t fd; int events; std::shared_ptr reaction; }; + struct IOFinished { + IOFinished(const uint64_t& id) : id(id) {} + uint64_t id; + }; + /** * @brief * This is used to trigger reactions based on standard I/O operations using file descriptors. @@ -68,15 +75,25 @@ namespace dsl { * @par Implements * Bind */ - struct IO : public Single { + struct IO { // On windows we use different wait events #ifdef _WIN32 // NOLINTNEXTLINE(google-runtime-int) - enum EventType : short{READ = FD_READ | FD_OOB | FD_ACCEPT, WRITE = FD_WRITE, CLOSE = FD_CLOSE, ERROR = 0}; + enum EventType : short { + READ = FD_READ | FD_OOB | FD_ACCEPT, + WRITE = FD_WRITE, + CLOSE = FD_CLOSE, + ERROR = 0, + }; #else // NOLINTNEXTLINE(google-runtime-int) - enum EventType : short { READ = POLLIN, WRITE = POLLOUT, CLOSE = POLLHUP, ERROR = POLLNVAL | POLLERR }; + enum EventType : short { + READ = POLLIN, + WRITE = POLLOUT, + CLOSE = POLLHUP, + ERROR = POLLNVAL | POLLERR, + }; #endif struct Event { @@ -114,6 +131,11 @@ namespace dsl { // Otherwise return an invalid event return Event{INVALID_SOCKET, 0}; } + + template + static inline void postcondition(threading::ReactionTask& task) { + task.parent.reactor.emit(std::make_unique(task.parent.id)); + } }; } // namespace word diff --git a/src/dsl/word/TCP.hpp b/src/dsl/word/TCP.hpp index 5d69f8ce4..bc39229e7 100644 --- a/src/dsl/word/TCP.hpp +++ b/src/dsl/word/TCP.hpp @@ -58,7 +58,7 @@ namespace dsl { * @par Implements * Bind */ - struct TCP { + struct TCP : public IO { struct Connection { @@ -124,10 +124,8 @@ namespace dsl { port = ntohs(address.sin_port); // Generate a reaction for the IO system that closes on death - const fd_t cfd = fd; - reaction->unbinders.push_back([](const threading::Reaction& r) { - r.reactor.emit(std::make_unique>(r.id)); - }); + const fd_t cfd = fd.release(); + reaction->unbinders.push_back([cfd](const threading::Reaction&) { ::shutdown(cfd, SHUT_RDWR); #ifdef _WIN32 @@ -137,10 +135,8 @@ namespace dsl { #endif }); - auto io_config = std::make_unique(IOConfiguration{fd.release(), IO::READ, reaction}); - - // Send our configuration out - reaction->reactor.emit(io_config); + // Bind using the IO system + IO::bind(reaction, cfd, IO::READ | IO::CLOSE); // Return our handles return std::make_tuple(port, cfd); diff --git a/src/dsl/word/UDP.hpp b/src/dsl/word/UDP.hpp index d3b7720ab..76065dca4 100644 --- a/src/dsl/word/UDP.hpp +++ b/src/dsl/word/UDP.hpp @@ -58,7 +58,7 @@ namespace dsl { * @par Implements * Bind */ - struct UDP { + struct UDP : public IO { struct Packet { Packet() = default; @@ -148,16 +148,9 @@ namespace dsl { port = ntohs(address.sin_port); // Generate a reaction for the IO system that closes on death - const fd_t cfd = fd; - reaction->unbinders.push_back([](const threading::Reaction& r) { - r.reactor.emit(std::make_unique>(r.id)); - }); - reaction->unbinders.push_back([cfd](const threading::Reaction&) { close(cfd); }); - - auto io_config = std::make_unique(IOConfiguration{fd.release(), IO::READ, reaction}); - - // Send our configuration out - reaction->reactor.emit(io_config); + const fd_t cfd = fd.release(); + reaction->unbinders.push_back([cfd](const threading::Reaction&) { ::close(cfd); }); + IO::bind(reaction, cfd, IO::READ | IO::CLOSE); // Return our handles and our bound port return std::make_tuple(port, cfd); @@ -248,7 +241,7 @@ namespace dsl { return p; } - struct Broadcast { + struct Broadcast : public IO { template static inline std::tuple bind(const std::shared_ptr& reaction, @@ -306,17 +299,9 @@ namespace dsl { port = ntohs(address.sin_port); // Generate a reaction for the IO system that closes on death - const fd_t cfd = fd; - reaction->unbinders.push_back([](const threading::Reaction& r) { - r.reactor.emit(std::make_unique>(r.id)); - }); - reaction->unbinders.push_back([cfd](const threading::Reaction&) { close(cfd); }); - - auto io_config = - std::make_unique(IOConfiguration{fd.release(), IO::READ, reaction}); - - // Send our configuration out - reaction->reactor.emit(io_config); + const fd_t cfd = fd.release(); + reaction->unbinders.push_back([cfd](const threading::Reaction&) { ::close(cfd); }); + IO::bind(reaction, cfd, IO::READ | IO::CLOSE); // Return our handles and our bound port return std::make_tuple(port, cfd); @@ -328,7 +313,7 @@ namespace dsl { } }; - struct Multicast { + struct Multicast : public IO { template static inline std::tuple bind(const std::shared_ptr& reaction, @@ -412,17 +397,9 @@ namespace dsl { } // Generate a reaction for the IO system that closes on death - const fd_t cfd = fd; - reaction->unbinders.push_back([](const threading::Reaction& r) { - r.reactor.emit(std::make_unique>(r.id)); - }); + const fd_t cfd = fd.release(); reaction->unbinders.push_back([cfd](const threading::Reaction&) { close(cfd); }); - - auto io_config = - std::make_unique(IOConfiguration{fd.release(), IO::READ, reaction}); - - // Send our configuration out for each file descriptor (same reaction) - reaction->reactor.emit(io_config); + IO::bind(reaction, cfd, IO::READ | IO::CLOSE); // Return our handles return std::make_tuple(port, cfd); diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index b6c99edf1..f476aea7b 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -35,99 +35,254 @@ namespace extension { class IOController : public Reactor { private: + /** + * @brief A task that is waiting for an IO event + */ struct Task { Task() = default; // NOLINTNEXTLINE(google-runtime-int) Task(const fd_t& fd, short events, std::shared_ptr reaction) : fd(fd), events(events), reaction(std::move(reaction)) {} + /// @brief The file descriptor we are waiting on fd_t fd{-1}; + /// @brief The events that are waiting to be fired + short waiting_events{0}; // NOLINT(google-runtime-int) + /// @brief The events that the task is interested in short events{0}; // NOLINT(google-runtime-int) + /// @brief The reaction that is waiting for this event std::shared_ptr reaction{nullptr}; + /** + * @brief Sorts the tasks by their file descriptor + * + * The tasks are sorted by file descriptor so that when we rebuild the list of file descriptors to poll we + * can assume that if the same file descriptor shows up multiple times it will be next to each other. This + * allows the events that are being watched to be or'ed together. + * + * @param other the other task to compare to + * + * @return true if this task is less than the other + * @return false if this task is greater than or equal to the other + */ bool operator<(const Task& other) const { return fd == other.fd ? events < other.events : fd < other.fd; } }; + /** + * @brief Rebuilds the list of file descriptors to poll + * + * This function is called when the list of file descriptors to poll changes. It will rebuild the list of file + * descriptors used by poll + */ + void rebuild_list() { + // Get the lock so we don't concurrently modify the list + const std::lock_guard lock(tasks_mutex); + + // Clear our fds to be rebuilt + fds.resize(0); + + // Insert our notify fd + fds.push_back(pollfd{notify_recv, POLLIN, 0}); + + for (const auto& r : tasks) { + // If we are the same fd, then add our interest set + if (r.fd == fds.back().fd) { + fds.back().events = short(fds.back().events | r.events); // NOLINT(google-runtime-int) + } + // Otherwise add a new one + else { + fds.push_back(pollfd{r.fd, r.events, 0}); + } + } + + // We just cleaned the list! + dirty = false; + } + + void collect_events() { + + // Get the lock so we don't concurrently modify the list + const std::lock_guard lock(tasks_mutex); + + for (auto& fd : fds) { + + // Something happened + if (fd.revents != 0) { + + // It's our notification handle + if (fd.fd == notify_recv) { + // Read our value to clear it's read status + char val = 0; + if (::read(fd.fd, &val, sizeof(char)) < 0) { + throw std::system_error(network_errno, + std::system_category(), + "There was an error reading our notification pipe?"); + }; + } + // It's a regular handle + else { + // Check how many bytes are available to read, if it's 0 and we have a read event the + // descriptor is sending EOF and we should fire a CLOSE event too and stop watching + if ((fd.revents & IO::READ) != 0) { + int bytes_available; + bool valid = ::ioctl(fd.fd, FIONREAD, &bytes_available) == 0; + if (valid && bytes_available == 0) { + fd.revents |= IO::CLOSE; + } + } + + // Find our relevant tasks + auto range = std::equal_range(std::begin(tasks), + std::end(tasks), + Task{fd.fd, 0, nullptr}, + [](const Task& a, const Task& b) { return a.fd < b.fd; }); + + // There are no tasks for this! + if (range.first == std::end(tasks)) { + // If this happens then our list is definitely dirty... + dirty = true; + } + else { + // Loop through our values + for (auto it = range.first; it != range.second; ++it) { + + // Load in the relevant events that happened into the waiting events + it->waiting_events = it->waiting_events | (it->events & fd.revents); + } + } + } + + // Clear the events from poll to avoid double firing + fd.revents = 0; + } + } + } + + void fire_events() { + const std::lock_guard lock(tasks_mutex); + + // Go through every reaction and if it has events and isn't already running then run it + for (auto it = tasks.begin(); it != tasks.end();) { + if (it->reaction->active_tasks == 0 && it->waiting_events != 0) { + + // Make our event to pass through and store it in the local cache + IO::Event e{}; + e.fd = it->fd; + e.events = it->waiting_events; + + // Submit the task (which should run the get) + try { + IO::ThreadEventStore::value = &e; + auto task = it->reaction->get_task(); + IO::ThreadEventStore::value = nullptr; + // Only actually submit this task if it is the only active task for this reaction + if (task) { + powerplant.submit(std::move(task)); + } + } + catch (...) { + } + + // Reset our value + + if ((it->waiting_events & IO::CLOSE) != 0) { + dirty = true; + it = tasks.erase(it); + } + else { + ++it; + } + } + else { + ++it; + } + } + } + + void bump() { + // Check if there was an error + uint8_t val = 1; + if (::write(notify_send, &val, sizeof(val)) < 0) { + throw std::system_error(network_errno, + std::system_category(), + "There was an error while writing to the notification pipe"); + } + } + public: - explicit IOController(std::unique_ptr environment) - : Reactor(std::move(environment)), notify_recv(), notify_send() { + explicit IOController(std::unique_ptr environment) : Reactor(std::move(environment)) { std::array vals = {-1, -1}; - - const int i = ::pipe(vals.data()); + const int i = ::pipe(vals.data()); if (i < 0) { throw std::system_error(network_errno, std::system_category(), "We were unable to make the notification pipe for IO"); } - notify_recv = vals[0]; notify_send = vals[1]; - // Add our notification pipe to our list of fds - fds.push_back(pollfd{notify_recv, POLLIN, 0}); + // Start by rebuliding the list + rebuild_list(); on>().then( "Configure IO Reaction", [this](const dsl::word::IOConfiguration& config) { // Lock our mutex to avoid concurrent modification - const std::lock_guard lock(reaction_mutex); + const std::lock_guard lock(tasks_mutex); // NOLINTNEXTLINE(google-runtime-int) - reactions.emplace_back(config.fd, static_cast(config.events), config.reaction); + tasks.emplace_back(config.fd, static_cast(config.events), config.reaction); // Resort our list - std::sort(std::begin(reactions), std::end(reactions)); + std::sort(std::begin(tasks), std::end(tasks)); // Let the poll command know that stuff happened dirty = true; + bump(); + }); - // Check if there was an error - if (::write(notify_send, &dirty, 1) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "There was an error while writing to the notification pipe"); - } + on>().then("IO Finished", [this](const dsl::word::IOFinished& event) { + // Get the lock so we don't concurrently modify the list + const std::lock_guard lock(tasks_mutex); + + // Find the reaction that finished processing + auto task = std::find_if(std::begin(tasks), std::end(tasks), [&event](const Task& t) { + return t.reaction->id == event.id; }); + // If we found it then clear the waiting events + if (task != std::end(tasks)) { + task->waiting_events = 0; + } + }); + on>>().then( "Unbind IO Reaction", [this](const dsl::operation::Unbind& unbind) { // Lock our mutex to avoid concurrent modification - const std::lock_guard lock(reaction_mutex); + const std::lock_guard lock(tasks_mutex); // Find our reaction - auto reaction = std::find_if(std::begin(reactions), std::end(reactions), [&unbind](const Task& t) { + auto reaction = std::find_if(std::begin(tasks), std::end(tasks), [&unbind](const Task& t) { return t.reaction->id == unbind.id; }); - if (reaction != std::end(reactions)) { - reactions.erase(reaction); + if (reaction != std::end(tasks)) { + tasks.erase(reaction); } // Let the poll command know that stuff happened dirty = true; - if (::write(notify_send, &dirty, 1) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "There was an error while writing to the notification pipe"); - } + bump(); }); on().then("Shutdown IO Controller", [this] { // Set shutdown to true so it won't try to poll again shutdown.store(true); - // A byte to send down the pipe - char val = 0; - - // Send a single byte down the pipe - if (::write(notify_send, &val, 1) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "There was an error while writing to the notification pipe"); - } + bump(); }); on().then("IO Controller", [this] { @@ -135,143 +290,44 @@ namespace extension { // shutdown keeps us out here if (!shutdown.load()) { + // Rebuild the list if something changed + if (dirty) { + rebuild_list(); + } - // TODO(trent): check for dirty here - - - // Poll our file descriptors for events - const int result = ::poll(fds.data(), static_cast(fds.size()), -1); - - // Check if we had an error on our Poll request - if (result < 0) { + // Wait for an event to happen on one of our file descriptors + if (::poll(fds.data(), static_cast(fds.size()), -1) < 0) { throw std::system_error(network_errno, std::system_category(), "There was an IO error while attempting to poll the file descriptors"); } - for (auto& fd : fds) { - - // Something happened - if (fd.revents != 0) { - - // It's our notification handle - if (fd.fd == notify_recv) { - // Read our value to clear it's read status - char val = 0; - if (::read(fd.fd, &val, sizeof(char)) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "There was an error reading our notification pipe?"); - }; - } - // It's a regular handle - else { - - // Find our relevant reactions - auto range = std::equal_range(std::begin(reactions), - std::end(reactions), - Task{fd.fd, 0, nullptr}, - [](const Task& a, const Task& b) { return a.fd < b.fd; }); - - - // There are no reactions for this! - if (range.first == std::end(reactions)) { - // If this happens then our list is definitely dirty... - dirty = true; - } - else { - - - // TODO(trent): we also want to swap this element to the back of the list and - // remove it so that it does not fire again - - - // Loop through our values - for (auto it = range.first; it != range.second; ++it) { - - // We should emit if the reaction is interested - if ((it->events & fd.revents) != 0) { - - // Make our event to pass through - IO::Event e{}; - e.fd = fd.fd; - - // Evaluate and store our set in thread store - e.events = fd.revents; - - // Store the event in our thread local cache - IO::ThreadEventStore::value = &e; - - // Submit the task (which should run the get) - try { - auto task = it->reaction->get_task(); - if (task) { - powerplant.submit(std::move(task)); - } - } - catch (...) { - } - - // Reset our value - IO::ThreadEventStore::value = nullptr; - - // TODO(trent): If we had a close, or error stop listening? - } - } - - if ((fd.revents & IO::CLOSE) != 0) { - // Remove all the reactions that wanted this fd and flag the list as dirty - reactions.erase(range.first, range.second); - dirty = true; - } - } - } - - // Reset our events - fd.revents = 0; - } - } - - // If our list is dirty - if (dirty) { - // Get the lock so we don't concurrently modify the list - const std::lock_guard lock(reaction_mutex); - - // Clear our fds to be rebuilt - fds.resize(0); + // Collect the events that happened into the tasks list + collect_events(); - // Insert our notify fd - fds.push_back(pollfd{notify_recv, POLLIN, 0}); - for (const auto& r : reactions) { - - // If we are the same fd, then add our interest set - if (r.fd == fds.back().fd) { - // NOLINTNEXTLINE(google-runtime-int) - fds.back().events = static_cast(fds.back().events | r.events); - } - // Otherwise add a new one - else { - fds.push_back(pollfd{r.fd, r.events, 0}); - } - } - - // We just cleaned the list! - dirty = false; - } + // Fire the events that happened if we can + fire_events(); } }); } private: + /// @brief The receive file descriptor for our notification pipe fd_t notify_recv{-1}; + /// @brief The send file descriptor for our notification pipe fd_t notify_send{-1}; + /// @brief Whether or not we are shutting down std::atomic shutdown{false}; + /// @brief The mutex that protects the tasks list + std::mutex tasks_mutex; + /// @brief Whether or not the list of file descriptors is dirty compared to tasks bool dirty = true; - std::mutex reaction_mutex; + /// @brief The list of file descriptors to poll std::vector fds{}; - std::vector reactions{}; + /// @brief The list of tasks that are waiting for IO events + std::vector tasks{}; }; } // namespace extension diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index 7fe9dbd4f..4f2b3bf52 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -35,9 +35,9 @@ std::vector read_events; // NOLINT(cppcoreguidelines-avoid-non-con /// @brief Events that occur during the test writing std::vector write_events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { std::array fds{-1, -1}; @@ -47,14 +47,18 @@ class TestReactor : public NUClear::Reactor { in = fds[0]; out = fds[1]; - on(in.get(), IO::READ).then([this](const IO::Event& e) { - // Read from our fd - char c{0}; - auto bytes = ::read(e.fd, &c, 1); + on(in.get(), IO::READ | IO::CLOSE).then([this](const IO::Event& e) { + if ((e.events & IO::READ) != 0) { + // Read from our fd + char c{0}; + auto bytes = ::read(e.fd, &c, 1); - read_events.push_back("Read " + std::to_string(bytes) + " bytes (" + c + ") from pipe"); + read_events.push_back("Read " + std::to_string(bytes) + " bytes (" + c + ") from pipe"); + } - if (c == 'o') { + // FD was closed + if ((e.events & IO::CLOSE) != 0) { + read_events.push_back("Closed pipe"); powerplant.shutdown(); } }); @@ -67,7 +71,7 @@ class TestReactor : public NUClear::Reactor { write_events.push_back("Wrote " + std::to_string(sent) + " bytes (" + c + ") to pipe"); if (char_no == 5) { - writer.unbind(); + ::close(e.fd); } }); } @@ -93,6 +97,7 @@ TEST_CASE("Testing the IO extension", "[api][io]") { "Read 1 bytes (l) from pipe", "Read 1 bytes (l) from pipe", "Read 1 bytes (o) from pipe", + "Closed pipe", }; // Make an info print the diff in an easy to read way if we fail diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index c71becd18..e0c2576d5 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -37,9 +37,12 @@ struct TestConnection { in_port_t port; }; -struct FinishTest {}; +struct Finished { + Finished(std::string name) : name(std::move(name)) {} + std::string name; +}; -class TestReactor : public test_util::TestBase { +class TestReactor : public test_util::TestBase { public: void handle_data(const std::string& name, const IO::Event& event) { // We have data to read @@ -48,15 +51,16 @@ class TestReactor : public test_util::TestBase { // Read into the buffer std::array buff{}; const ssize_t len = ::recv(event.fd, buff.data(), socklen_t(TEST_STRING.size()), 0); - if (len == 0) { - events.push_back(name + " closed"); - emit(std::make_unique()); - } - else { + if (len != 0) { events.push_back(name + " received: " + std::string(buff.data(), len)); ::send(event.fd, buff.data(), socklen_t(len), 0); } } + + if ((event.events & IO::CLOSE) != 0) { + events.push_back(name + " closed"); + emit(std::make_unique(name)); + } } TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { @@ -109,26 +113,25 @@ class TestReactor : public test_util::TestBase { events.push_back(target.name + " echoed: " + std::string(buff.data(), recv)); }); - on, Sync>().then([this] { - ++count; - if (count == 2) { + on, Sync>().then([this, ephemeral_port](const Finished& test) { + if (test.name == "Known Port") { + emit(std::make_unique("Ephemeral Port", ephemeral_port)); + } + else if (test.name == "Ephemeral Port") { events.push_back("Finishing Test"); powerplant.shutdown(); } }); - on().then([this, ephemeral_port] { - // Emit a message just so it will be when everything is running + on().then([this] { + // Start the first test emit(std::make_unique("Known Port", PORT)); - emit(std::make_unique("Ephemeral Port", ephemeral_port)); }); } private: NUClear::util::FileDescriptor known_port_fd; NUClear::util::FileDescriptor ephemeral_port_fd; - // How many tests were run - int count = 0; }; } // namespace diff --git a/tests/test_util/TestBase.hpp b/tests/test_util/TestBase.hpp index bb2ec25f5..eef2eb6aa 100644 --- a/tests/test_util/TestBase.hpp +++ b/tests/test_util/TestBase.hpp @@ -19,6 +19,7 @@ #ifndef TEST_UTIL_TESTBASE_HPP #define TEST_UTIL_TESTBASE_HPP +#include #include #include #include @@ -56,7 +57,10 @@ class TestBase : public NUClear::Reactor { }); // Timeout if the test doesn't complete in time - on>().then([this] { powerplant.shutdown(); }); + on, MainThread>().then([this] { + powerplant.shutdown(); + FAIL("Test timed out"); + }); } }; From 9173fd0373cbb4c35f697e9fc5ca340dbd435d7d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 11:28:23 +1000 Subject: [PATCH 055/176] OSX closes differently --- tests/dsl/IO.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index 4f2b3bf52..085a89899 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -53,7 +53,9 @@ class TestReactor : public test_util::TestBase { char c{0}; auto bytes = ::read(e.fd, &c, 1); - read_events.push_back("Read " + std::to_string(bytes) + " bytes (" + c + ") from pipe"); + if (bytes > 0) { + read_events.push_back("Read " + std::to_string(bytes) + " bytes (" + c + ") from pipe"); + } } // FD was closed From f0a12f4cc581840116d1b0bd5da0be05f6e840e4 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 13:57:59 +1000 Subject: [PATCH 056/176] fix warnings clang-tidy etc --- src/dsl/word/IO.hpp | 4 ++-- src/dsl/word/TCP.hpp | 4 ---- src/extension/IOController_Posix.hpp | 11 +++++++++-- src/util/FileDescriptor.cpp | 4 ---- src/util/platform.cpp | 2 +- src/util/platform.hpp | 10 +++++----- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/dsl/word/IO.hpp b/src/dsl/word/IO.hpp index d96d07b9b..095580620 100644 --- a/src/dsl/word/IO.hpp +++ b/src/dsl/word/IO.hpp @@ -32,8 +32,8 @@ namespace dsl { namespace word { struct IOConfiguration { - IOConfiguration(fd_t fd, int events, const std::shared_ptr& reaction) - : fd(fd), events(events), reaction(reaction) {} + IOConfiguration(fd_t fd, int events, std::shared_ptr reaction) + : fd(fd), events(events), reaction(std::move(reaction)) {} fd_t fd; int events; std::shared_ptr reaction; diff --git a/src/dsl/word/TCP.hpp b/src/dsl/word/TCP.hpp index bc39229e7..3e9fd94e4 100644 --- a/src/dsl/word/TCP.hpp +++ b/src/dsl/word/TCP.hpp @@ -128,11 +128,7 @@ namespace dsl { reaction->unbinders.push_back([cfd](const threading::Reaction&) { ::shutdown(cfd, SHUT_RDWR); -#ifdef _WIN32 - ::closesocket(cfd); -#else ::close(cfd); -#endif }); // Bind using the IO system diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index f476aea7b..71bd7e8a8 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -126,8 +126,8 @@ namespace extension { // Check how many bytes are available to read, if it's 0 and we have a read event the // descriptor is sending EOF and we should fire a CLOSE event too and stop watching if ((fd.revents & IO::READ) != 0) { - int bytes_available; - bool valid = ::ioctl(fd.fd, FIONREAD, &bytes_available) == 0; + int bytes_available = 0; + bool valid = ::ioctl(fd.fd, FIONREAD, &bytes_available) == 0; if (valid && bytes_available == 0) { fd.revents |= IO::CLOSE; } @@ -201,6 +201,13 @@ namespace extension { } } + /** + * @brief Bumps the notification pipe to wake up the poll command + * + * If the poll command is waiting it will wait forever if something doesn't happen. + * When trying to update what to poll or shut down we need to wake it up so it can. + */ + // NOLINTNEXTLINE(readability-make-member-function-const) this changes states void bump() { // Check if there was an error uint8_t val = 1; diff --git a/src/util/FileDescriptor.cpp b/src/util/FileDescriptor.cpp index 9517efb88..bf8988c5f 100644 --- a/src/util/FileDescriptor.cpp +++ b/src/util/FileDescriptor.cpp @@ -21,11 +21,7 @@ namespace util { if (cleanup) { cleanup(fd); } -#ifdef _WIN32 - ::closesocket(fd); -#else ::close(fd); -#endif fd = INVALID_SOCKET; } } diff --git a/src/util/platform.cpp b/src/util/platform.cpp index 22672e8f3..4f97dba42 100644 --- a/src/util/platform.cpp +++ b/src/util/platform.cpp @@ -44,7 +44,7 @@ void initialize_WSARecvMsg() { throw std::runtime_error("We could not get WSARecvMsg from the OS"); } - closesocket(sock); + ::closesocket(sock); } namespace NUClear { diff --git a/src/util/platform.hpp b/src/util/platform.hpp index 0a98c9dfc..5cd3c4568 100644 --- a/src/util/platform.hpp +++ b/src/util/platform.hpp @@ -106,6 +106,11 @@ using ssize_t = SSIZE_T; using in_port_t = uint16_t; using in_addr_t = uint32_t; +// Make close call closesocket +inline int close(fd_t fd) { + return ::closesocket(fd); +} + namespace NUClear { // For us file descriptors will just be sockets @@ -113,11 +118,6 @@ using fd_t = SOCKET; using socklen_t = int; -// This is defined here rather than in the global namespace so it doesn't get in the way -inline int close(fd_t fd) { - return ::closesocket(fd); -} - inline int ioctl(SOCKET s, long cmd, u_long* argp) { return ioctlsocket(s, cmd, argp); } From 1e5c5e95749a66a014ed8c540e38615242905702 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 14:51:09 +1000 Subject: [PATCH 057/176] Test margin of 1ms --- tests/individual/CustomClock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/individual/CustomClock.cpp b/tests/individual/CustomClock.cpp index dcb6aac7c..c4bfcf766 100644 --- a/tests/individual/CustomClock.cpp +++ b/tests/individual/CustomClock.cpp @@ -85,5 +85,5 @@ TEST_CASE("Testing custom clock works correctly", "[api][custom_clock]") { REQUIRE((custom_total / steady_total) == Approx(0.5)); // The amount of time that passed should be (n - 1) * 2 * 10ms - REQUIRE(steady_total == Approx(2.0 * (times.size() - 1) * 1e-2)); + REQUIRE(steady_total == Approx(2.0 * (times.size() - 1) * 1e-2).margin(1e-3)); } From 2a5fbf3d117405bad01e25f074914da77cf463e6 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 15:01:27 +1000 Subject: [PATCH 058/176] clang-tidy --- src/extension/IOController_Posix.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index 71bd7e8a8..f143c70d1 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -149,7 +149,7 @@ namespace extension { for (auto it = range.first; it != range.second; ++it) { // Load in the relevant events that happened into the waiting events - it->waiting_events = it->waiting_events | (it->events & fd.revents); + it->waiting_events = short(it->waiting_events | (it->events & fd.revents)); } } } From 5f8daa9b2fe564dc6db24ee72a8e28dc3f28a565 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 15:13:13 +1000 Subject: [PATCH 059/176] windows platform changes --- src/util/platform.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/util/platform.hpp b/src/util/platform.hpp index 5cd3c4568..f06a6389e 100644 --- a/src/util/platform.hpp +++ b/src/util/platform.hpp @@ -107,10 +107,14 @@ using in_port_t = uint16_t; using in_addr_t = uint32_t; // Make close call closesocket -inline int close(fd_t fd) { +inline int close(SOCKET fd) { return ::closesocket(fd); } +inline int ioctl(SOCKET s, long cmd, u_long* argp) { + return ioctlsocket(s, cmd, argp); +} + namespace NUClear { // For us file descriptors will just be sockets @@ -118,9 +122,6 @@ using fd_t = SOCKET; using socklen_t = int; -inline int ioctl(SOCKET s, long cmd, u_long* argp) { - return ioctlsocket(s, cmd, argp); -} // Network errors come from WSAGetLastError() #define network_errno WSAGetLastError() From 414034f2b72f52cccc55efcab5105185e5ce8330 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 15:14:36 +1000 Subject: [PATCH 060/176] clang-tidy --- src/extension/IOController_Posix.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index f143c70d1..51969dac6 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -149,6 +149,7 @@ namespace extension { for (auto it = range.first; it != range.second; ++it) { // Load in the relevant events that happened into the waiting events + // NOLINTNEXTLINE(google-runtime-int) it->waiting_events = short(it->waiting_events | (it->events & fd.revents)); } } From c0e9081faf6c0dbe91906beeab418f7b7d8a345b Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 15:50:56 +1000 Subject: [PATCH 061/176] Fix the reactor name for windows --- src/PowerPlant.ipp | 8 ++++++-- src/Reactor.hpp | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/PowerPlant.ipp b/src/PowerPlant.ipp index d1b0dce57..283888f58 100644 --- a/src/PowerPlant.ipp +++ b/src/PowerPlant.ipp @@ -55,9 +55,13 @@ void PowerPlant::install() { // Make sure that the class that we received is a reactor static_assert(std::is_base_of::value, "You must install Reactors"); + // Get the demangled reactor name and strip `struct ` and `class ` from it + std::string reactor_name = util::demangle(typeid(T).name()); + reactor_name = std::regex_replace(reactor_name, std::regex(R"(struct\s+)"), ""); + reactor_name = std::regex_replace(reactor_name, std::regex(R"(class\s+)"), ""); + // The reactor constructor should handle subscribing to events - reactors.push_back( - std::make_unique(std::make_unique(*this, util::demangle(typeid(T).name()), level))); + reactors.push_back(std::make_unique(std::make_unique(*this, reactor_name, level))); } // Default emit with no types diff --git a/src/Reactor.hpp b/src/Reactor.hpp index 422a6f904..e92da8f61 100644 --- a/src/Reactor.hpp +++ b/src/Reactor.hpp @@ -159,7 +159,7 @@ class Reactor { std::vector reaction_handles{}; public: - /// @brief TODO + /// @brief The powerplant that this reactor is running in PowerPlant& powerplant; /// @brief The demangled string name of this reactor From 96e2000396127bbba4d49dc2d312ae28de2d7ac6 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 15:53:26 +1000 Subject: [PATCH 062/176] Fix log level --- src/Environment.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Environment.hpp b/src/Environment.hpp index 69afe8dba..f79578cfc 100644 --- a/src/Environment.hpp +++ b/src/Environment.hpp @@ -40,8 +40,8 @@ class PowerPlant; */ class Environment { public: - Environment(PowerPlant& powerplant, std::string&& reactor_name, LogLevel log_level) - : powerplant(powerplant), log_level(log_level), reactor_name(reactor_name) {} + Environment(PowerPlant& powerplant, std::string reactor_name, const LogLevel& log_level) + : powerplant(powerplant), log_level(log_level), reactor_name(std::move(reactor_name)) {} private: friend class PowerPlant; From 54a4e7ba50afb1ae93f893ad1b42f4985d722ab0 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 16:20:37 +1000 Subject: [PATCH 063/176] /shrug --- src/Environment.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Environment.hpp b/src/Environment.hpp index f79578cfc..c7845a84e 100644 --- a/src/Environment.hpp +++ b/src/Environment.hpp @@ -40,8 +40,8 @@ class PowerPlant; */ class Environment { public: - Environment(PowerPlant& powerplant, std::string reactor_name, const LogLevel& log_level) - : powerplant(powerplant), log_level(log_level), reactor_name(std::move(reactor_name)) {} + Environment(PowerPlant& powerplant, const LogLevel& log_level, std::string reactor_name) + : powerplant(powerplant), reactor_name(std::move(reactor_name)), log_level(log_level) {} private: friend class PowerPlant; @@ -49,10 +49,10 @@ class Environment { /// @brief The PowerPlant to use in this reactor PowerPlant& powerplant; - /// @brief The log level for this reactor - LogLevel log_level; /// @brief The name of the reactor std::string reactor_name; + /// @brief The log level for this reactor + LogLevel log_level; }; } // namespace NUClear From c6c57bcf178240ab6a916df12088c9ce842a3a36 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 16:27:20 +1000 Subject: [PATCH 064/176] Didn't mean to swap both --- src/Environment.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Environment.hpp b/src/Environment.hpp index c7845a84e..9381604f6 100644 --- a/src/Environment.hpp +++ b/src/Environment.hpp @@ -40,7 +40,7 @@ class PowerPlant; */ class Environment { public: - Environment(PowerPlant& powerplant, const LogLevel& log_level, std::string reactor_name) + Environment(PowerPlant& powerplant, std::string reactor_name, const LogLevel& log_level) : powerplant(powerplant), reactor_name(std::move(reactor_name)), log_level(log_level) {} private: From 7c217c842f9dcf66c4821cc533fc3989df48358a Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 17:06:22 +1000 Subject: [PATCH 065/176] How could this possibly have made windows fail to run reaction statistics --- src/PowerPlant.ipp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/PowerPlant.ipp b/src/PowerPlant.ipp index 283888f58..e8227969d 100644 --- a/src/PowerPlant.ipp +++ b/src/PowerPlant.ipp @@ -57,8 +57,8 @@ void PowerPlant::install() { // Get the demangled reactor name and strip `struct ` and `class ` from it std::string reactor_name = util::demangle(typeid(T).name()); - reactor_name = std::regex_replace(reactor_name, std::regex(R"(struct\s+)"), ""); - reactor_name = std::regex_replace(reactor_name, std::regex(R"(class\s+)"), ""); + // reactor_name = std::regex_replace(reactor_name, std::regex(R"(struct\s+)"), ""); + // reactor_name = std::regex_replace(reactor_name, std::regex(R"(class\s+)"), ""); // The reactor constructor should handle subscribing to events reactors.push_back(std::make_unique(std::make_unique(*this, reactor_name, level))); From df816d7200940dbe18914f4b3a22c7491e22f629 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 9 Aug 2023 20:15:20 +1000 Subject: [PATCH 066/176] How does this make reaction stats not run on windows? --- src/PowerPlant.ipp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/PowerPlant.ipp b/src/PowerPlant.ipp index e8227969d..283888f58 100644 --- a/src/PowerPlant.ipp +++ b/src/PowerPlant.ipp @@ -57,8 +57,8 @@ void PowerPlant::install() { // Get the demangled reactor name and strip `struct ` and `class ` from it std::string reactor_name = util::demangle(typeid(T).name()); - // reactor_name = std::regex_replace(reactor_name, std::regex(R"(struct\s+)"), ""); - // reactor_name = std::regex_replace(reactor_name, std::regex(R"(class\s+)"), ""); + reactor_name = std::regex_replace(reactor_name, std::regex(R"(struct\s+)"), ""); + reactor_name = std::regex_replace(reactor_name, std::regex(R"(class\s+)"), ""); // The reactor constructor should handle subscribing to events reactors.push_back(std::make_unique(std::make_unique(*this, reactor_name, level))); From 7b2697417f3e9680fd1ec9f99d7e6b36138bc3cf Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 10:45:48 +1000 Subject: [PATCH 067/176] Move the demangling normalisation into demangle --- src/PowerPlant.ipp | 2 -- src/Reactor.hpp | 3 --- src/util/demangle.cpp | 35 ++++++++++++++++++++++---------- tests/api/ReactionStatistics.cpp | 5 ++--- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/PowerPlant.ipp b/src/PowerPlant.ipp index 283888f58..b9ded1e4f 100644 --- a/src/PowerPlant.ipp +++ b/src/PowerPlant.ipp @@ -57,8 +57,6 @@ void PowerPlant::install() { // Get the demangled reactor name and strip `struct ` and `class ` from it std::string reactor_name = util::demangle(typeid(T).name()); - reactor_name = std::regex_replace(reactor_name, std::regex(R"(struct\s+)"), ""); - reactor_name = std::regex_replace(reactor_name, std::regex(R"(class\s+)"), ""); // The reactor constructor should handle subscribing to events reactors.push_back(std::make_unique(std::make_unique(*this, reactor_name, level))); diff --git a/src/Reactor.hpp b/src/Reactor.hpp index e92da8f61..287a15b28 100644 --- a/src/Reactor.hpp +++ b/src/Reactor.hpp @@ -303,9 +303,6 @@ class Reactor { // Regex replace NUClear::dsl::Parse with NUClear::Reactor::on so that it reads more like what is expected std::string dsl = util::demangle(typeid(DSL).name()); dsl = std::regex_replace(dsl, std::regex(R"(NUClear::dsl::Parse<)"), "NUClear::Reactor::on<"); - dsl = std::regex_replace(dsl, std::regex(R"(struct\s+)"), ""); - dsl = std::regex_replace(dsl, std::regex(R"(class\s+)"), ""); - dsl = std::regex_replace(dsl, std::regex(R"(\s+)"), ""); // Generate the identifier threading::ReactionIdentifiers identifiers{label, diff --git a/src/util/demangle.cpp b/src/util/demangle.cpp index 6f280e41c..14e4e7a1e 100644 --- a/src/util/demangle.cpp +++ b/src/util/demangle.cpp @@ -18,16 +18,20 @@ #include "demangle.hpp" +#include + // Windows symbol demangler #ifdef _WIN32 - // Turn off clang-format to avoid moving platform.h after Dbghelp.h - // (Dbghelp.h depends on types from Windows.h) - // clang-format off -# include "platform.hpp" -# include -# include -// clang-format on + // Dbghelp.h depends on types from Windows.h so it needs to be included first + // the define keeps clang-tidy from moving it + #ifdef _WIN32 + #include "platform.hpp" + #endif + + #include + + #include #pragma comment(lib, "Dbghelp.lib") @@ -61,10 +65,14 @@ namespace util { init_symbols(); } - char name[256]; + std::array name; if (int len = UnDecorateSymbolName(symbol, name, sizeof(name), 0)) { - return std::string(name, len); + std::string demangled(name, len); + demangled = std::regex_replace(demangled, std::regex(R"(struct\s+)"), ""); + demangled = std::regex_replace(demangled, std::regex(R"(class\s+)"), ""); + demangled = std::regex_replace(demangled, std::regex(R"(\s+)"), ""); + return std::demangled; } else { return symbol; @@ -94,11 +102,16 @@ namespace util { */ std::string demangle(const char* symbol) { - int status = -4; // some arbitrary value to eliminate the compiler warning + int status = -1; const std::unique_ptr res{abi::__cxa_demangle(symbol, nullptr, nullptr, &status), std::free}; + if (status == 0) { + std::string demangled = res.get(); + demangled = std::regex_replace(demangled, std::regex(R"(\s+)"), ""); + return demangled; + } - return status == 0 ? res.get() : symbol; + return symbol; } } // namespace util diff --git a/tests/api/ReactionStatistics.cpp b/tests/api/ReactionStatistics.cpp index 74e4e1c92..bddef652c 100644 --- a/tests/api/ReactionStatistics.cpp +++ b/tests/api/ReactionStatistics.cpp @@ -45,11 +45,10 @@ class TestReactor : public test_util::TestBase { on>().then("No Statistics", [] {}); - on>().then("Reaction Stats Handler", [](const ReactionStatistics& stats) { + on>().then("Reaction Stats Handler", [this](const ReactionStatistics& stats) { // Other reactions statistics run on this because of built in NUClear reactors (e.g. chrono controller etc) // We want to filter those out so only our own stats are shown - if (stats.identifiers.name.empty() - || stats.identifiers.reactor != NUClear::util::demangle(typeid(TestReactor).name())) { + if (stats.identifiers.name.empty() || stats.identifiers.reactor != reactor_name) { return; } events.push_back("Stats for " + stats.identifiers.name + " from " + stats.identifiers.reactor); From dca757d95c58a0592f53b8f4df1cb261d3b731f4 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 10:51:56 +1000 Subject: [PATCH 068/176] . --- src/util/demangle.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util/demangle.cpp b/src/util/demangle.cpp index 14e4e7a1e..69a18f6b8 100644 --- a/src/util/demangle.cpp +++ b/src/util/demangle.cpp @@ -31,6 +31,7 @@ #include + #include #include #pragma comment(lib, "Dbghelp.lib") @@ -67,7 +68,7 @@ namespace util { std::array name; - if (int len = UnDecorateSymbolName(symbol, name, sizeof(name), 0)) { + if (int len = UnDecorateSymbolName(symbol, name.data(), name.size(), 0)) { std::string demangled(name, len); demangled = std::regex_replace(demangled, std::regex(R"(struct\s+)"), ""); demangled = std::regex_replace(demangled, std::regex(R"(class\s+)"), ""); From fff3a4bf7c1e083e2f84a6ac7bbbd984b0b709c6 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 10:59:02 +1000 Subject: [PATCH 069/176] . --- src/util/demangle.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/demangle.cpp b/src/util/demangle.cpp index 69a18f6b8..be0c20935 100644 --- a/src/util/demangle.cpp +++ b/src/util/demangle.cpp @@ -69,7 +69,7 @@ namespace util { std::array name; if (int len = UnDecorateSymbolName(symbol, name.data(), name.size(), 0)) { - std::string demangled(name, len); + std::string demangled(name.data(), len); demangled = std::regex_replace(demangled, std::regex(R"(struct\s+)"), ""); demangled = std::regex_replace(demangled, std::regex(R"(class\s+)"), ""); demangled = std::regex_replace(demangled, std::regex(R"(\s+)"), ""); From cd9a11f94f889618700c9749f065942213dfccd4 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 14:58:29 +1000 Subject: [PATCH 070/176] Demangle --- src/util/demangle.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/util/demangle.cpp b/src/util/demangle.cpp index be0c20935..87829b9e8 100644 --- a/src/util/demangle.cpp +++ b/src/util/demangle.cpp @@ -66,14 +66,15 @@ namespace util { init_symbols(); } - std::array name; + std::array name{}; + auto len = UnDecorateSymbolName(symbol, name.data(), name.size(), 0); - if (int len = UnDecorateSymbolName(symbol, name.data(), name.size(), 0)) { + if (len > 0) { std::string demangled(name.data(), len); demangled = std::regex_replace(demangled, std::regex(R"(struct\s+)"), ""); demangled = std::regex_replace(demangled, std::regex(R"(class\s+)"), ""); demangled = std::regex_replace(demangled, std::regex(R"(\s+)"), ""); - return std::demangled; + return demangled; } else { return symbol; From ac1b47118ca1bf0486af8d6ffd03edf8a0b72e4e Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 15:03:53 +1000 Subject: [PATCH 071/176] Review comments --- src/extension/IOController_Posix.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index 51969dac6..aa52d4393 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -101,6 +101,9 @@ namespace extension { dirty = false; } + /** + * @brief Collects the events that have happened and stores them on the reactions to be fired + */ void collect_events() { // Get the lock so we don't concurrently modify the list @@ -161,6 +164,9 @@ namespace extension { } } + /** + * @brief Fires the events that have been collected when the reactions are ready + */ void fire_events() { const std::lock_guard lock(tasks_mutex); From 41ce4f4f8fdecacecdbe5819d9bb7ab4bc3264b3 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 15:09:53 +1000 Subject: [PATCH 072/176] more review comments --- src/dsl/word/Watchdog.hpp | 2 +- src/util/FileDescriptor.cpp | 8 +++++--- tests/dsl/Every.cpp | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/dsl/word/Watchdog.hpp b/src/dsl/word/Watchdog.hpp index 74aa6a66f..85672df0b 100644 --- a/src/dsl/word/Watchdog.hpp +++ b/src/dsl/word/Watchdog.hpp @@ -277,7 +277,7 @@ namespace dsl { } // Now automatically service the watchdog - time = time + period(ticks); + time += period(ticks); } // Change our wait time to our new watchdog time else { diff --git a/src/util/FileDescriptor.cpp b/src/util/FileDescriptor.cpp index bf8988c5f..4715fdddb 100644 --- a/src/util/FileDescriptor.cpp +++ b/src/util/FileDescriptor.cpp @@ -26,16 +26,18 @@ namespace util { } } - FileDescriptor::FileDescriptor(FileDescriptor&& rhs) noexcept : fd{rhs.fd} { + FileDescriptor::FileDescriptor(FileDescriptor&& rhs) noexcept { if (this != &rhs) { - rhs.fd = INVALID_SOCKET; + fd = std::exchange(rhs.fd, INVALID_SOCKET); + cleanup = std::move(rhs.cleanup); } } FileDescriptor& FileDescriptor::operator=(FileDescriptor&& rhs) noexcept { if (this != &rhs) { this->close(); - fd = std::exchange(rhs.fd, INVALID_SOCKET); + fd = std::exchange(rhs.fd, INVALID_SOCKET); + cleanup = std::move(rhs.cleanup); } return *this; } diff --git a/tests/dsl/Every.cpp b/tests/dsl/Every.cpp index 0856d99de..982a04eeb 100644 --- a/tests/dsl/Every.cpp +++ b/tests/dsl/Every.cpp @@ -60,7 +60,7 @@ void test_results(const std::vector& times) { const double sum = std::accumulate(std::begin(diff), std::end(diff), 0.0); const double mean = sum / double(diff.size()); const double variance = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0); - const double stddev = std::sqrt(variance / double(diff.size())); + const double stddev = std::sqrt(variance / double(diff.size() - 1)); // As time goes on the average wait should be close to 0 INFO("Average error in timing: " << mean << "±" << stddev); From 6831bf1d5211d9b732f66724149f957617c8a4b1 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 15:28:50 +1000 Subject: [PATCH 073/176] Build already! --- src/util/demangle.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/demangle.cpp b/src/util/demangle.cpp index 87829b9e8..8a8757568 100644 --- a/src/util/demangle.cpp +++ b/src/util/demangle.cpp @@ -67,7 +67,7 @@ namespace util { } std::array name{}; - auto len = UnDecorateSymbolName(symbol, name.data(), name.size(), 0); + auto len = UnDecorateSymbolName(symbol, name.data(), DWORD(name.size()), 0); if (len > 0) { std::string demangled(name.data(), len); From edeefa363a83808af7b1e0b09a5acb6c22ef88cd Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 21:18:54 +1000 Subject: [PATCH 074/176] Test a potential refactor for chrono controller to improve windows accuracy --- src/extension/ChronoController.hpp | 113 ++++++++++++++++++++--------- src/util/precise_sleep.cpp | 59 +++++++++++++++ src/util/precise_sleep.hpp | 32 ++++++++ 3 files changed, 170 insertions(+), 34 deletions(-) create mode 100644 src/util/precise_sleep.cpp create mode 100644 src/util/precise_sleep.hpp diff --git a/src/extension/ChronoController.hpp b/src/extension/ChronoController.hpp index 71e5e21fe..97f76516e 100644 --- a/src/extension/ChronoController.hpp +++ b/src/extension/ChronoController.hpp @@ -21,6 +21,7 @@ #include "../PowerPlant.hpp" #include "../Reactor.hpp" +#include "../util/precise_sleep.hpp" namespace NUClear { namespace extension { @@ -30,8 +31,7 @@ namespace extension { using ChronoTask = NUClear::dsl::operation::ChronoTask; public: - explicit ChronoController(std::unique_ptr environment) - : Reactor(std::move(environment)), wait_offset(std::chrono::milliseconds(0)) { + explicit ChronoController(std::unique_ptr environment) : Reactor(std::move(environment)) { on>().then("Add Chrono task", [this](const std::shared_ptr& task) { // Lock the mutex while we're doing stuff @@ -40,6 +40,7 @@ namespace extension { // Add our new task to the heap if we are still running if (running) { tasks.push_back(*task); + std::push_heap(tasks.begin(), tasks.end(), std::greater<>()); } // Poke the system @@ -60,6 +61,7 @@ namespace extension { // Remove if it exists if (it != tasks.end()) { tasks.erase(it); + std::make_heap(tasks.begin(), tasks.end(), std::greater<>()); } // Poke the system to make sure it's not waiting on something that's gone @@ -74,65 +76,108 @@ namespace extension { }); on().then("Chrono Controller", [this] { - // Acquire the mutex lock so we can wait on it - std::unique_lock lock(mutex); - - if (!running) { - return; + // Estimate the accuracy of our cv wait and nanosleep + for (int i = 0; i < 10; ++i) { + // Estimate the accuracy of our cv wait + std::mutex test; + std::unique_lock lock(test); + const auto cv_s = NUClear::clock::now(); + wait.wait_for(lock, std::chrono::milliseconds(1)); + const auto cv_e = NUClear::clock::now(); + cv_accuracy = + NUClear::clock::duration(std::abs((cv_e - cv_s - std::chrono::milliseconds(1)).count())); + + // Estimate the accuracy of our nanosleep + const auto ns_s = NUClear::clock::now(); + util::precise_sleep(std::chrono::milliseconds(1)); + const auto ns_e = NUClear::clock::now(); + ns_accuracy = + NUClear::clock::duration(std::abs((ns_e - ns_s - std::chrono::milliseconds(1)).count())); + + std::cout << "CV:" << std::endl; + std::cout << "\tStart: " << cv_s.time_since_epoch().count() << std::endl; + std::cout << "\tEnd: " << cv_e.time_since_epoch().count() << std::endl; + std::cout << "\tDelta: " << (cv_e - cv_s).count() << std::endl; + std::cout << "\tAccuracy: " + << std::chrono::duration_cast>( + cv_e - cv_s - std::chrono::milliseconds(1)) + .count() + << "ms" << std::endl; + + std::cout << "NS:" << std::endl; + std::cout << "\tStart: " << ns_s.time_since_epoch().count() << std::endl; + std::cout << "\tEnd: " << ns_e.time_since_epoch().count() << std::endl; + std::cout << "\tDelta: " << (ns_e - ns_s).count() << std::endl; + std::cout << "\tAccuracy: " + << std::chrono::duration_cast>( + ns_e - ns_s - std::chrono::milliseconds(1)) + .count() + << "ms" << std::endl; + std::cout << std::endl; } + exit(1); - // If we have tasks to do - if (!tasks.empty()) { + // Run until we are told to stop + while (running) { - // Make the list into a heap so we can remove the soonest ones - std::make_heap(tasks.begin(), tasks.end(), std::greater<>()); + // Acquire the mutex lock so we can wait on it + std::unique_lock lock(mutex); - // If we are within the wait offset of the time, spinlock until we get there for greater - // accuracy - if (NUClear::clock::now() + wait_offset > tasks.front().time) { + // If we have no chrono tasks wait until we are notified + if (tasks.empty()) { + wait.wait(lock); + } + else { + auto now = NUClear::clock::now(); + auto target = tasks.front().time; - // Spinlock! - while (NUClear::clock::now() < tasks.front().time) { + if (target - now > cv_accuracy) { + // Wait on the cv + wait.wait_until(lock, target - cv_accuracy); } + else if (target - now > ns_accuracy) { + // Wait on nanosleep + util::precise_sleep(target - now - ns_accuracy); + } + else { + // Spinlock until we get to the time + while (NUClear::clock::now() < tasks.front().time) { + } - const NUClear::clock::time_point now = NUClear::clock::now(); - - // Move back from the end poping the heap - for (auto end = tasks.end(); end != tasks.begin() && tasks.front().time < now;) { // Run our task and if it returns false remove it const bool renew = tasks.front()(); // Move this to the back of the list - std::pop_heap(tasks.begin(), end, std::greater<>()); + std::pop_heap(tasks.begin(), tasks.end(), std::greater<>()); - if (!renew) { - end = tasks.erase(--end); + if (renew) { + // Put the item back in the list + std::push_heap(tasks.begin(), tasks.end(), std::greater<>()); } else { - --end; + // Remove the item from the list + tasks.pop_back(); } } } - // Otherwise we wait for the next event using a wait_for (with a small offset for greater - // accuracy) Either that or until we get interrupted with a new event - else { - wait.wait_until(lock, tasks.front().time - wait_offset); - } - } - // Otherwise we wait for something to happen - else { - wait.wait(lock); } }); } private: + /// @brief The list of tasks we need to process std::vector tasks; + /// @brief The mutex we use to lock the task list std::mutex mutex; + /// @brief The condition variable we use to wait on std::condition_variable wait; + /// @brief If we are running or not bool running{true}; - NUClear::clock::duration wait_offset; + /// @brief The temporal accuracy when waiting on a condition variable + NUClear::clock::duration cv_accuracy; + /// @brief The temporal accuracy when waiting on nanosleep + NUClear::clock::duration ns_accuracy; }; } // namespace extension diff --git a/src/util/precise_sleep.cpp b/src/util/precise_sleep.cpp new file mode 100644 index 000000000..d8c873000 --- /dev/null +++ b/src/util/precise_sleep.cpp @@ -0,0 +1,59 @@ +#include "precise_sleep.hpp" + +namespace NUClear { +namespace util { + +#if defined(_WIN32) + + #define WIN32_LEAN_AND_MEAN + #include + + uint64_t_t get_performance_frequency() { + ::LARGE_INTEGER freq; + ::QueryPerformanceFrequency(&freq); + return freq.QuadPart; + } + + uint64_t performance_frequency() { + static uint64 xFreq = GetPerfFrequency(); + return xFreq; + } + + uint64_t performance_counter() { + ::LARGE_INTEGER counter; + ::QueryPerformanceCounter(&counter); + return counter.QuadPart; + } + + void precise_sleep(const std::chrono::nanoseconds& ns) { + ::LARGE_INTEGER ft; + // TODO if ns is negative make it 0 as otherwise it'll become absolute time + // Negative for relative time, positive for absolute time + // Measures in 100ns increments so divide by 100 + ft.QuadPart = -static_cast(ns.count / 100); + + ::HANDLE timer = ::CreateWaitableTimer(nullptr, TRUE, nullptr); + ::SetWaitableTimer(timer, &ft, 0, nullptr, nullptr, 0); + ::WaitForSingleObject(timer, INFINITE); + ::CloseHandle(timer); + } + +#else + + #include + #include + #include + + void precise_sleep(const std::chrono::nanoseconds& ns) { + struct timespec ts; + ts.tv_sec = std::chrono::duration_cast(ns).count(); + ts.tv_nsec = (ns - std::chrono::seconds(ts.tv_sec)).count(); + + while (::nanosleep(&ts, &ts) == -1 && errno == EINTR) { + } + } + +#endif + +} // namespace util +} // namespace NUClear diff --git a/src/util/precise_sleep.hpp b/src/util/precise_sleep.hpp new file mode 100644 index 000000000..950870653 --- /dev/null +++ b/src/util/precise_sleep.hpp @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_UTIL_SLEEPER_HPP +#define NUCLEAR_UTIL_SLEEPER_HPP + +#include + +namespace NUClear { +namespace util { + + void precise_sleep(const std::chrono::nanoseconds& duration); + +} // namespace util +} // namespace NUClear + +#endif // NUCLEAR_UTIL_SLEEPER_HPP From 2df9f820b6e47da803b2627a770e8b4fe156bb98 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 21:24:30 +1000 Subject: [PATCH 075/176] . --- src/util/precise_sleep.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/util/precise_sleep.cpp b/src/util/precise_sleep.cpp index d8c873000..c38e52408 100644 --- a/src/util/precise_sleep.cpp +++ b/src/util/precise_sleep.cpp @@ -8,18 +8,21 @@ namespace util { #define WIN32_LEAN_AND_MEAN #include - uint64_t_t get_performance_frequency() { + #include + #include + + int64_t_t get_performance_frequency() { ::LARGE_INTEGER freq; ::QueryPerformanceFrequency(&freq); return freq.QuadPart; } - uint64_t performance_frequency() { - static uint64 xFreq = GetPerfFrequency(); - return xFreq; + int64_t performance_frequency() { + static int64_t freq = get_performance_frequency(); + return freq; } - uint64_t performance_counter() { + int64_t performance_counter() { ::LARGE_INTEGER counter; ::QueryPerformanceCounter(&counter); return counter.QuadPart; From 4daf4185dff44b7de7202a62a8c23684ef5a92e4 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 21:51:36 +1000 Subject: [PATCH 076/176] . --- src/util/precise_sleep.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/precise_sleep.cpp b/src/util/precise_sleep.cpp index c38e52408..df38723d3 100644 --- a/src/util/precise_sleep.cpp +++ b/src/util/precise_sleep.cpp @@ -11,7 +11,7 @@ namespace util { #include #include - int64_t_t get_performance_frequency() { + int64_t get_performance_frequency() { ::LARGE_INTEGER freq; ::QueryPerformanceFrequency(&freq); return freq.QuadPart; From 92781e4600d4169964a8e3fb3f530cd9e42ec78a Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 21:57:49 +1000 Subject: [PATCH 077/176] . --- src/util/precise_sleep.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/precise_sleep.cpp b/src/util/precise_sleep.cpp index df38723d3..2ea633633 100644 --- a/src/util/precise_sleep.cpp +++ b/src/util/precise_sleep.cpp @@ -6,11 +6,11 @@ namespace util { #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN - #include - #include #include + #include "platform.hpp" + int64_t get_performance_frequency() { ::LARGE_INTEGER freq; ::QueryPerformanceFrequency(&freq); From 5da9e1011209c1ca9eecec341bdea283ddd4de56 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 21:58:28 +1000 Subject: [PATCH 078/176] I don't even need those! --- src/util/precise_sleep.cpp | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/util/precise_sleep.cpp b/src/util/precise_sleep.cpp index 2ea633633..25f48a5c1 100644 --- a/src/util/precise_sleep.cpp +++ b/src/util/precise_sleep.cpp @@ -11,23 +11,6 @@ namespace util { #include "platform.hpp" - int64_t get_performance_frequency() { - ::LARGE_INTEGER freq; - ::QueryPerformanceFrequency(&freq); - return freq.QuadPart; - } - - int64_t performance_frequency() { - static int64_t freq = get_performance_frequency(); - return freq; - } - - int64_t performance_counter() { - ::LARGE_INTEGER counter; - ::QueryPerformanceCounter(&counter); - return counter.QuadPart; - } - void precise_sleep(const std::chrono::nanoseconds& ns) { ::LARGE_INTEGER ft; // TODO if ns is negative make it 0 as otherwise it'll become absolute time From e741d47be3d541b452ffb23d6b3efd370ac86402 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 21:58:41 +1000 Subject: [PATCH 079/176] . --- src/util/precise_sleep.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/util/precise_sleep.cpp b/src/util/precise_sleep.cpp index 25f48a5c1..1082738cb 100644 --- a/src/util/precise_sleep.cpp +++ b/src/util/precise_sleep.cpp @@ -5,7 +5,6 @@ namespace util { #if defined(_WIN32) - #define WIN32_LEAN_AND_MEAN #include #include From 4fed00d62bb9e39b9de2904a9e44c85ed74c42ec Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 22:11:18 +1000 Subject: [PATCH 080/176] . --- src/util/precise_sleep.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/util/precise_sleep.cpp b/src/util/precise_sleep.cpp index 1082738cb..8b87cebc9 100644 --- a/src/util/precise_sleep.cpp +++ b/src/util/precise_sleep.cpp @@ -10,12 +10,17 @@ namespace util { #include "platform.hpp" + // Keep this include here + #ifdef _WIN32 + #include + #endif + void precise_sleep(const std::chrono::nanoseconds& ns) { ::LARGE_INTEGER ft; // TODO if ns is negative make it 0 as otherwise it'll become absolute time // Negative for relative time, positive for absolute time // Measures in 100ns increments so divide by 100 - ft.QuadPart = -static_cast(ns.count / 100); + ft.QuadPart = -static_cast(ns.count() / 100); ::HANDLE timer = ::CreateWaitableTimer(nullptr, TRUE, nullptr); ::SetWaitableTimer(timer, &ft, 0, nullptr, nullptr, 0); From 37a7a4b87a8bef6799e8f2727c420ff7ba4895e5 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 10 Aug 2023 22:14:57 +1000 Subject: [PATCH 081/176] . --- src/util/precise_sleep.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/util/precise_sleep.cpp b/src/util/precise_sleep.cpp index 8b87cebc9..e47a0d7f8 100644 --- a/src/util/precise_sleep.cpp +++ b/src/util/precise_sleep.cpp @@ -1,8 +1,5 @@ #include "precise_sleep.hpp" -namespace NUClear { -namespace util { - #if defined(_WIN32) #include @@ -10,10 +7,8 @@ namespace util { #include "platform.hpp" - // Keep this include here - #ifdef _WIN32 - #include - #endif +namespace NUClear { +namespace util { void precise_sleep(const std::chrono::nanoseconds& ns) { ::LARGE_INTEGER ft; @@ -28,12 +23,18 @@ namespace util { ::CloseHandle(timer); } +} // namespace util +} // namespace NUClear + #else #include #include #include +namespace NUClear { +namespace util { + void precise_sleep(const std::chrono::nanoseconds& ns) { struct timespec ts; ts.tv_sec = std::chrono::duration_cast(ns).count(); @@ -43,7 +44,7 @@ namespace util { } } -#endif - } // namespace util } // namespace NUClear + +#endif From 59ce1f15663e1629e38349a99ed6e67cc6bd5260 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 08:52:02 +1000 Subject: [PATCH 082/176] See if this makes windows pass the tests --- src/extension/ChronoController.hpp | 58 +++++++++--------------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/src/extension/ChronoController.hpp b/src/extension/ChronoController.hpp index 97f76516e..3b2cd8bdd 100644 --- a/src/extension/ChronoController.hpp +++ b/src/extension/ChronoController.hpp @@ -33,6 +33,23 @@ namespace extension { public: explicit ChronoController(std::unique_ptr environment) : Reactor(std::move(environment)) { + // Estimate the accuracy of our cv wait and precise sleep + { + // Estimate the accuracy of our cv wait + std::mutex test; + std::unique_lock lock(test); + const auto cv_s = NUClear::clock::now(); + wait.wait_for(lock, std::chrono::milliseconds(1)); + const auto cv_e = NUClear::clock::now(); + cv_accuracy = NUClear::clock::duration(std::abs((cv_e - cv_s - std::chrono::milliseconds(1)).count())); + + // Estimate the accuracy of our precise sleep + const auto ns_s = NUClear::clock::now(); + util::precise_sleep(std::chrono::milliseconds(1)); + const auto ns_e = NUClear::clock::now(); + ns_accuracy = NUClear::clock::duration(std::abs((ns_e - ns_s - std::chrono::milliseconds(1)).count())); + } + on>().then("Add Chrono task", [this](const std::shared_ptr& task) { // Lock the mutex while we're doing stuff const std::lock_guard lock(mutex); @@ -76,47 +93,6 @@ namespace extension { }); on().then("Chrono Controller", [this] { - // Estimate the accuracy of our cv wait and nanosleep - for (int i = 0; i < 10; ++i) { - // Estimate the accuracy of our cv wait - std::mutex test; - std::unique_lock lock(test); - const auto cv_s = NUClear::clock::now(); - wait.wait_for(lock, std::chrono::milliseconds(1)); - const auto cv_e = NUClear::clock::now(); - cv_accuracy = - NUClear::clock::duration(std::abs((cv_e - cv_s - std::chrono::milliseconds(1)).count())); - - // Estimate the accuracy of our nanosleep - const auto ns_s = NUClear::clock::now(); - util::precise_sleep(std::chrono::milliseconds(1)); - const auto ns_e = NUClear::clock::now(); - ns_accuracy = - NUClear::clock::duration(std::abs((ns_e - ns_s - std::chrono::milliseconds(1)).count())); - - std::cout << "CV:" << std::endl; - std::cout << "\tStart: " << cv_s.time_since_epoch().count() << std::endl; - std::cout << "\tEnd: " << cv_e.time_since_epoch().count() << std::endl; - std::cout << "\tDelta: " << (cv_e - cv_s).count() << std::endl; - std::cout << "\tAccuracy: " - << std::chrono::duration_cast>( - cv_e - cv_s - std::chrono::milliseconds(1)) - .count() - << "ms" << std::endl; - - std::cout << "NS:" << std::endl; - std::cout << "\tStart: " << ns_s.time_since_epoch().count() << std::endl; - std::cout << "\tEnd: " << ns_e.time_since_epoch().count() << std::endl; - std::cout << "\tDelta: " << (ns_e - ns_s).count() << std::endl; - std::cout << "\tAccuracy: " - << std::chrono::duration_cast>( - ns_e - ns_s - std::chrono::milliseconds(1)) - .count() - << "ms" << std::endl; - std::cout << std::endl; - } - exit(1); - // Run until we are told to stop while (running) { From 5440261449427a4736491e8fe0a0255af3bb1289 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 09:28:20 +1000 Subject: [PATCH 083/176] Try a few loops to get better wait accuracy --- src/extension/ChronoController.hpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/extension/ChronoController.hpp b/src/extension/ChronoController.hpp index 3b2cd8bdd..51cd9b9f5 100644 --- a/src/extension/ChronoController.hpp +++ b/src/extension/ChronoController.hpp @@ -34,20 +34,24 @@ namespace extension { explicit ChronoController(std::unique_ptr environment) : Reactor(std::move(environment)) { // Estimate the accuracy of our cv wait and precise sleep - { + for (int i = 0; i < 5; ++i) { // Estimate the accuracy of our cv wait std::mutex test; std::unique_lock lock(test); const auto cv_s = NUClear::clock::now(); wait.wait_for(lock, std::chrono::milliseconds(1)); const auto cv_e = NUClear::clock::now(); - cv_accuracy = NUClear::clock::duration(std::abs((cv_e - cv_s - std::chrono::milliseconds(1)).count())); + const auto cv_a = NUClear::clock::duration(cv_e - cv_s - std::chrono::milliseconds(1)); // Estimate the accuracy of our precise sleep const auto ns_s = NUClear::clock::now(); util::precise_sleep(std::chrono::milliseconds(1)); const auto ns_e = NUClear::clock::now(); - ns_accuracy = NUClear::clock::duration(std::abs((ns_e - ns_s - std::chrono::milliseconds(1)).count())); + const auto ns_a = NUClear::clock::duration(ns_e - ns_s - std::chrono::milliseconds(1)); + + // Use the largest time we have seen + cv_accuracy = cv_a > cv_accuracy ? cv_a : cv_accuracy; + ns_accuracy = ns_a > ns_accuracy ? ns_a : ns_accuracy; } on>().then("Add Chrono task", [this](const std::shared_ptr& task) { @@ -151,9 +155,9 @@ namespace extension { bool running{true}; /// @brief The temporal accuracy when waiting on a condition variable - NUClear::clock::duration cv_accuracy; + NUClear::clock::duration cv_accuracy{0}; /// @brief The temporal accuracy when waiting on nanosleep - NUClear::clock::duration ns_accuracy; + NUClear::clock::duration ns_accuracy{0}; }; } // namespace extension From 97523346f416c4ec9bd77738db856fb8545db431 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 09:52:02 +1000 Subject: [PATCH 084/176] Remove connection accepted from TCP as it's implied by having a connection and sometimes is out of order --- tests/dsl/TCP.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index e0c2576d5..7709ea4e4 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -67,7 +67,6 @@ class TestReactor : public test_util::TestBase { // Bind to a known port on(PORT).then([this](const TCP::Connection& connection) { - events.push_back("Known Port connection accepted"); on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { handle_data("Known Port", event); }); @@ -76,7 +75,6 @@ class TestReactor : public test_util::TestBase { // Bind to an unknown port and get the port number in_port_t ephemeral_port = 0; std::tie(std::ignore, ephemeral_port, std::ignore) = on().then([this](const TCP::Connection& connection) { - events.push_back("Ephemeral Port connection accepted"); on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { handle_data("Ephemeral Port", event); }); @@ -145,12 +143,10 @@ TEST_CASE("Testing listening for TCP connections and receiving data messages", " const std::vector expected = { "Known Port sending", - "Known Port connection accepted", "Known Port received: Hello TCP World!", "Known Port echoed: Hello TCP World!", "Known Port closed", "Ephemeral Port sending", - "Ephemeral Port connection accepted", "Ephemeral Port received: Hello TCP World!", "Ephemeral Port echoed: Hello TCP World!", "Ephemeral Port closed", From c0f74843ef21ee9b01e234b22428cb0bcf84d420 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 09:52:14 +1000 Subject: [PATCH 085/176] clang-tidy --- src/util/precise_sleep.cpp | 2 +- src/util/precise_sleep.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/precise_sleep.cpp b/src/util/precise_sleep.cpp index e47a0d7f8..ec5d13086 100644 --- a/src/util/precise_sleep.cpp +++ b/src/util/precise_sleep.cpp @@ -36,7 +36,7 @@ namespace NUClear { namespace util { void precise_sleep(const std::chrono::nanoseconds& ns) { - struct timespec ts; + timespec ts{}; ts.tv_sec = std::chrono::duration_cast(ns).count(); ts.tv_nsec = (ns - std::chrono::seconds(ts.tv_sec)).count(); diff --git a/src/util/precise_sleep.hpp b/src/util/precise_sleep.hpp index 950870653..407bd6029 100644 --- a/src/util/precise_sleep.hpp +++ b/src/util/precise_sleep.hpp @@ -24,7 +24,7 @@ namespace NUClear { namespace util { - void precise_sleep(const std::chrono::nanoseconds& duration); + void precise_sleep(const std::chrono::nanoseconds& ns); } // namespace util } // namespace NUClear From e2a534955175b1da711b97424a379061b4645118 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 10:43:18 +1000 Subject: [PATCH 086/176] Continuously update the error --- src/extension/ChronoController.hpp | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/extension/ChronoController.hpp b/src/extension/ChronoController.hpp index 51cd9b9f5..5de6108d2 100644 --- a/src/extension/ChronoController.hpp +++ b/src/extension/ChronoController.hpp @@ -34,7 +34,7 @@ namespace extension { explicit ChronoController(std::unique_ptr environment) : Reactor(std::move(environment)) { // Estimate the accuracy of our cv wait and precise sleep - for (int i = 0; i < 5; ++i) { + for (int i = 0; i < 3; ++i) { // Estimate the accuracy of our cv wait std::mutex test; std::unique_lock lock(test); @@ -108,16 +108,30 @@ namespace extension { wait.wait(lock); } else { - auto now = NUClear::clock::now(); + auto start = NUClear::clock::now(); auto target = tasks.front().time; - if (target - now > cv_accuracy) { + if (target - start > cv_accuracy) { // Wait on the cv wait.wait_until(lock, target - cv_accuracy); + + // Update the accuracy of our cv wait + const auto end = NUClear::clock::now(); + const auto error = end - (target - cv_accuracy); // when ended - when wanted to end + if (error.count() > 0) { // only if we were late + cv_accuracy = error > cv_accuracy ? error : ((cv_accuracy * 99 + error) / 100); + } } - else if (target - now > ns_accuracy) { + else if (target - start > ns_accuracy) { // Wait on nanosleep - util::precise_sleep(target - now - ns_accuracy); + util::precise_sleep(target - start - ns_accuracy); + + // Update the accuracy of our precise sleep + const auto end = NUClear::clock::now(); + const auto error = end - (target - ns_accuracy); // when ended - when wanted to end + if (error.count() > 0) { // only if we were late + ns_accuracy = error > ns_accuracy ? error : ((ns_accuracy * 99 + error) / 100); + } } else { // Spinlock until we get to the time From 19a61dd5603782748c7682fb655eb933fe6583ea Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 12:25:51 +1000 Subject: [PATCH 087/176] . --- src/util/precise_sleep.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/util/precise_sleep.cpp b/src/util/precise_sleep.cpp index 1185bcb39..1fd2957fa 100644 --- a/src/util/precise_sleep.cpp +++ b/src/util/precise_sleep.cpp @@ -1,4 +1,3 @@ - /* * Copyright (C) 2013 Trent Houliston , Jake Woods * 2014-2023 Trent Houliston From 6abd417f05fae708717dba17f945e96dc5b22bcc Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 13:01:22 +1000 Subject: [PATCH 088/176] Attempt to update the windows iocontroller to be similar to posix --- src/extension/IOController_Posix.hpp | 43 ++- src/extension/IOController_Windows.hpp | 350 +++++++++++++++---------- 2 files changed, 227 insertions(+), 166 deletions(-) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index aa52d4393..8ac887cdf 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -81,19 +81,19 @@ namespace extension { const std::lock_guard lock(tasks_mutex); // Clear our fds to be rebuilt - fds.resize(0); + watches.resize(0); // Insert our notify fd - fds.push_back(pollfd{notify_recv, POLLIN, 0}); + watches.push_back(pollfd{notify_recv, POLLIN, 0}); for (const auto& r : tasks) { // If we are the same fd, then add our interest set - if (r.fd == fds.back().fd) { - fds.back().events = short(fds.back().events | r.events); // NOLINT(google-runtime-int) + if (r.fd == watches.back().fd) { + watches.back().events = short(watches.back().events | r.events); // NOLINT(google-runtime-int) } // Otherwise add a new one else { - fds.push_back(pollfd{r.fd, r.events, 0}); + watches.push_back(pollfd{r.fd, r.events, 0}); } } @@ -137,13 +137,13 @@ namespace extension { } // Find our relevant tasks - auto range = std::equal_range(std::begin(tasks), - std::end(tasks), + auto range = std::equal_range(tasks.begin(), + tasks.end(), Task{fd.fd, 0, nullptr}, [](const Task& a, const Task& b) { return a.fd < b.fd; }); // There are no tasks for this! - if (range.first == std::end(tasks)) { + if (range.first == tasks.end()) { // If this happens then our list is definitely dirty... dirty = true; } @@ -192,15 +192,10 @@ namespace extension { catch (...) { } - // Reset our value - - if ((it->waiting_events & IO::CLOSE) != 0) { - dirty = true; - it = tasks.erase(it); - } - else { - ++it; - } + // Remove if we received a close event + bool closed = (it->waiting_events & IO::CLOSE) != 0; + dirty |= closed; + it = closed ? tasks.erase(it) : std::next(it); } else { ++it; @@ -248,10 +243,10 @@ namespace extension { const std::lock_guard lock(tasks_mutex); // NOLINTNEXTLINE(google-runtime-int) - tasks.emplace_back(config.fd, static_cast(config.events), config.reaction); + tasks.emplace_back(config.fd, short(config.events), config.reaction); // Resort our list - std::sort(std::begin(tasks), std::end(tasks)); + std::sort(tasks.begin(), tasks.end()); // Let the poll command know that stuff happened dirty = true; @@ -263,12 +258,12 @@ namespace extension { const std::lock_guard lock(tasks_mutex); // Find the reaction that finished processing - auto task = std::find_if(std::begin(tasks), std::end(tasks), [&event](const Task& t) { + auto task = std::find_if(tasks.begin(), tasks.end(), [&event](const Task& t) { return t.reaction->id == event.id; }); // If we found it then clear the waiting events - if (task != std::end(tasks)) { + if (task != tasks.end()) { task->waiting_events = 0; } }); @@ -280,11 +275,11 @@ namespace extension { const std::lock_guard lock(tasks_mutex); // Find our reaction - auto reaction = std::find_if(std::begin(tasks), std::end(tasks), [&unbind](const Task& t) { + auto reaction = std::find_if(tasks.begin(), tasks.end(), [&unbind](const Task& t) { return t.reaction->id == unbind.id; }); - if (reaction != std::end(tasks)) { + if (reaction != tasks.end()) { tasks.erase(reaction); } @@ -339,7 +334,7 @@ namespace extension { /// @brief Whether or not the list of file descriptors is dirty compared to tasks bool dirty = true; /// @brief The list of file descriptors to poll - std::vector fds{}; + std::vector watches{}; /// @brief The list of tasks that are waiting for IO events std::vector tasks{}; }; diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index d3e1402c6..9833cd86a 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -28,6 +28,155 @@ namespace NUClear { namespace extension { class IOController : public Reactor { + private: + /** + * @brief A task that is waiting for an IO event + */ + struct Task { + Task() = default; + // NOLINTNEXTLINE(google-runtime-int) + Task(const fd_t& fd, short events, std::shared_ptr reaction) + : fd(fd), events(events), reaction(std::move(reaction)) {} + + /// @brief The socket we are waiting on + fd_t fd; + /// @brief The events that the task is interested in + short events{0}; // NOLINT(google-runtime-int) + /// @brief The events that are waiting to be fired + short waiting_events{0}; // NOLINT(google-runtime-int) + /// @brief The reaction that is waiting for this event + std::shared_ptr reaction{nullptr}; + }; + + /** + * @brief Rebuilds the list of file descriptors to poll + * + * This function is called when the list of file descriptors to poll changes. It will rebuild the list of file + * descriptors used by poll + */ + void rebuild_list() { + // Get the lock so we don't concurrently modify the list + const std::lock_guard lock(tasks_mutex); + + // Clear our fds to be rebuilt + watches.resize(0); + + // Insert our notify fd + watches.push_back(notifier); + + for (const auto& r : tasks) { + watches.push_back(r.first); + } + + // We just cleaned the list! + dirty = false; + } + + /** + * @brief Collects the events that have happened and stores them on the reactions to be fired + */ + void collect_events(const WSAEVENT& event) { + + // Get the lock so we don't concurrently modify the list + const std::lock_guard lock(tasks_mutex); + + if (event == notifier) { + // Reset the notifier signal + if (!WSAResetEvent(event)) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSAResetEvent() for notifier failed"); + } + } + else { + // Get our associated Event object, which has the reaction + auto r = tasks.find(event); + + // If it was found... + if (r != tasks.end()) { + // Enum the socket events to work out which ones fired + WSANETWORKEVENTS wsae; + if (WSAEnumNetworkEvents(r->second.fd, event, &wsae) == SOCKET_ERROR) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSAEnumNetworkEvents() failed"); + } + + r->second.waiting_events = r->second.waiting_events | wsae.lNetworkEvents; + } + // If we can't find the event then our list is dirty + else { + dirty = true; + } + } + } + + /** + * @brief Fires the events that have been collected when the reactions are ready + */ + void fire_events() { + const std::lock_guard lock(tasks_mutex); + + // Go through every reaction and if it has events and isn't already running then run it + for (auto it = tasks.begin(); it != tasks.end();) { + auto& event = it->first; + auto& task = it->second; + + if (task.reaction->active_tasks == 0 && task.waiting_events != 0) { + + // Make our event to pass through and store it in the local cache + IO::Event e{}; + e.fd = task.fd; + e.events = task.waiting_events; + + // Submit the task (which should run the get) + try { + IO::ThreadEventStore::value = &e; + auto reaction_task = task.reaction->get_task(); + IO::ThreadEventStore::value = nullptr; + // Only actually submit this task if it is the only active task for this reaction + if (reaction_task) { + powerplant.submit(std::move(reaction_task)); + } + } + catch (...) { + } + + // Reset our value + it = ((task.waiting_events & IO::CLOSE) != 0) ? remove_task(it) : std::next(it); + } + else { + ++it; + } + } + } + + /** + * @brief Bumps the notification pipe to wake up the poll command + * + * If the poll command is waiting it will wait forever if something doesn't happen. + * When trying to update what to poll or shut down we need to wake it up so it can. + */ + // NOLINTNEXTLINE(readability-make-member-function-const) this changes states + void bump() { + if (!WSASetEvent(notifier)) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSASetEvent() for configure io reaction failed"); + } + } + + std::map::iterator remove_task(std::map::iterator it) { + // Close the event + if (!WSACloseEvent(it->first)) { + throw std::system_error(WSAGetLastError(), std::system_category(), "WSACloseEvent() failed"); + } + + // Remove the task + return tasks.erase(it); + } + + public: explicit IOController(std::unique_ptr environment) : Reactor(std::move(environment)) { @@ -40,11 +189,6 @@ namespace extension { throw std::system_error(startup_status, std::system_category(), "WSAStartup() failed"); } - // Reserve 1024 event slots - // Hopefully we won't have more events than that - // Even if we do it should be fine (after a glitch) - events.reserve(1024); - // Create an event to use for the notifier (used for getting out of WSAWaitForMultipleEvents()) notifier = WSACreateEvent(); if (notifier == WSA_INVALID_EVENT) { @@ -53,14 +197,14 @@ namespace extension { "WSACreateEvent() for notifier failed"); } - // We always have the notifier in the event list - events.push_back(notifier); + // Start by rebuliding the list + rebuild_list(); on>().then( "Configure IO Reaction", [this](const dsl::word::IOConfiguration& config) { // Lock our mutex - std::lock_guard lock(reaction_mutex); + std::lock_guard lock(tasks_mutex); // Make an event for this SOCKET auto event = WSACreateEvent(); @@ -76,67 +220,63 @@ namespace extension { } // Add all the information to the list and mark the list as dirty, to sync with the list of events - reactions.insert(std::make_pair(event, Event{config.fd, config.reaction, config.events})); - reactions_list_dirty = true; + tasks.insert(std::make_pair(event, Task{config.fd, config.events, config.reaction})); + dirty = true; - // Signal the notifier event to return from WSAWaitForMultipleEvents() and sync the dirty list - if (!WSASetEvent(notifier)) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSASetEvent() for configure io reaction failed"); - } + bump(); }); + on>().then("IO Finished", [this](const dsl::word::IOFinished& event) { + // Get the lock so we don't concurrently modify the list + const std::lock_guard lock(tasks_mutex); + + // Find the reaction that finished processing + auto task = std::find_if(tasks.begin(), tasks.end(), [&event](const Task& t) { + return t.reaction->id == event.id; + }); + + // If we found it then clear the waiting events + if (task != tasks.end()) { + task->waiting_events = 0; + } + }); + on>>().then( "Unbind IO Reaction", [this](const dsl::operation::Unbind& unbind) { - // Lock our mutex - std::lock_guard lock(reaction_mutex); - - // Find this reaction in our list of reactions - auto reaction = std::find_if(std::begin(reactions), - std::end(reactions), - [&unbind](const std::pair& item) { - return item.second.reaction->id == unbind.id; - }); - - // If the reaction was found - if (reaction != std::end(reactions)) { - // Remove it from the list of reactions - reactions.erase(reaction); - - // Queue the associated event for closing when we sync - events_to_close.push_back(reaction->first); - } - else { - // Fail silently: we've unbound a reaction that somehow isn't in our list of reactions! - } + // Lock our mutex to avoid concurrent modification + const std::lock_guard lock(tasks_mutex); - // Flag that our list is dirty - reactions_list_dirty = true; + // Find our reaction + auto it = std::find_if(tasks.begin(), tasks.end(), [&unbind](const Task& t) { + return t.reaction->id == unbind.id; + }); - // Signal the notifier event to return from WSAWaitForMultipleEvents() and sync the dirty list - if (!WSASetEvent(notifier)) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSASetEvent() for unbind io reaction failed"); + if (it != tasks.end()) { + remove_task(it); } + + // Let the poll command know that stuff happened + dirty = true; + bump(); }); on().then("Shutdown IO Controller", [this] { - // Set shutdown to true + // Set shutdown to true so it won't try to poll again shutdown.store(true); - - // Signal the notifier event to return from WSAWaitForMultipleEvents() and shutdown - if (!WSASetEvent(notifier)) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSASetEvent() for shutdown failed"); - } + bump(); }); on().then("IO Controller", [this] { + // To make sure we don't get caught in a weird loop + // shutdown keeps us out here if (!shutdown.load()) { + + // Rebuild the list if something changed + if (dirty) { + rebuild_list(); + } + // Wait for events auto event_index = WSAWaitForMultipleEvents(static_cast(events.size()), events.data(), @@ -149,86 +289,12 @@ namespace extension { // Get the signalled event auto& event = events[event_index - WSA_WAIT_EVENT_0]; - if (event == notifier) { - // Reset the notifier signal - if (!WSAResetEvent(event)) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSAResetEvent() for notifier failed"); - } - } - else { - // Get our associated Event object, which has the reaction - auto r = reactions.find(event); - - // If it was found... - if (r != reactions.end()) { - // Enum the socket events to work out which ones fired - WSANETWORKEVENTS wsae; - if (WSAEnumNetworkEvents(r->second.fd, event, &wsae) == SOCKET_ERROR) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSAEnumNetworkEvents() failed"); - } - - // Make our IO event to pass through - IO::Event io_event; - io_event.fd = r->second.fd; - - // The events that fired are what we got from the enum events call - io_event.events = wsae.lNetworkEvents; - - // Store the IO event in our thread local cache - IO::ThreadEventStore::value = &io_event; - - // Submit the task (which should run the get) - try { - auto task = r->second.reaction->get_task(); - if (task) { - powerplant.submit(std::move(task)); - } - } - catch (...) { - } - - // Reset our value - IO::ThreadEventStore::value = nullptr; - } - } - } - } - - if (reactions_list_dirty || !events_to_close.empty()) { - // Get the lock so we don't concurrently modify the list - std::lock_guard lock(reaction_mutex); - - // Close any events we've queued for closing - if (!events_to_close.empty()) { - for (auto& event : events_to_close) { - if (!WSACloseEvent(event)) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSACloseEvent() failed"); - } - } - - // Clear the queue of closed events - events_to_close.clear(); - } - - // Clear the list of events, to be rebuilt - events.resize(0); - - // Add back the notifier event - events.push_back(notifier); + // Collect the events that happened into the tasks list + collect_events(event); - // Sync the list of reactions to the list of events - for (const auto& r : reactions) { - events.push_back(r.first); + // Fire the events that happened if we can + fire_events(); } - - // The list has been synced - reactions_list_dirty = false; } }); } @@ -238,22 +304,22 @@ namespace extension { WSACleanup(); } - private: - struct Event { - SOCKET fd; - std::shared_ptr reaction; - int events; - }; + private: + /// @brief The event that is used to wake up the WaitForMultipleEvents call WSAEVENT notifier; + /// @brief Whether or not we are shutting down std::atomic shutdown{false}; - bool reactions_list_dirty = false; - - std::mutex reaction_mutex; - std::map reactions; - std::vector events; - std::vector events_to_close; + /// @brief The mutex that protects the tasks list + std::mutex tasks_mutex; + /// @brief Whether or not the list of file descriptors is dirty compared to tasks + bool dirty = true; + + /// @brief The list of tasks that are currently being processed + std::vector watches; + /// @brief The list of tasks that are waiting for IO events + std::map tasks; }; } // namespace extension From 9d3149e01664931b143bce3d6926feec76635cab Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 13:06:16 +1000 Subject: [PATCH 089/176] . --- src/extension/IOController_Posix.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index 8ac887cdf..33a1e74b8 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -109,7 +109,7 @@ namespace extension { // Get the lock so we don't concurrently modify the list const std::lock_guard lock(tasks_mutex); - for (auto& fd : fds) { + for (auto& fd : watches) { // Something happened if (fd.revents != 0) { @@ -305,7 +305,7 @@ namespace extension { } // Wait for an event to happen on one of our file descriptors - if (::poll(fds.data(), static_cast(fds.size()), -1) < 0) { + if (::poll(watches.data(), nfds_t(watches.size()), -1) < 0) { throw std::system_error(network_errno, std::system_category(), "There was an IO error while attempting to poll the file descriptors"); From 401c13fd1940db2eab5dc3a76cc6c42ae6e2084d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 13:10:52 +1000 Subject: [PATCH 090/176] . --- src/extension/IOController_Windows.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index 9833cd86a..ab18a51c9 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -319,7 +319,7 @@ namespace extension { /// @brief The list of tasks that are currently being processed std::vector watches; /// @brief The list of tasks that are waiting for IO events - std::map tasks; + std::map tasks; }; } // namespace extension From 31bda94106ec15787e318e0cea0e076c180944e9 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 13:23:51 +1000 Subject: [PATCH 091/176] . --- src/dsl/word/IO.hpp | 33 ++++++++++++++++++++------ src/extension/IOController_Windows.hpp | 6 ++--- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/dsl/word/IO.hpp b/src/dsl/word/IO.hpp index 095580620..961bb8bf4 100644 --- a/src/dsl/word/IO.hpp +++ b/src/dsl/word/IO.hpp @@ -31,16 +31,32 @@ namespace NUClear { namespace dsl { namespace word { +#ifdef _WIN32 + using event_t = long; +#else + using event_t = short; +#endif + + /** + * @brief This message is sent to the IO controller to configure a new IO operation. + */ struct IOConfiguration { - IOConfiguration(fd_t fd, int events, std::shared_ptr reaction) + IOConfiguration(fd_t fd, event_t events, std::shared_ptr reaction) : fd(fd), events(events), reaction(std::move(reaction)) {} + /// @brief The file descriptor to watch fd_t fd; - int events; + /// @brief The events to watch for on this file descriptor + event_t events; + /// @brief The reaction to trigger when this file descriptor has an event std::shared_ptr reaction; }; + /** + * @brief This is emitted when an IO operation has finished. + */ struct IOFinished { IOFinished(const uint64_t& id) : id(id) {} + /// @brief The id of the reaction that has finished uint64_t id; }; @@ -80,7 +96,7 @@ namespace dsl { // On windows we use different wait events #ifdef _WIN32 // NOLINTNEXTLINE(google-runtime-int) - enum EventType : short { + enum EventType : event_t { READ = FD_READ | FD_OOB | FD_ACCEPT, WRITE = FD_WRITE, CLOSE = FD_CLOSE, @@ -88,7 +104,7 @@ namespace dsl { }; #else // NOLINTNEXTLINE(google-runtime-int) - enum EventType : short { + enum EventType : event_t { READ = POLLIN, WRITE = POLLOUT, CLOSE = POLLHUP, @@ -97,18 +113,21 @@ namespace dsl { #endif struct Event { + /// @brief The file descriptor that this event is for fd_t fd; - int events; + /// @brief The events that have occurred on this file descriptor + event_t events; + /// @brief Returns true if the event is for the given event type operator bool() const { - return fd != -1; + return fd != INVALID_SOCKET; } }; using ThreadEventStore = dsl::store::ThreadStore; template - static inline void bind(const std::shared_ptr& reaction, fd_t fd, int watch_set) { + static inline void bind(const std::shared_ptr& reaction, fd_t fd, event_t watch_set) { reaction->unbinders.push_back([](const threading::Reaction& r) { r.reactor.emit(std::make_unique>(r.id)); diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index ab18a51c9..db7023c93 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -35,15 +35,15 @@ namespace extension { struct Task { Task() = default; // NOLINTNEXTLINE(google-runtime-int) - Task(const fd_t& fd, short events, std::shared_ptr reaction) + Task(const fd_t& fd, long events, std::shared_ptr reaction) : fd(fd), events(events), reaction(std::move(reaction)) {} /// @brief The socket we are waiting on fd_t fd; /// @brief The events that the task is interested in - short events{0}; // NOLINT(google-runtime-int) + long events{0}; // NOLINT(google-runtime-int) /// @brief The events that are waiting to be fired - short waiting_events{0}; // NOLINT(google-runtime-int) + long waiting_events{0}; // NOLINT(google-runtime-int) /// @brief The reaction that is waiting for this event std::shared_ptr reaction{nullptr}; }; From f7f0d1f6a45386fba65a2a23d62ec7d8794441c4 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 13:28:33 +1000 Subject: [PATCH 092/176] . --- src/extension/IOController_Windows.hpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index db7023c93..472afe550 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -119,8 +119,7 @@ namespace extension { // Go through every reaction and if it has events and isn't already running then run it for (auto it = tasks.begin(); it != tasks.end();) { - auto& event = it->first; - auto& task = it->second; + auto& task = it->second; if (task.reaction->active_tasks == 0 && task.waiting_events != 0) { @@ -237,7 +236,7 @@ namespace extension { // If we found it then clear the waiting events if (task != tasks.end()) { - task->waiting_events = 0; + task->second.waiting_events = 0; } }); @@ -278,16 +277,16 @@ namespace extension { } // Wait for events - auto event_index = WSAWaitForMultipleEvents(static_cast(events.size()), - events.data(), + auto event_index = WSAWaitForMultipleEvents(static_cast(watches.size()), + watches.data(), false, WSA_INFINITE, false); // Check if the return value is an event in our list - if (event_index >= WSA_WAIT_EVENT_0 && event_index < WSA_WAIT_EVENT_0 + events.size()) { + if (event_index >= WSA_WAIT_EVENT_0 && event_index < WSA_WAIT_EVENT_0 + watches.size()) { // Get the signalled event - auto& event = events[event_index - WSA_WAIT_EVENT_0]; + auto& event = watches[event_index - WSA_WAIT_EVENT_0]; // Collect the events that happened into the tasks list collect_events(event); From 3613ffd128ff5db4ab9a4b3a4d35d471be0bc482 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 13:32:19 +1000 Subject: [PATCH 093/176] . --- src/extension/IOController_Windows2.hpp | 267 ++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 src/extension/IOController_Windows2.hpp diff --git a/src/extension/IOController_Windows2.hpp b/src/extension/IOController_Windows2.hpp new file mode 100644 index 000000000..e148fa261 --- /dev/null +++ b/src/extension/IOController_Windows2.hpp @@ -0,0 +1,267 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2017 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_EXTENSION_IOCONTROLLER_WINDOWS_HPP +#define NUCLEAR_EXTENSION_IOCONTROLLER_WINDOWS_HPP + +#include "../PowerPlant.hpp" +#include "../Reactor.hpp" +#include "../dsl/word/IO.hpp" +#include "../util/platform.hpp" + +namespace NUClear { +namespace extension { + + class IOController : public Reactor { + private: + struct Event { + SOCKET fd; + std::shared_ptr reaction; + int events; + }; + + public: + explicit IOController(std::unique_ptr environment) : Reactor(std::move(environment)) { + + // Startup WSA for IO + WORD version = MAKEWORD(2, 2); + WSADATA wsa_data; + + int startup_status = WSAStartup(version, &wsa_data); + if (startup_status != 0) { + throw std::system_error(startup_status, std::system_category(), "WSAStartup() failed"); + } + + // Reserve 1024 event slots + // Hopefully we won't have more events than that + // Even if we do it should be fine (after a glitch) + events.reserve(1024); + + // Create an event to use for the notifier (used for getting out of WSAWaitForMultipleEvents()) + notifier = WSACreateEvent(); + if (notifier == WSA_INVALID_EVENT) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSACreateEvent() for notifier failed"); + } + + // We always have the notifier in the event list + events.push_back(notifier); + + on>().then( + "Configure IO Reaction", + [this](const dsl::word::IOConfiguration& config) { + // Lock our mutex + std::lock_guard lock(reaction_mutex); + + // Make an event for this SOCKET + auto event = WSACreateEvent(); + if (event == WSA_INVALID_EVENT) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSACreateEvent() for configure io reaction failed"); + } + + // Link the event to signal when there are events on the socket + if (WSAEventSelect(config.fd, event, config.events) == SOCKET_ERROR) { + throw std::system_error(WSAGetLastError(), std::system_category(), "WSAEventSelect() failed"); + } + + // Add all the information to the list and mark the list as dirty, to sync with the list of events + reactions.insert(std::make_pair(event, Event{config.fd, config.reaction, config.events})); + reactions_list_dirty = true; + + // Signal the notifier event to return from WSAWaitForMultipleEvents() and sync the dirty list + if (!WSASetEvent(notifier)) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSASetEvent() for configure io reaction failed"); + } + }); + + on>().then("IO Finished", [this](const dsl::word::IOFinished& event) { + // TODO find this reaction, and clear the revents that are in this reactions events request + }); + + on>>().then( + "Unbind IO Reaction", + [this](const dsl::operation::Unbind& unbind) { + // Lock our mutex + std::lock_guard lock(reaction_mutex); + + // Find this reaction in our list of reactions + auto reaction = std::find_if(std::begin(reactions), + std::end(reactions), + [&unbind](const std::pair& item) { + return item.second.reaction->id == unbind.id; + }); + + // If the reaction was found + if (reaction != std::end(reactions)) { + // Remove it from the list of reactions + reactions.erase(reaction); + + // Queue the associated event for closing when we sync + events_to_close.push_back(reaction->first); + } + else { + // Fail silently: we've unbound a reaction that somehow isn't in our list of reactions! + } + + // Flag that our list is dirty + reactions_list_dirty = true; + + // Signal the notifier event to return from WSAWaitForMultipleEvents() and sync the dirty list + if (!WSASetEvent(notifier)) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSASetEvent() for unbind io reaction failed"); + } + }); + + on().then("Shutdown IO Controller", [this] { + // Set shutdown to true + shutdown.store(true); + + // Signal the notifier event to return from WSAWaitForMultipleEvents() and shutdown + if (!WSASetEvent(notifier)) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSASetEvent() for shutdown failed"); + } + }); + + on().then("IO Controller", [this] { + if (!shutdown.load()) { + // Wait for events + auto event_index = WSAWaitForMultipleEvents(static_cast(events.size()), + events.data(), + false, + WSA_INFINITE, + false); + + // Check if the return value is an event in our list + if (event_index >= WSA_WAIT_EVENT_0 && event_index < WSA_WAIT_EVENT_0 + events.size()) { + // Get the signalled event + auto& event = events[event_index - WSA_WAIT_EVENT_0]; + + if (event == notifier) { + // Reset the notifier signal + if (!WSAResetEvent(event)) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSAResetEvent() for notifier failed"); + } + } + else { + // Get our associated Event object, which has the reaction + auto r = reactions.find(event); + + // If it was found... + if (r != reactions.end()) { + // Enum the socket events to work out which ones fired + WSANETWORKEVENTS wsae; + if (WSAEnumNetworkEvents(r->second.fd, event, &wsae) == SOCKET_ERROR) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSAEnumNetworkEvents() failed"); + } + + // Make our IO event to pass through + IO::Event io_event; + io_event.fd = r->second.fd; + + // The events that fired are what we got from the enum events call + io_event.events = wsae.lNetworkEvents; + + // Store the IO event in our thread local cache + IO::ThreadEventStore::value = &io_event; + + // Submit the task (which should run the get) + try { + auto task = r->second.reaction->get_task(); + if (task) { + powerplant.submit(std::move(task)); + } + } + catch (...) { + } + + // Reset our value + IO::ThreadEventStore::value = nullptr; + } + } + } + } + + if (reactions_list_dirty || !events_to_close.empty()) { + // Get the lock so we don't concurrently modify the list + std::lock_guard lock(reaction_mutex); + + // Close any events we've queued for closing + if (!events_to_close.empty()) { + for (auto& event : events_to_close) { + if (!WSACloseEvent(event)) { + throw std::system_error(WSAGetLastError(), + std::system_category(), + "WSACloseEvent() failed"); + } + } + + // Clear the queue of closed events + events_to_close.clear(); + } + + // Clear the list of events, to be rebuilt + events.resize(0); + + // Add back the notifier event + events.push_back(notifier); + + // Sync the list of reactions to the list of events + for (const auto& r : reactions) { + events.push_back(r.first); + } + + // The list has been synced + reactions_list_dirty = false; + } + }); + } + + // We need a destructor to cleanup WSA stuff + virtual ~IOController() { + WSACleanup(); + } + + private: + WSAEVENT notifier; + + std::atomic shutdown{false}; + bool reactions_list_dirty = false; + + std::mutex reaction_mutex; + std::map reactions; + std::vector events; + std::vector events_to_close; + }; + +} // namespace extension +} // namespace NUClear + +#endif // NUCLEAR_EXTENSION_IOCONTROLLER_WINDOWS_HPP From 7d381509581b0f41a5a6899cccdbda90342f36ab Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 13:37:07 +1000 Subject: [PATCH 094/176] . --- src/extension/IOController_Windows.hpp | 8 +- src/extension/IOController_Windows2.hpp | 267 ------------------------ 2 files changed, 4 insertions(+), 271 deletions(-) delete mode 100644 src/extension/IOController_Windows2.hpp diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index 472afe550..b2f7bff94 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -230,8 +230,8 @@ namespace extension { const std::lock_guard lock(tasks_mutex); // Find the reaction that finished processing - auto task = std::find_if(tasks.begin(), tasks.end(), [&event](const Task& t) { - return t.reaction->id == event.id; + auto task = std::find_if(tasks.begin(), tasks.end(), [&event](const std::pair& t) { + return t.second.reaction->id == event.id; }); // If we found it then clear the waiting events @@ -247,8 +247,8 @@ namespace extension { const std::lock_guard lock(tasks_mutex); // Find our reaction - auto it = std::find_if(tasks.begin(), tasks.end(), [&unbind](const Task& t) { - return t.reaction->id == unbind.id; + auto it = std::find_if(tasks.begin(), tasks.end(), [&unbind](const std::pair& t) { + return t.second.reaction->id == unbind.id; }); if (it != tasks.end()) { diff --git a/src/extension/IOController_Windows2.hpp b/src/extension/IOController_Windows2.hpp deleted file mode 100644 index e148fa261..000000000 --- a/src/extension/IOController_Windows2.hpp +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef NUCLEAR_EXTENSION_IOCONTROLLER_WINDOWS_HPP -#define NUCLEAR_EXTENSION_IOCONTROLLER_WINDOWS_HPP - -#include "../PowerPlant.hpp" -#include "../Reactor.hpp" -#include "../dsl/word/IO.hpp" -#include "../util/platform.hpp" - -namespace NUClear { -namespace extension { - - class IOController : public Reactor { - private: - struct Event { - SOCKET fd; - std::shared_ptr reaction; - int events; - }; - - public: - explicit IOController(std::unique_ptr environment) : Reactor(std::move(environment)) { - - // Startup WSA for IO - WORD version = MAKEWORD(2, 2); - WSADATA wsa_data; - - int startup_status = WSAStartup(version, &wsa_data); - if (startup_status != 0) { - throw std::system_error(startup_status, std::system_category(), "WSAStartup() failed"); - } - - // Reserve 1024 event slots - // Hopefully we won't have more events than that - // Even if we do it should be fine (after a glitch) - events.reserve(1024); - - // Create an event to use for the notifier (used for getting out of WSAWaitForMultipleEvents()) - notifier = WSACreateEvent(); - if (notifier == WSA_INVALID_EVENT) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSACreateEvent() for notifier failed"); - } - - // We always have the notifier in the event list - events.push_back(notifier); - - on>().then( - "Configure IO Reaction", - [this](const dsl::word::IOConfiguration& config) { - // Lock our mutex - std::lock_guard lock(reaction_mutex); - - // Make an event for this SOCKET - auto event = WSACreateEvent(); - if (event == WSA_INVALID_EVENT) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSACreateEvent() for configure io reaction failed"); - } - - // Link the event to signal when there are events on the socket - if (WSAEventSelect(config.fd, event, config.events) == SOCKET_ERROR) { - throw std::system_error(WSAGetLastError(), std::system_category(), "WSAEventSelect() failed"); - } - - // Add all the information to the list and mark the list as dirty, to sync with the list of events - reactions.insert(std::make_pair(event, Event{config.fd, config.reaction, config.events})); - reactions_list_dirty = true; - - // Signal the notifier event to return from WSAWaitForMultipleEvents() and sync the dirty list - if (!WSASetEvent(notifier)) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSASetEvent() for configure io reaction failed"); - } - }); - - on>().then("IO Finished", [this](const dsl::word::IOFinished& event) { - // TODO find this reaction, and clear the revents that are in this reactions events request - }); - - on>>().then( - "Unbind IO Reaction", - [this](const dsl::operation::Unbind& unbind) { - // Lock our mutex - std::lock_guard lock(reaction_mutex); - - // Find this reaction in our list of reactions - auto reaction = std::find_if(std::begin(reactions), - std::end(reactions), - [&unbind](const std::pair& item) { - return item.second.reaction->id == unbind.id; - }); - - // If the reaction was found - if (reaction != std::end(reactions)) { - // Remove it from the list of reactions - reactions.erase(reaction); - - // Queue the associated event for closing when we sync - events_to_close.push_back(reaction->first); - } - else { - // Fail silently: we've unbound a reaction that somehow isn't in our list of reactions! - } - - // Flag that our list is dirty - reactions_list_dirty = true; - - // Signal the notifier event to return from WSAWaitForMultipleEvents() and sync the dirty list - if (!WSASetEvent(notifier)) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSASetEvent() for unbind io reaction failed"); - } - }); - - on().then("Shutdown IO Controller", [this] { - // Set shutdown to true - shutdown.store(true); - - // Signal the notifier event to return from WSAWaitForMultipleEvents() and shutdown - if (!WSASetEvent(notifier)) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSASetEvent() for shutdown failed"); - } - }); - - on().then("IO Controller", [this] { - if (!shutdown.load()) { - // Wait for events - auto event_index = WSAWaitForMultipleEvents(static_cast(events.size()), - events.data(), - false, - WSA_INFINITE, - false); - - // Check if the return value is an event in our list - if (event_index >= WSA_WAIT_EVENT_0 && event_index < WSA_WAIT_EVENT_0 + events.size()) { - // Get the signalled event - auto& event = events[event_index - WSA_WAIT_EVENT_0]; - - if (event == notifier) { - // Reset the notifier signal - if (!WSAResetEvent(event)) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSAResetEvent() for notifier failed"); - } - } - else { - // Get our associated Event object, which has the reaction - auto r = reactions.find(event); - - // If it was found... - if (r != reactions.end()) { - // Enum the socket events to work out which ones fired - WSANETWORKEVENTS wsae; - if (WSAEnumNetworkEvents(r->second.fd, event, &wsae) == SOCKET_ERROR) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSAEnumNetworkEvents() failed"); - } - - // Make our IO event to pass through - IO::Event io_event; - io_event.fd = r->second.fd; - - // The events that fired are what we got from the enum events call - io_event.events = wsae.lNetworkEvents; - - // Store the IO event in our thread local cache - IO::ThreadEventStore::value = &io_event; - - // Submit the task (which should run the get) - try { - auto task = r->second.reaction->get_task(); - if (task) { - powerplant.submit(std::move(task)); - } - } - catch (...) { - } - - // Reset our value - IO::ThreadEventStore::value = nullptr; - } - } - } - } - - if (reactions_list_dirty || !events_to_close.empty()) { - // Get the lock so we don't concurrently modify the list - std::lock_guard lock(reaction_mutex); - - // Close any events we've queued for closing - if (!events_to_close.empty()) { - for (auto& event : events_to_close) { - if (!WSACloseEvent(event)) { - throw std::system_error(WSAGetLastError(), - std::system_category(), - "WSACloseEvent() failed"); - } - } - - // Clear the queue of closed events - events_to_close.clear(); - } - - // Clear the list of events, to be rebuilt - events.resize(0); - - // Add back the notifier event - events.push_back(notifier); - - // Sync the list of reactions to the list of events - for (const auto& r : reactions) { - events.push_back(r.first); - } - - // The list has been synced - reactions_list_dirty = false; - } - }); - } - - // We need a destructor to cleanup WSA stuff - virtual ~IOController() { - WSACleanup(); - } - - private: - WSAEVENT notifier; - - std::atomic shutdown{false}; - bool reactions_list_dirty = false; - - std::mutex reaction_mutex; - std::map reactions; - std::vector events; - std::vector events_to_close; - }; - -} // namespace extension -} // namespace NUClear - -#endif // NUCLEAR_EXTENSION_IOCONTROLLER_WINDOWS_HPP From 6793e0c3435fcd42dc40977cdfb76f708192dc58 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 13:44:38 +1000 Subject: [PATCH 095/176] . --- src/dsl/word/IO.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dsl/word/IO.hpp b/src/dsl/word/IO.hpp index 961bb8bf4..bd5079d79 100644 --- a/src/dsl/word/IO.hpp +++ b/src/dsl/word/IO.hpp @@ -32,9 +32,9 @@ namespace dsl { namespace word { #ifdef _WIN32 - using event_t = long; + using event_t = long; // NOLINT(google-runtime-int) #else - using event_t = short; + using event_t = short; // NOLINT(google-runtime-int) #endif /** From 6ff441f768cf5bb2a2dcb6d37d1890038ece4697 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 14:03:17 +1000 Subject: [PATCH 096/176] Try the same bytes trick on windows --- src/extension/IOController_Windows.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index b2f7bff94..d260016b6 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -102,6 +102,16 @@ namespace extension { "WSAEnumNetworkEvents() failed"); } + // Check how many bytes are available to read, if it's 0 and we have a read event the + // descriptor is sending EOF and we should fire a CLOSE event too and stop watching + if ((wsae.lNetworkEvents & IO::READ) != 0) { + int bytes_available = 0; + bool valid = ::ioctlsocket(r->second.fd, FIONREAD, &bytes_available) == 0; + if (valid && bytes_available == 0) { + wsae.lNetworkEvents |= IO::CLOSE; + } + } + r->second.waiting_events = r->second.waiting_events | wsae.lNetworkEvents; } // If we can't find the event then our list is dirty From fb9c8443e419dbe13bd09b592592199b07e84dab Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 14:07:14 +1000 Subject: [PATCH 097/176] Windows loves being different --- src/extension/IOController_Windows.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index d260016b6..9b8b92617 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -105,8 +105,8 @@ namespace extension { // Check how many bytes are available to read, if it's 0 and we have a read event the // descriptor is sending EOF and we should fire a CLOSE event too and stop watching if ((wsae.lNetworkEvents & IO::READ) != 0) { - int bytes_available = 0; - bool valid = ::ioctlsocket(r->second.fd, FIONREAD, &bytes_available) == 0; + u_long bytes_available = 0; + bool valid = ::ioctlsocket(r->second.fd, FIONREAD, &bytes_available) == 0; if (valid && bytes_available == 0) { wsae.lNetworkEvents |= IO::CLOSE; } From 907d5d7dbb60ed11f7a2a410ef540c44c3f48296 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 14:25:05 +1000 Subject: [PATCH 098/176] Doubt this makes a difference --- src/extension/IOController_Windows.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index 9b8b92617..a0edc3beb 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -104,15 +104,16 @@ namespace extension { // Check how many bytes are available to read, if it's 0 and we have a read event the // descriptor is sending EOF and we should fire a CLOSE event too and stop watching - if ((wsae.lNetworkEvents & IO::READ) != 0) { + long events = wsae.lNetworkEvents; + if ((events & IO::READ) != 0) { u_long bytes_available = 0; bool valid = ::ioctlsocket(r->second.fd, FIONREAD, &bytes_available) == 0; if (valid && bytes_available == 0) { - wsae.lNetworkEvents |= IO::CLOSE; + events = events | IO::CLOSE; } } - r->second.waiting_events = r->second.waiting_events | wsae.lNetworkEvents; + r->second.waiting_events = r->second.waiting_events | events; } // If we can't find the event then our list is dirty else { From 9cd60164ec277586752409028be74a28532ef0f7 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 14:25:24 +1000 Subject: [PATCH 099/176] Make it clearer that it's a bitwise or --- src/extension/IOController_Posix.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index 33a1e74b8..9f0d29a7f 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -132,7 +132,7 @@ namespace extension { int bytes_available = 0; bool valid = ::ioctl(fd.fd, FIONREAD, &bytes_available) == 0; if (valid && bytes_available == 0) { - fd.revents |= IO::CLOSE; + fd.revents = fd.revents | IO::CLOSE; } } From 38144fcbc3c3ec01937a4f5dcc5783dbd0e3007b Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 15:18:29 +1000 Subject: [PATCH 100/176] Try disabling the eof detector again? --- src/extension/IOController_Windows.hpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index a0edc3beb..2eb975b25 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -105,13 +105,13 @@ namespace extension { // Check how many bytes are available to read, if it's 0 and we have a read event the // descriptor is sending EOF and we should fire a CLOSE event too and stop watching long events = wsae.lNetworkEvents; - if ((events & IO::READ) != 0) { - u_long bytes_available = 0; - bool valid = ::ioctlsocket(r->second.fd, FIONREAD, &bytes_available) == 0; - if (valid && bytes_available == 0) { - events = events | IO::CLOSE; - } - } + // if ((events & IO::READ) != 0) { + // u_long bytes_available = 0; + // bool valid = ::ioctlsocket(r->second.fd, FIONREAD, &bytes_available) == 0; + // if (valid && bytes_available == 0) { + // events = events | IO::CLOSE; + // } + // } r->second.waiting_events = r->second.waiting_events | events; } @@ -178,12 +178,15 @@ namespace extension { std::map::iterator remove_task(std::map::iterator it) { // Close the event - if (!WSACloseEvent(it->first)) { - throw std::system_error(WSAGetLastError(), std::system_category(), "WSACloseEvent() failed"); - } + WSAEVENT event = it->first; // Remove the task return tasks.erase(it); + + // Try to close the WSA event + if (!WSACloseEvent(it->first)) { + throw std::system_error(WSAGetLastError(), std::system_category(), "WSACloseEvent() failed"); + } } @@ -314,7 +317,6 @@ namespace extension { WSACleanup(); } - private: /// @brief The event that is used to wake up the WaitForMultipleEvents call WSAEVENT notifier; From d6efc4925c05c9a6e13be98c27eb71634e62d62e Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 15:21:11 +1000 Subject: [PATCH 101/176] . --- src/extension/IOController_Windows.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index 2eb975b25..d1c13f01e 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -184,7 +184,7 @@ namespace extension { return tasks.erase(it); // Try to close the WSA event - if (!WSACloseEvent(it->first)) { + if (!WSACloseEvent(event)) { throw std::system_error(WSAGetLastError(), std::system_category(), "WSACloseEvent() failed"); } } From f4351c7013ba5ae81098769dcfc6cf39e603606d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 16:09:06 +1000 Subject: [PATCH 102/176] Add timeouts to UDP --- tests/dsl/UDP.cpp | 11 +++++++---- tests/dsl/UDPBroadcast.cpp | 6 ++++-- tests/dsl/UDPMulticastKnownPort.cpp | 6 ++++-- tests/dsl/UDPMulticastUnknownPort.cpp | 6 ++++-- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index dd42a8875..f5b1c9216 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -19,6 +19,8 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { constexpr uint16_t PORT = 40000; @@ -28,9 +30,9 @@ bool received_b = false; // NOLINT(cppcoreguidelines struct Message {}; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { // Known port on(PORT).then([this](const UDP::Packet& packet) { @@ -64,8 +66,9 @@ class TestReactor : public NUClear::Reactor { // Send a test for a known port // does not need to include the port in the lambda capture. This is a global variable to the unit test, so // the function will have access to it. - on>().then( - [this] { emit(std::make_unique(TEST_STRING), INADDR_LOOPBACK, PORT); }); + on>().then([this] { // + emit(std::make_unique(TEST_STRING), INADDR_LOOPBACK, PORT); + }); // Send a test for an unknown port diff --git a/tests/dsl/UDPBroadcast.cpp b/tests/dsl/UDPBroadcast.cpp index 4b29e9744..2c960c1ba 100644 --- a/tests/dsl/UDPBroadcast.cpp +++ b/tests/dsl/UDPBroadcast.cpp @@ -19,6 +19,8 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { constexpr uint16_t PORT = 40001; @@ -29,10 +31,10 @@ std::size_t num_addresses = 0; // NOLINT(cppcoreguidelines-avoid-non-const- struct Message {}; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: in_port_t bound_port = 0; - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { // Known port on(PORT).then([this](const UDP::Packet& packet) { diff --git a/tests/dsl/UDPMulticastKnownPort.cpp b/tests/dsl/UDPMulticastKnownPort.cpp index b31983594..7fbf82c6a 100644 --- a/tests/dsl/UDPMulticastKnownPort.cpp +++ b/tests/dsl/UDPMulticastKnownPort.cpp @@ -19,6 +19,8 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { constexpr in_port_t PORT = 40002; @@ -29,11 +31,11 @@ std::size_t num_addresses = 0; // NOLINT(cppcoreguidelines-avoid-non- struct Message {}; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: bool shutdown_flag = false; - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { // Terminates the test if it takes too long - longer than 200 ms since this reaction first runs on>().then([this]() { diff --git a/tests/dsl/UDPMulticastUnknownPort.cpp b/tests/dsl/UDPMulticastUnknownPort.cpp index 29defc5c7..505365b34 100644 --- a/tests/dsl/UDPMulticastUnknownPort.cpp +++ b/tests/dsl/UDPMulticastUnknownPort.cpp @@ -19,6 +19,8 @@ #include #include +#include "test_util/TestBase.hpp" + namespace { const std::string TEST_STRING = "Hello UDP Multicast World!"; // NOLINT(cert-err58-cpp) @@ -28,11 +30,11 @@ std::size_t num_addresses = 0; // NOLINT(cppcoreguidelines-avoid-non- struct Message {}; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: bool shutdown_flag = false; - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { // Terminates the test if it takes too long - longer than 200 ms since this reaction first runs on>().then([this]() { From 6cd5a3eb34be20cf677d13615575f7f18e1f5ef6 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 16:21:52 +1000 Subject: [PATCH 103/176] Update min cmake version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5b3686b0..09a3fb642 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -cmake_minimum_required(VERSION 3.1.0) +cmake_minimum_required(VERSION 3.15.0) # We use additional modules that cmake needs to know about set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") From 0261a49abe1759caea2f4f3f2e43e6705b6b4718 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 16:44:18 +1000 Subject: [PATCH 104/176] This works fine locally --- src/extension/IOController_Windows.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index d1c13f01e..8521dc502 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -105,13 +105,13 @@ namespace extension { // Check how many bytes are available to read, if it's 0 and we have a read event the // descriptor is sending EOF and we should fire a CLOSE event too and stop watching long events = wsae.lNetworkEvents; - // if ((events & IO::READ) != 0) { - // u_long bytes_available = 0; - // bool valid = ::ioctlsocket(r->second.fd, FIONREAD, &bytes_available) == 0; - // if (valid && bytes_available == 0) { - // events = events | IO::CLOSE; - // } - // } + if ((events & IO::READ) != 0) { + u_long bytes_available = 0; + bool valid = ::ioctlsocket(r->second.fd, FIONREAD, &bytes_available) == 0; + if (valid && bytes_available == 0) { + events = events | IO::CLOSE; + } + } r->second.waiting_events = r->second.waiting_events | events; } From 3c46aa953a59514ac1fd87a1627f67b55eb1fac8 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 16:29:07 +1000 Subject: [PATCH 105/176] clang-tidy --- src/extension/IOController_Posix.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index 9f0d29a7f..8f098c9e9 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -132,7 +132,8 @@ namespace extension { int bytes_available = 0; bool valid = ::ioctl(fd.fd, FIONREAD, &bytes_available) == 0; if (valid && bytes_available == 0) { - fd.revents = fd.revents | IO::CLOSE; + // NOLINTNEXTLINE(google-runtime-int) + fd.revents = short(fd.revents | IO::CLOSE); } } From db1075db3a7ec63deb5b49d1eef6cf82a1b1d5ba Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 11 Aug 2023 17:05:35 +1000 Subject: [PATCH 106/176] Add timeouts to emit tests (need to refactor these) --- tests/dsl/emit/Delay.cpp | 6 ++++-- tests/dsl/emit/EmitFusion.cpp | 6 ++++-- tests/dsl/emit/Initialise.cpp | 6 ++++-- tests/dsl/emit/UDP.cpp | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/dsl/emit/Delay.cpp b/tests/dsl/emit/Delay.cpp index f3cba12c0..0e4994950 100644 --- a/tests/dsl/emit/Delay.cpp +++ b/tests/dsl/emit/Delay.cpp @@ -19,6 +19,8 @@ #include #include +#include "../../test_util/TestBase.hpp" + // Anonymous namespace to keep everything file local namespace { @@ -35,9 +37,9 @@ NUClear::clock::time_point delay_received; // NOLINTNEXTLINE(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) NUClear::clock::time_point at_time_received; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { emit(std::make_unique(5)); // This message should come in later diff --git a/tests/dsl/emit/EmitFusion.cpp b/tests/dsl/emit/EmitFusion.cpp index 8af9ee1ed..da23c530b 100644 --- a/tests/dsl/emit/EmitFusion.cpp +++ b/tests/dsl/emit/EmitFusion.cpp @@ -20,6 +20,8 @@ #include #include +#include "../../test_util/TestBase.hpp" + // Anonymous namespace to keep everything file local namespace { @@ -53,9 +55,9 @@ struct EmitTester2 { } }; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { // Make some things to emit auto t1 = std::make_unique(8); diff --git a/tests/dsl/emit/Initialise.cpp b/tests/dsl/emit/Initialise.cpp index ffe9333a8..9a4d46cd1 100644 --- a/tests/dsl/emit/Initialise.cpp +++ b/tests/dsl/emit/Initialise.cpp @@ -19,14 +19,16 @@ #include #include +#include "../../test_util/TestBase.hpp" + // Anonymous namespace to keep everything file local namespace { struct ShutdownNowPlx {}; -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { emit(std::make_unique(5)); on>().then([this](const int& v) { diff --git a/tests/dsl/emit/UDP.cpp b/tests/dsl/emit/UDP.cpp index b8d94f81f..df4e09f12 100644 --- a/tests/dsl/emit/UDP.cpp +++ b/tests/dsl/emit/UDP.cpp @@ -19,16 +19,18 @@ #include #include +#include "../../test_util/TestBase.hpp" + // Anonymous namespace to keep everything file local namespace { int received_messages = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -class TestReactor : public NUClear::Reactor { +class TestReactor : public test_util::TestBase { public: in_port_t bound_port = 0; - TestReactor(std::unique_ptr environment) : Reactor(std::move(environment)) { + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { emit(std::make_unique(5)); std::tie(std::ignore, bound_port, std::ignore) = on().then([this](const UDP::Packet& packet) { From 8dad0a40e6da4771b7b41ad4377bff70cca2a38d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 14 Aug 2023 10:30:47 +1000 Subject: [PATCH 107/176] Update the delay test --- tests/dsl/emit/Delay.cpp | 109 +++++++++++++++++++++++++++------------ 1 file changed, 75 insertions(+), 34 deletions(-) diff --git a/tests/dsl/emit/Delay.cpp b/tests/dsl/emit/Delay.cpp index 0e4994950..952dd80e7 100644 --- a/tests/dsl/emit/Delay.cpp +++ b/tests/dsl/emit/Delay.cpp @@ -24,46 +24,74 @@ // Anonymous namespace to keep everything file local namespace { -struct DelayMessage {}; -struct AtTimeMessage {}; -struct NormalMessage {}; - -// NOLINTNEXTLINE(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) -NUClear::clock::time_point sent; -// NOLINTNEXTLINE(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) -NUClear::clock::time_point normal_received; -// NOLINTNEXTLINE(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) -NUClear::clock::time_point delay_received; -// NOLINTNEXTLINE(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) -NUClear::clock::time_point at_time_received; +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +/// @brief Test units are the time units the test is performed in +using TestUnits = std::chrono::duration>; +/// @brief Perform this many different time points for the test +constexpr int test_loops = 5; + +struct DelayedMessage { + DelayedMessage(const NUClear::clock::duration& delay) : time(NUClear::clock::now()), delay(delay) {} + NUClear::clock::time_point time; + NUClear::clock::duration delay; +}; + +struct TargetTimeMessage { + TargetTimeMessage(const NUClear::clock::time_point& target) : time(NUClear::clock::now()), target(target) {} + NUClear::clock::time_point time; + NUClear::clock::time_point target; +}; + +struct FinishTest {}; class TestReactor : public test_util::TestBase { public: - TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - emit(std::make_unique(5)); + TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { - // This message should come in later - on>().then([this] { - delay_received = NUClear::clock::now(); + // Measure when messages were sent and received and print those values + on>().then([](const DelayedMessage& m) { + auto true_delta = std::chrono::duration_cast(NUClear::clock::now() - m.time); + auto delta = std::chrono::duration_cast(m.delay); - powerplant.shutdown(); + // Print the debug message + events.push_back("delayed " + std::to_string(true_delta.count()) + " received " + + std::to_string(delta.count())); }); - on>().then([] { - // Don't shut down here we are first - at_time_received = NUClear::clock::now(); + on>().then([](const TargetTimeMessage& m) { + auto true_delta = std::chrono::duration_cast(NUClear::clock::now() - m.time); + auto delta = std::chrono::duration_cast(m.target - m.time); + + // Print the debug message + events.push_back("at_time " + std::to_string(true_delta.count()) + " received " + + std::to_string(delta.count())); + }); + + on>().then([this] { + events.push_back("Finished"); + powerplant.shutdown(); }); - on>().then([] { normal_received = NUClear::clock::now(); }); on().then([this] { - sent = NUClear::clock::now(); - emit(std::make_unique()); + // Get our jump size in milliseconds + int jump_unit = (TestUnits::period::num * 1000) / TestUnits::period::den; + // Delay with consistent jumps + for (int i = 0; i < test_loops; ++i) { + auto delay = std::chrono::milliseconds(jump_unit * i); + emit(std::make_unique(delay), delay); + } + + // Target time with consistent jumps that interleave the first set + for (int i = 0; i < test_loops; ++i) { + auto target = NUClear::clock::now() + std::chrono::milliseconds(jump_unit / 2 + jump_unit * i); + emit(std::make_unique(target), target); + } - // Delay by 200, and a message 100ms in the future, the 200ms one should come in first - emit(std::make_unique(), std::chrono::milliseconds(200)); - emit(std::make_unique(), - NUClear::clock::now() + std::chrono::milliseconds(100)); + // Emit a shutdown one time unit after + emit(std::make_unique(), std::chrono::milliseconds(jump_unit * (test_loops + 1))); }); } }; @@ -74,12 +102,25 @@ TEST_CASE("Testing the delay emit", "[api][emit][delay]") { config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); - // Ensure the message delays are correct, I would make these bounds tighter, but travis is pretty dumb - REQUIRE(std::chrono::duration_cast(delay_received - sent).count() > 190); - REQUIRE(std::chrono::duration_cast(delay_received - sent).count() < 225); - REQUIRE(std::chrono::duration_cast(at_time_received - sent).count() > 90); - REQUIRE(std::chrono::duration_cast(at_time_received - sent).count() < 120); + const std::vector expected = { + "delayed 0 received 0", + "at_time 0 received 0", + "delayed 1 received 1", + "at_time 1 received 1", + "delayed 2 received 2", + "at_time 2 received 2", + "delayed 3 received 3", + "at_time 3 received 3", + "delayed 4 received 4", + "at_time 4 received 4", + "Finished", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From 636b86acfffc3b783605ec520b5410bb881894db Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 14 Aug 2023 11:21:34 +1000 Subject: [PATCH 108/176] Update emit fusion test --- tests/dsl/emit/EmitFusion.cpp | 126 ++++++++++++++-------------------- 1 file changed, 51 insertions(+), 75 deletions(-) diff --git a/tests/dsl/emit/EmitFusion.cpp b/tests/dsl/emit/EmitFusion.cpp index da23c530b..95e3c2f8f 100644 --- a/tests/dsl/emit/EmitFusion.cpp +++ b/tests/dsl/emit/EmitFusion.cpp @@ -25,33 +25,28 @@ // Anonymous namespace to keep everything file local namespace { -int v1 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int v2 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int v3 = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -int stored_a = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -double stored_c = 0.0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -double stored_d = 0.0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -std::string stored_b; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) template -struct EmitTester1 { +struct E1 { static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, int a, std::string b) { - v1 = *p; - stored_a = a; - stored_b = std::move(b); + events.push_back("E1a " + *p + " " + std::to_string(a) + " " + b); } - static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, double c) { - v2 = *p; - stored_c = c; + static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, std::string c) { + events.push_back("E1b " + *p + " " + c); } }; template -struct EmitTester2 { - static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, double d) { - v3 = *p; - stored_d = d; +struct E2 { + static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, bool d) { + events.push_back("E2a " + *p + " " + (d ? "true" : "false")); + } + + static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, int e, std::string f) { + events.push_back("E2b " + *p + " " + std::to_string(e) + " " + f); } }; @@ -59,71 +54,52 @@ class TestReactor : public test_util::TestBase { public: TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - // Make some things to emit - auto t1 = std::make_unique(8); - auto t2 = std::make_unique(10); - auto t3 = std::make_unique(52); - auto t4 = std::make_unique(100); - - // Test using the second overload - emit(t1, 7.2); - REQUIRE(v1 == 0); - REQUIRE(v2 == 8); - v2 = 0; - REQUIRE(v3 == 0); - REQUIRE(stored_a == 0); - REQUIRE(stored_b.empty()); - REQUIRE(stored_c == 7.2); - stored_c = 0; - REQUIRE(stored_d == 0); - - // Test using the first overload - emit(t2, 1337, "This is text"); - REQUIRE(v1 == 10); - v1 = 0; - REQUIRE(v2 == 0); - REQUIRE(v3 == 0); - REQUIRE(stored_a == 1337); - stored_a = 0; - REQUIRE(stored_b == "This is text"); - stored_b = ""; - REQUIRE(stored_c == 0); - REQUIRE(stored_d == 0); - - // Test multiple functions - emit(t3, 15, 8.3); - REQUIRE(v1 == 0); - REQUIRE(v2 == 52); - v2 = 0; - REQUIRE(v3 == 52); - v3 = 0; - REQUIRE(stored_a == 0); - REQUIRE(stored_b.empty()); - REQUIRE(stored_c == 15); - stored_c = 0; - REQUIRE(stored_d == 8.3); - stored_d = 0; - - // Test even more multiple functions - emit(t4, 2, "Hello World", 9.2, 5); - REQUIRE(v1 == 100); - REQUIRE(v2 == 100); - REQUIRE(v3 == 100); - REQUIRE(stored_a == 2); - REQUIRE(stored_b == "Hello World"); - REQUIRE(stored_c == 5); - REQUIRE(stored_d == 9.2); - - on().then([this] { powerplant.shutdown(); }); + // Emit some messages + emit(std::make_unique("message1"), "test1"); // 1b + events.push_back("End test 1"); + + emit(std::make_unique("message2"), 1337, "test2"); // 1a + events.push_back("End test 2"); + + emit(std::make_unique("message3"), 15, "test3", true); // 1a, 2a + events.push_back("End test 3"); + + emit(std::make_unique("message4"), 2, "Hello World", false, "test4"); // 1a, 2a, 1b + events.push_back("End test 4"); + + emit(std::make_unique("message5"), 5, "test5a", 10, "test5b"); // 1a, 2b + events.push_back("End test 5"); } }; } // namespace TEST_CASE("Testing emit function fusion", "[api][emit][fusion]") { + NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + const std::vector expected = { + "E1b message1 test1", + "End test 1", + "E1a message2 1337 test2", + "End test 2", + "E1a message3 15 test3", + "E2a message3 true", + "End test 3", + "E1a message4 2 Hello World", + "E2a message4 false", + "E1b message4 test4", + "End test 4", + "E1a message5 5 test5a", + "E2b message5 10 test5b", + "End test 5", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From a0c9517f811c154909c116ae1f311ceed6270424 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 15 Aug 2023 16:03:10 +1000 Subject: [PATCH 109/176] Refactor the task scheduler to take generic tasks and use it to update initialise --- src/PowerPlant.cpp | 21 +++++- src/PowerPlant.hpp | 19 +++++- src/PowerPlant.ipp | 4 +- src/dsl/word/Always.hpp | 32 ++------- src/dsl/word/Every.hpp | 18 +---- src/dsl/word/Watchdog.hpp | 11 +--- src/dsl/word/emit/Direct.hpp | 21 +----- src/dsl/word/emit/Initialise.hpp | 15 ++++- src/dsl/word/emit/Local.hpp | 22 ++----- src/extension/ChronoController.hpp | 2 +- src/extension/IOController_Posix.hpp | 14 +--- src/extension/IOController_Windows.hpp | 14 +--- src/extension/NetworkController.hpp | 5 +- src/threading/ReactionTask.hpp | 21 ------ src/threading/TaskScheduler.cpp | 66 +++++++++++-------- src/threading/TaskScheduler.hpp | 91 ++++++++++++++++++-------- tests/dsl/emit/Initialise.cpp | 47 +++++++++---- 17 files changed, 216 insertions(+), 207 deletions(-) diff --git a/src/PowerPlant.cpp b/src/PowerPlant.cpp index 961c04d50..b437281f9 100644 --- a/src/PowerPlant.cpp +++ b/src/PowerPlant.cpp @@ -45,8 +45,25 @@ void PowerPlant::start() { scheduler.start(); } -void PowerPlant::submit(std::unique_ptr&& task, const bool& immediate) { - scheduler.submit(std::move(task), immediate); +void PowerPlant::submit(const uint64_t& id, + const int& priority, + const util::GroupDescriptor& group, + const util::ThreadPoolDescriptor& pool, + const bool& immediate, + std::function&& task) { + scheduler.submit(id, priority, group, pool, immediate, std::move(task)); +} + +void PowerPlant::submit(std::unique_ptr&& task, const bool& immediate) noexcept { + // Only submit non null tasks + if (task) { + try { + std::shared_ptr t(std::move(task)); + submit(t->id, t->priority, t->group_descriptor, t->thread_pool_descriptor, immediate, [t]() { t->run(); }); + } + catch (...) { + } + } } void PowerPlant::shutdown() { diff --git a/src/PowerPlant.hpp b/src/PowerPlant.hpp index d7b17b85a..3c2a670e1 100644 --- a/src/PowerPlant.hpp +++ b/src/PowerPlant.hpp @@ -141,13 +141,30 @@ class PowerPlant { template void install(); + /** + * @brief Generic submit function for submitting tasks to the thread pool. + * + * @param id an id for ordering the task + * @param priority the priority of the task between 0 and 1000 + * @param group the details of the execution group this task will run in + * @param pool the details of the thread pool this task will run from + * @param immediate if this task should run immediately in the current thread + * @param task the wrapped function to be executed + */ + void submit(const uint64_t& id, + const int& priority, + const util::GroupDescriptor& group, + const util::ThreadPoolDescriptor& pool, + const bool& immediate, + std::function&& task); + /** * @brief Submits a new task to the ThreadPool to be queued and then executed. * * @param task The Reaction task to be executed in the thread pool * @param immediate if this task should run immediately in the current thread */ - void submit(std::unique_ptr&& task, const bool& immediate = false); + void submit(std::unique_ptr&& task, const bool& immediate = false) noexcept; /** * @brief Log a message through NUClear's system. diff --git a/src/PowerPlant.ipp b/src/PowerPlant.ipp index b9ded1e4f..55dccd748 100644 --- a/src/PowerPlant.ipp +++ b/src/PowerPlant.ipp @@ -45,7 +45,7 @@ inline PowerPlant::PowerPlant(Configuration config, int argc, const char* argv[] args.emplace_back(argv[i]); } - // We emit this twice, so the data is available for extensions, it will also be emitted in start + // Emit our command line arguments emit(std::make_unique(args)); } @@ -55,7 +55,7 @@ void PowerPlant::install() { // Make sure that the class that we received is a reactor static_assert(std::is_base_of::value, "You must install Reactors"); - // Get the demangled reactor name and strip `struct ` and `class ` from it + // Get the demangled reactor name std::string reactor_name = util::demangle(typeid(T).name()); // The reactor constructor should handle subscribing to events diff --git a/src/dsl/word/Always.hpp b/src/dsl/word/Always.hpp index 3dd9dc4d5..39aad3c4f 100644 --- a/src/dsl/word/Always.hpp +++ b/src/dsl/word/Always.hpp @@ -112,23 +112,10 @@ namespace dsl { [always_reaction](threading::Reaction& idle_reaction) -> util::GeneratedCallback { auto callback = [&idle_reaction, always_reaction](threading::ReactionTask& /*task*/) { // Get a task for the always reaction and submit it to the scheduler - auto always_task = always_reaction->get_task(); - if (always_task) { - always_reaction->reactor.powerplant.submit(std::move(always_task)); - } + always_reaction->reactor.powerplant.submit(always_reaction->get_task()); // Get a task for the idle reaction and submit it to the scheduler - auto idle_task = idle_reaction.get_task(); - if (idle_task) { - // Set the thread pool on the task - idle_task->thread_pool_descriptor = DSL::pool(*always_reaction); - - // Make sure that idle reaction always has lower priority than the always reaction - idle_task->priority = DSL::priority(*always_reaction) - 1; - - // Submit the task to be run - idle_reaction.reactor.powerplant.submit(std::move(idle_task)); - } + idle_reaction.reactor.powerplant.submit(idle_reaction.get_task()); }; // Make sure that idle reaction always has lower priority than the always reaction @@ -152,25 +139,16 @@ namespace dsl { }); // Get a task for the always reaction and submit it to the scheduler - auto always_task = always_reaction->get_task(); - if (always_task) { - always_reaction->reactor.powerplant.submit(std::move(always_task)); - } + always_reaction->reactor.powerplant.submit(always_reaction->get_task()); // Get a task for the idle reaction and submit it to the scheduler - auto idle_task = idle_reaction->get_task(); - if (idle_task) { - idle_reaction->reactor.powerplant.submit(std::move(idle_task)); - } + idle_reaction->reactor.powerplant.submit(idle_reaction->get_task()); } template static inline void postcondition(threading::ReactionTask& task) { // Get a task for the always reaction and submit it to the scheduler - auto new_task = task.parent.get_task(); - if (new_task) { - task.parent.reactor.powerplant.submit(std::move(new_task)); - } + task.parent.reactor.powerplant.submit(task.parent.get_task()); } }; diff --git a/src/dsl/word/Every.hpp b/src/dsl/word/Every.hpp index 0c4555c74..e5dacfe41 100644 --- a/src/dsl/word/Every.hpp +++ b/src/dsl/word/Every.hpp @@ -93,22 +93,8 @@ namespace dsl { // Send our configuration out reaction->reactor.emit(std::make_unique( [reaction, jump](NUClear::clock::time_point& time) { - try { - // submit the reaction to the thread pool - auto task = reaction->get_task(); - if (task) { - reaction->reactor.powerplant.submit(std::move(task)); - } - } - // If there is an exception while generating a reaction print it here, this shouldn't happen - catch (const std::exception& ex) { - reaction->reactor.log("There was an exception while generating a reaction", - ex.what()); - } - catch (...) { - reaction->reactor.log( - "There was an unknown exception while generating a reaction"); - } + // submit the reaction to the thread pool + reaction->reactor.powerplant.submit(reaction->get_task()); time += jump; diff --git a/src/dsl/word/Watchdog.hpp b/src/dsl/word/Watchdog.hpp index 85672df0b..beffe5518 100644 --- a/src/dsl/word/Watchdog.hpp +++ b/src/dsl/word/Watchdog.hpp @@ -266,15 +266,8 @@ namespace dsl { // Check if our watchdog has timed out if (NUClear::clock::now() > (service_time + period(ticks))) { - try { - // Submit the reaction to the thread pool - auto task = reaction->get_task(); - if (task) { - reaction->reactor.powerplant.submit(std::move(task)); - } - } - catch (...) { - } + // Submit the reaction to the thread pool + reaction->reactor.powerplant.submit(reaction->get_task()); // Now automatically service the watchdog time += period(ticks); diff --git a/src/dsl/word/emit/Direct.hpp b/src/dsl/word/emit/Direct.hpp index e5bb62913..1335b02ed 100644 --- a/src/dsl/word/emit/Direct.hpp +++ b/src/dsl/word/emit/Direct.hpp @@ -56,24 +56,9 @@ namespace dsl { // Run all our reactions that are interested for (auto& reaction : store::TypeCallbackStore::get()) { - try { - - // Set our thread local store data each time (as during direct it can be overwritten) - store::ThreadStore>::value = &data; - - auto task = reaction->get_task(); - if (task) { - powerplant.submit(std::move(task), true); - } - } - catch (const std::exception& ex) { - PowerPlant::log("There was an exception while generating a reaction", - ex.what()); - } - catch (...) { - PowerPlant::log( - "There was an unknown exception while generating a reaction"); - } + // Set our thread local store data each time (as during direct it can be overwritten) + store::ThreadStore>::value = &data; + powerplant.submit(reaction->get_task(), true); } // Unset our thread local store data diff --git a/src/dsl/word/emit/Initialise.hpp b/src/dsl/word/emit/Initialise.hpp index 4e8a3a437..8a60ce73d 100644 --- a/src/dsl/word/emit/Initialise.hpp +++ b/src/dsl/word/emit/Initialise.hpp @@ -53,9 +53,18 @@ namespace dsl { static void emit(PowerPlant& powerplant, std::shared_ptr data) { - // Delay the emit by 0 seconds, this will delay the emit until the chrono controller starts, which - // will be when the system starts - Delay::emit(powerplant, data, std::chrono::seconds(0)); + static std::atomic id = 0; + + // Submit a task to the power plant to emit this object + powerplant.submit(++id, + 1000, + util::GroupDescriptor{}, + util::ThreadPoolDescriptor{}, + false, + [&powerplant, data] { + // Emit the data + powerplant.emit_shared(data); + }); } }; diff --git a/src/dsl/word/emit/Local.hpp b/src/dsl/word/emit/Local.hpp index 38397accc..7f09e29dc 100644 --- a/src/dsl/word/emit/Local.hpp +++ b/src/dsl/word/emit/Local.hpp @@ -51,26 +51,12 @@ namespace dsl { static void emit(PowerPlant& powerplant, std::shared_ptr data) { - // Set our thread local store data - store::ThreadStore>::value = &data; - // Run all our reactions that are interested for (auto& reaction : store::TypeCallbackStore::get()) { - try { - auto task = reaction->get_task(); - if (task) { - powerplant.submit(std::move(task)); - } - } - // If there is an exception while generating a reaction print it here, this shouldn't happen - catch (const std::exception& ex) { - PowerPlant::log("There was an exception while generating a reaction", - ex.what()); - } - catch (...) { - PowerPlant::log( - "There was an unknown exception while generating a reaction"); - } + + // Set our thread local store data + store::ThreadStore>::value = &data; + powerplant.submit(reaction->get_task()); } // Unset our thread local store data diff --git a/src/extension/ChronoController.hpp b/src/extension/ChronoController.hpp index 5de6108d2..b7f8cfb25 100644 --- a/src/extension/ChronoController.hpp +++ b/src/extension/ChronoController.hpp @@ -166,7 +166,7 @@ namespace extension { /// @brief The condition variable we use to wait on std::condition_variable wait; /// @brief If we are running or not - bool running{true}; + volatile bool running{true}; /// @brief The temporal accuracy when waiting on a condition variable NUClear::clock::duration cv_accuracy{0}; diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index 8f098c9e9..9e79b8fb4 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -181,17 +181,9 @@ namespace extension { e.events = it->waiting_events; // Submit the task (which should run the get) - try { - IO::ThreadEventStore::value = &e; - auto task = it->reaction->get_task(); - IO::ThreadEventStore::value = nullptr; - // Only actually submit this task if it is the only active task for this reaction - if (task) { - powerplant.submit(std::move(task)); - } - } - catch (...) { - } + IO::ThreadEventStore::value = &e; + powerplant.submit(it->reaction->get_task()); + IO::ThreadEventStore::value = nullptr; // Remove if we received a close event bool closed = (it->waiting_events & IO::CLOSE) != 0; diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index 8521dc502..dd8cca5e9 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -140,17 +140,9 @@ namespace extension { e.events = task.waiting_events; // Submit the task (which should run the get) - try { - IO::ThreadEventStore::value = &e; - auto reaction_task = task.reaction->get_task(); - IO::ThreadEventStore::value = nullptr; - // Only actually submit this task if it is the only active task for this reaction - if (reaction_task) { - powerplant.submit(std::move(reaction_task)); - } - } - catch (...) { - } + IO::ThreadEventStore::value = &e; + powerplant.submit(task.reaction->get_task()); + IO::ThreadEventStore::value = nullptr; // Reset our value it = ((task.waiting_events & IO::CLOSE) != 0) ? remove_task(it) : std::next(it); diff --git a/src/extension/NetworkController.hpp b/src/extension/NetworkController.hpp index 093b2316e..7ae5253c6 100644 --- a/src/extension/NetworkController.hpp +++ b/src/extension/NetworkController.hpp @@ -67,10 +67,7 @@ namespace extension { // Execute on our interested reactions for (auto it = rs.first; it != rs.second; ++it) { - auto task = it->second->get_task(); - if (task) { - powerplant.submit(std::move(task)); - } + powerplant.submit(it->second->get_task()); } } diff --git a/src/threading/ReactionTask.hpp b/src/threading/ReactionTask.hpp index 298629a8f..581c64954 100644 --- a/src/threading/ReactionTask.hpp +++ b/src/threading/ReactionTask.hpp @@ -145,27 +145,6 @@ namespace threading { template ATTRIBUTE_TLS Task* Task::current_task = nullptr; // NOLINT - /** - * @brief This overload is used to sort reactions by priority. - * - * @param a the reaction task a - * @param b the reaction task b - * - * @return true if a has lower priority than b, false otherwise - */ - template - inline bool operator<(const std::unique_ptr>& a, const std::unique_ptr>& b) { - - // Comparision order is as follows: - // nullptr is greater than anything else so it's removed from a queue first - // higher priority tasks are greater than lower priority tasks - // tasks created first (smaller ids) should run before tasks created later - return a == nullptr ? true - : b == nullptr ? false - : a->priority == b->priority ? a->id < b->id - : a->priority > b->priority; - } - // Alias the templated Task so that public API remains intact class Reaction; using ReactionTask = Task; diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 38640062f..4ed30a1d9 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -32,31 +32,34 @@ namespace NUClear { namespace threading { - bool TaskScheduler::is_runnable(const std::unique_ptr& task) { + bool TaskScheduler::is_runnable(const util::GroupDescriptor& group) { - if (groups.count(task->group_descriptor.group_id) == 0) { - groups[task->group_descriptor.group_id] = 0; + // Default group is always runnable + if (group.group_id == 0) { + return true; + } + + if (groups.count(group.group_id) == 0) { + groups[group.group_id] = 0; } // Task can run if the group it belongs to has spare threads - if (groups.at(task->group_descriptor.group_id) < task->group_descriptor.thread_count) { + if (groups.at(group.group_id) < group.thread_count) { // This task is about to run in this group, increase the number of active tasks in the group - groups.at(task->group_descriptor.group_id)++; + groups.at(group.group_id)++; return true; } return false; } - void TaskScheduler::run_task(std::unique_ptr&& task) { - if (task) { - task->run(); + void TaskScheduler::run_task(Task&& task) { + task.run(); - // This task is no longer running, decrease the number of active tasks in the group - /* mutex scope */ { - const std::lock_guard group_lock(group_mutex); - groups.at(task->group_descriptor.group_id)--; - } + // We need to do group counting if this isn't the default group + if (task.group_descriptor.group_id != 0) { + const std::lock_guard group_lock(group_mutex); + --groups.at(task.group_descriptor.group_id); } } @@ -66,12 +69,16 @@ namespace threading { current_queue = &pool; // When task is nullptr there are no more tasks to get and the scheduler is shutting down - for (auto task = get_task(); task != nullptr; task = get_task()) { - // Run the current task - run_task(std::move(task)); + while (running.load() || !pool->queue.empty()) { + try { + // Run the next task + run_task(get_task()); + } + catch (...) { + } } - // Clear the current queue so it can eventually be deleted + // Clear the current queue current_queue = nullptr; } @@ -158,7 +165,15 @@ namespace threading { } } - void TaskScheduler::submit(std::unique_ptr&& task, const bool& immediate) { + void TaskScheduler::submit(const uint64_t& id, + const int& priority, + const util::GroupDescriptor& group, + const util::ThreadPoolDescriptor& pool, + const bool& immediate, + std::function&& func) { + + // Move the arguments into a struct + Task task{id, priority, group, pool, std::move(func)}; // Immediate tasks are executed directly on the current thread if they can be // If something is blocking them from running right now they are added to the queue @@ -166,7 +181,7 @@ namespace threading { bool runnable = false; /* mutex scope */ { const std::lock_guard group_lock(group_mutex); - runnable = is_runnable(task); + runnable = is_runnable(group); } if (runnable) { run_task(std::move(task)); @@ -177,7 +192,7 @@ namespace threading { // We do not accept new tasks once we are shutdown if (running.load()) { // Get the appropiate pool for this task - const std::shared_ptr pool = get_pool_queue(task->thread_pool_descriptor); + const std::shared_ptr pool = get_pool_queue(task.thread_pool_descriptor); // Find where to insert the new task to maintain task order const std::lock_guard queue_lock(pool->mutex); @@ -190,7 +205,7 @@ namespace threading { } } - std::unique_ptr TaskScheduler::get_task() { + TaskScheduler::Task TaskScheduler::get_task() { // Wait at a high (but not realtime) priority to reduce latency for picking up a new task update_current_thread_priority(1000); @@ -216,10 +231,10 @@ namespace threading { for (auto it = queue.begin(); it != queue.end(); ++it) { // Check if we can run the task - if (is_runnable(*it)) { + if (is_runnable(it->group_descriptor)) { // Move the task out of the queue - std::unique_ptr task = std::move(*it); + Task task = std::move(*it); // Erase the old position in the queue queue.erase(it); @@ -234,7 +249,7 @@ namespace threading { // So if we can't find a task to run, just return nullptr and let the thread die. if (!running.load()) { condition.notify_all(); - return nullptr; + throw std::runtime_error("Task scheduler has shutdown"); } } @@ -242,8 +257,7 @@ namespace threading { condition.wait(lock); } - // No more tasks and scheduler has shutdown - return nullptr; + throw std::runtime_error("Task scheduler has shutdown"); } // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/src/threading/TaskScheduler.hpp b/src/threading/TaskScheduler.hpp index 5be139151..a9b25019b 100644 --- a/src/threading/TaskScheduler.hpp +++ b/src/threading/TaskScheduler.hpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -29,8 +30,7 @@ #include "../util/GroupDescriptor.hpp" #include "../util/ThreadPoolDescriptor.hpp" -#include "Reaction.hpp" -#include "ReactionTask.hpp" +#include "../util/platform.hpp" namespace NUClear { namespace threading { @@ -78,6 +78,50 @@ namespace threading { * running in a thread, or waiting in the Queue, then this task is ignored and dropped from the system. */ class TaskScheduler { + private: + /** + * @brief A struct which contains all the information about an individual task + */ + struct Task { + /// @brief The id of this task used for ordering + uint64_t id; + /// @brief The priority of this task + int priority; + /// @brief The group descriptor for this task + util::GroupDescriptor group_descriptor; + /// @brief The thread pool descriptor for this task + util::ThreadPoolDescriptor thread_pool_descriptor; + /// @brief The callback to be executed + std::function run; + + /** + * @brief Compare tasks based on their priority + * + * @param other the other task to compare to + * @return true if this task has a higher priority than the other task + */ + bool operator<(const Task& other) const { + return priority == other.priority ? id < other.id : priority > other.priority; + } + }; + + /** + * @brief A struct which contains all the information about an individual thread pool + */ + struct PoolQueue { + PoolQueue(const util::ThreadPoolDescriptor& pool_descriptor) : pool_descriptor(pool_descriptor) {} + /// @brief The descriptor for this thread pool + const util::ThreadPoolDescriptor pool_descriptor; + /// @brief The threads which are running in this thread pool + std::vector> threads; + /// @brief The queue of tasks for this specific thread pool + std::vector queue; + /// @brief The mutex which protects the queue + std::mutex mutex; + /// @brief The condition variable which threads wait on if they can't get a task + std::condition_variable condition; + }; + public: /** * @brief Constructs a new TaskScheduler instance, and builds the nullptr sync queue. @@ -106,30 +150,23 @@ namespace threading { * queue based on it's sync type and priority. It will then wait there until it is removed by a thread to * be processed. * - * @param task the task to be executed + * @param id the id of this task used for ordering tasks of the same priority + * @param priority the priority of this task + * @param group the group descriptor for this task + * @param pool the thread pool descriptor for this task * @param immediate if this task should run immediately in the current thread. If immediate execution of this - * task is not possible (e.g. due to group concurrency restrictions) this task will be queued as normal + * task is not possible (e.g. due to group concurrency restrictions) this task will be queued + * as normal + * @param func the function to execute */ - void submit(std::unique_ptr&& task, const bool& immediate = false); + void submit(const uint64_t& id, + const int& priority, + const util::GroupDescriptor& group, + const util::ThreadPoolDescriptor& pool, + const bool& immediate, + std::function&& func); private: - /** - * @brief A struct which contains all the information about an individual thread pool - */ - struct PoolQueue { - PoolQueue(const util::ThreadPoolDescriptor& pool_descriptor) : pool_descriptor(pool_descriptor) {} - /// @brief The descriptor for this thread pool - const util::ThreadPoolDescriptor pool_descriptor; - /// @brief The threads which are running in this thread pool - std::vector> threads; - /// @brief The queue of tasks for this specific thread pool - std::vector> queue; - /// @brief The mutex which protects the queue - std::mutex mutex; - /// @brief The condition variable which threads wait on if they can't get a task - std::condition_variable condition; - }; - /** * @brief Get a task object to be executed by a thread. * @@ -140,7 +177,7 @@ namespace threading { * * @return the task which has been given to be executed */ - std::unique_ptr get_task(); + Task get_task(); /** * @brief Gets a pool queue for the given thread pool descriptor or creates one if it does not exist @@ -170,8 +207,10 @@ namespace threading { * * @details After execution of the task has completed the number of active tasks in the tasks' group is * decremented + * + * @param task the task to execute */ - void run_task(std::unique_ptr&& task); + void run_task(Task&& task); /** * @brief Determines if the given task is able to be executed @@ -179,12 +218,12 @@ namespace threading { * @details If the current thread is able to be executed the number of active tasks in the tasks' groups is * incremented * - * @param task the task to inspect + * @param group the group descriptor for the task * * @return true if the task is currently runnable * @return false if the task is not currently runnable */ - bool is_runnable(const std::unique_ptr& task); + bool is_runnable(const util::GroupDescriptor& group); /// @brief if the scheduler is running, and accepting new tasks. If this is false and a new, non-immediate, task /// is submitted it will be ignored diff --git a/tests/dsl/emit/Initialise.cpp b/tests/dsl/emit/Initialise.cpp index 9a4d46cd1..734cd5ea2 100644 --- a/tests/dsl/emit/Initialise.cpp +++ b/tests/dsl/emit/Initialise.cpp @@ -24,24 +24,37 @@ // Anonymous namespace to keep everything file local namespace { -struct ShutdownNowPlx {}; +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +struct TestMessage { + TestMessage(std::string data) : data(std::move(data)) {} + std::string data; +}; class TestReactor : public test_util::TestBase { public: TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - emit(std::make_unique(5)); + emit(std::make_unique("Initialise before trigger")); + emit(std::make_unique("Normal before trigger")); - on>().then([this](const int& v) { - REQUIRE(v == 5); + on>().then([this](const TestMessage& v) { // + events.push_back("Triggered " + v.data); + }); - // We can't call shutdown here because - // we haven't started yet. That's because - // emits from Scope::INITIALIZE are not - // considered fully "initialized" - emit(std::make_unique()); + emit(std::make_unique("Normal after trigger")); + + on>>().then([this] { // + emit(std::make_unique("Initialise post startup")); + }); + on>>().then([this] { // + emit(std::make_unique("Normal post startup")); }); - on>().then([this] { powerplant.shutdown(); }); + on().then([this] { + emit(std::make_unique>()); + emit(std::make_unique>()); + }); } }; } // namespace @@ -51,6 +64,18 @@ TEST_CASE("Testing the Initialize scope", "[api][emit][initialize]") { config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + const std::vector expected = { + "Triggered Normal after trigger", + "Triggered Initialise before trigger", + "Triggered Initialise post startup", + "Triggered Normal post startup", + }; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } From d772d5d9a70bb5539c381e6f607601c4cca85ce7 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 15 Aug 2023 18:56:56 +1000 Subject: [PATCH 110/176] Try fix older gcc/clang --- src/dsl/word/emit/Initialise.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsl/word/emit/Initialise.hpp b/src/dsl/word/emit/Initialise.hpp index 8a60ce73d..21768fd5d 100644 --- a/src/dsl/word/emit/Initialise.hpp +++ b/src/dsl/word/emit/Initialise.hpp @@ -53,7 +53,7 @@ namespace dsl { static void emit(PowerPlant& powerplant, std::shared_ptr data) { - static std::atomic id = 0; + static std::atomic id{0}; // Submit a task to the power plant to emit this object powerplant.submit(++id, From 24baaed0490a3f70ec5e819413d262642a2ad8d3 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 15 Aug 2023 19:34:18 +1000 Subject: [PATCH 111/176] clang-tidy --- tests/dsl/emit/EmitFusion.cpp | 8 ++++---- tests/dsl/emit/Initialise.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/dsl/emit/EmitFusion.cpp b/tests/dsl/emit/EmitFusion.cpp index 95e3c2f8f..fe2c41fa4 100644 --- a/tests/dsl/emit/EmitFusion.cpp +++ b/tests/dsl/emit/EmitFusion.cpp @@ -30,22 +30,22 @@ std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-gl template struct E1 { - static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, int a, std::string b) { + static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, const int& a, const std::string& b) { events.push_back("E1a " + *p + " " + std::to_string(a) + " " + b); } - static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, std::string c) { + static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, const std::string& c) { events.push_back("E1b " + *p + " " + c); } }; template struct E2 { - static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, bool d) { + static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, const bool& d) { events.push_back("E2a " + *p + " " + (d ? "true" : "false")); } - static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, int e, std::string f) { + static inline void emit(NUClear::PowerPlant& /*unused*/, std::shared_ptr p, const int& e, const std::string& f) { events.push_back("E2b " + *p + " " + std::to_string(e) + " " + f); } }; diff --git a/tests/dsl/emit/Initialise.cpp b/tests/dsl/emit/Initialise.cpp index 734cd5ea2..4c11744f4 100644 --- a/tests/dsl/emit/Initialise.cpp +++ b/tests/dsl/emit/Initialise.cpp @@ -38,7 +38,7 @@ class TestReactor : public test_util::TestBase { emit(std::make_unique("Initialise before trigger")); emit(std::make_unique("Normal before trigger")); - on>().then([this](const TestMessage& v) { // + on>().then([](const TestMessage& v) { // events.push_back("Triggered " + v.data); }); From e89e351ac20b89b83d6569b6b30c8d6ef368ef2b Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 15 Aug 2023 19:36:35 +1000 Subject: [PATCH 112/176] Fix shadowing --- src/threading/TaskScheduler.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/threading/TaskScheduler.cpp b/src/threading/TaskScheduler.cpp index 4ed30a1d9..7869726af 100644 --- a/src/threading/TaskScheduler.cpp +++ b/src/threading/TaskScheduler.cpp @@ -167,13 +167,13 @@ namespace threading { void TaskScheduler::submit(const uint64_t& id, const int& priority, - const util::GroupDescriptor& group, - const util::ThreadPoolDescriptor& pool, + const util::GroupDescriptor& group_descriptor, + const util::ThreadPoolDescriptor& pool_descriptor, const bool& immediate, std::function&& func) { // Move the arguments into a struct - Task task{id, priority, group, pool, std::move(func)}; + Task task{id, priority, group_descriptor, pool_descriptor, std::move(func)}; // Immediate tasks are executed directly on the current thread if they can be // If something is blocking them from running right now they are added to the queue @@ -181,7 +181,7 @@ namespace threading { bool runnable = false; /* mutex scope */ { const std::lock_guard group_lock(group_mutex); - runnable = is_runnable(group); + runnable = is_runnable(task.group_descriptor); } if (runnable) { run_task(std::move(task)); From 49bc5b82252a3e238011fad8f0d5f555970de54c Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 17 Aug 2023 15:15:32 +1000 Subject: [PATCH 113/176] Clang-tidy --- src/PowerPlant.ipp | 6 ++---- src/extension/IOController_Posix.hpp | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/PowerPlant.ipp b/src/PowerPlant.ipp index 55dccd748..635122010 100644 --- a/src/PowerPlant.ipp +++ b/src/PowerPlant.ipp @@ -55,11 +55,9 @@ void PowerPlant::install() { // Make sure that the class that we received is a reactor static_assert(std::is_base_of::value, "You must install Reactors"); - // Get the demangled reactor name - std::string reactor_name = util::demangle(typeid(T).name()); - // The reactor constructor should handle subscribing to events - reactors.push_back(std::make_unique(std::make_unique(*this, reactor_name, level))); + reactors.push_back( + std::make_unique(std::make_unique(*this, util::demangle(typeid(T).name()), level))); } // Default emit with no types diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index bb98c3a65..0e772a36c 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -130,7 +130,7 @@ namespace extension { // descriptor is sending EOF and we should fire a CLOSE event too and stop watching if ((fd.revents & IO::READ) != 0) { int bytes_available = 0; - bool valid = ::ioctl(fd.fd, FIONREAD, &bytes_available) == 0; + const bool valid = ::ioctl(fd.fd, FIONREAD, &bytes_available) == 0; if (valid && bytes_available == 0) { // NOLINTNEXTLINE(google-runtime-int) fd.revents = short(fd.revents | IO::CLOSE); @@ -186,7 +186,7 @@ namespace extension { IO::ThreadEventStore::value = nullptr; // Remove if we received a close event - bool closed = (it->waiting_events & IO::CLOSE) != 0; + const bool closed = (it->waiting_events & IO::CLOSE) != 0; dirty |= closed; it = closed ? tasks.erase(it) : std::next(it); } From 0f0b0337c00b8bfb8ea8546fe018e11937cb5043 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 18 Aug 2023 13:04:44 +1000 Subject: [PATCH 114/176] Move socket size onto sock_t as a member --- src/extension/network/NUClearNetwork.cpp | 29 ++++++++---------------- src/util/network/sock_t.hpp | 12 ++++++++++ 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/extension/network/NUClearNetwork.cpp b/src/extension/network/NUClearNetwork.cpp index b75485e3a..58d6b866f 100644 --- a/src/extension/network/NUClearNetwork.cpp +++ b/src/extension/network/NUClearNetwork.cpp @@ -67,15 +67,6 @@ namespace extension { return {from, std::move(payload)}; } - - socklen_t socket_size(const util::network::sock_t& s) { - switch (s.sock.sa_family) { - case AF_INET: return sizeof(sockaddr_in); - case AF_INET6: return sizeof(sockaddr_in6); - default: throw std::runtime_error("unhandled socket address family"); - } - } - NUClearNetwork::PacketQueue::PacketTarget::PacketTarget(std::weak_ptr target, std::vector acked) : target(std::move(target)), acked(std::move(acked)), last_send(std::chrono::steady_clock::now()) {} @@ -182,7 +173,7 @@ namespace extension { } // Bind to the address, and if we fail throw an error - if (::bind(data_fd, &address.sock, socket_size(address)) != 0) { + if (::bind(data_fd, &address.sock, address.size()) != 0) { throw std::system_error(network_errno, std::system_category(), "Unable to bind the UDP socket to the port"); @@ -234,7 +225,7 @@ namespace extension { } // Bind to the address - if (::bind(announce_fd, &address.sock, socket_size(address)) != 0) { + if (::bind(announce_fd, &address.sock, address.size()) != 0) { throw std::system_error(network_errno, std::system_category(), "Unable to bind the UDP socket"); } @@ -335,7 +326,7 @@ namespace extension { sizeof(packet), 0, &it->second->target.sock, - socket_size(it->second->target)); + it->second->target.size()); } } @@ -590,7 +581,7 @@ namespace extension { static_cast(announce_packet.size()), 0, &it->second->target.sock, - socket_size(it->second->target)) + it->second->target.size()) < 0) { throw std::system_error(network_errno, std::system_category(), @@ -652,7 +643,7 @@ namespace extension { static_cast(announce_packet.size()), 0, &ptr->target.sock, - socket_size(ptr->target)); + ptr->target.size()); } } @@ -745,7 +736,7 @@ namespace extension { static_cast(r.size()), 0, &to.sock, - socket_size(to)); + to.size()); // We don't need to process this packet we already did return; @@ -776,7 +767,7 @@ namespace extension { sizeof(response), 0, &to.sock, - socket_size(to)); + to.size()); // Set this packet to have been recently received remote->recent_packets[++remote->recent_packets_index] = packet.packet_id; @@ -828,7 +819,7 @@ namespace extension { static_cast(r.size()), 0, &to.sock, - socket_size(to)); + to.size()); } // Clear our packets here (the one we just got will be added right after this) @@ -864,7 +855,7 @@ namespace extension { static_cast(r.size()), 0, &to.sock, - socket_size(to)); + to.size()); } // Check to see if we have enough to assemble the whole thing @@ -1073,7 +1064,7 @@ namespace extension { // Set our target and send (once again const cast is fine) message.msg_name = const_cast(&target.sock); // NOLINT - message.msg_namelen = socket_size(target); + message.msg_namelen = target.size(); // TODO(trent): if reliable, run select first to see if this socket is writeable // If it is not reliable just don't send the message instead of blocking diff --git a/src/util/network/sock_t.hpp b/src/util/network/sock_t.hpp index 6b2f8bd6e..75ef3b1c3 100644 --- a/src/util/network/sock_t.hpp +++ b/src/util/network/sock_t.hpp @@ -19,6 +19,8 @@ #ifndef NUCLEAR_UTIL_NETWORK_SOCK_T_HPP #define NUCLEAR_UTIL_NETWORK_SOCK_T_HPP +#include + #include "../platform.hpp" namespace NUClear { @@ -32,6 +34,16 @@ namespace util { sockaddr_in ipv4; sockaddr_in6 ipv6; }; + + socklen_t size() const { + switch (sock.sa_family) { + case AF_INET: return sizeof(sockaddr_in); + case AF_INET6: return sizeof(sockaddr_in6); + default: + throw std::runtime_error("Cannot get size for socket address family " + + std::to_string(sock.sa_family)); + } + } }; } // namespace network From cf408cc4872ee7027c105785f597d50bb93feb25 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 18 Aug 2023 13:06:04 +1000 Subject: [PATCH 115/176] Add a resolve utility --- src/util/network/resolve.cpp | 88 ++++++++++++++++++++++++++++++++++++ src/util/network/resolve.hpp | 48 ++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 src/util/network/resolve.cpp create mode 100644 src/util/network/resolve.hpp diff --git a/src/util/network/resolve.cpp b/src/util/network/resolve.cpp new file mode 100644 index 000000000..9b5943bee --- /dev/null +++ b/src/util/network/resolve.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "resolve.hpp" + +#include +#include +#include + +#include "../platform.hpp" + +namespace NUClear { +namespace util { + namespace network { + + NUClear::util::network::sock_t resolve(const std::string& address, const uint16_t& port) { + addrinfo hints{}; + memset(&hints, 0, sizeof hints); // make sure the struct is empty + hints.ai_family = AF_UNSPEC; // don't care about IPv4 or IPv6 + hints.ai_socktype = SOCK_STREAM; // using a tcp stream + + // Get our info on this address + addrinfo* servinfo_ptr = nullptr; + if (::getaddrinfo(address.c_str(), std::to_string(port).c_str(), &hints, &servinfo_ptr) != 0) { + throw std::runtime_error("Failed to get address information for " + address + ":" + + std::to_string(port)); + } + + // Check if we have any addresses to work with + if (servinfo_ptr == nullptr) { + throw std::runtime_error("Unable to find an address for " + address + ":" + std::to_string(port)); + } + + std::unique_ptr servinfo(servinfo_ptr, ::freeaddrinfo); + + // Clear our struct + NUClear::util::network::sock_t target{}; + std::memset(&target, 0, sizeof(target)); + + // The list is actually a linked list of valid addresses + // The address we choose is in the following priority, IPv4, IPv6, Other + for (addrinfo* p = servinfo.get(); p != nullptr; p = p->ai_next) { + + // If we find an IPv4 address, prefer that + if (servinfo->ai_family == AF_INET) { + + // Clear and set our struct + std::memcpy(&target, servinfo->ai_addr, servinfo->ai_addrlen); + + // We prefer IPv4 so use it and stop looking + return target; + } + + // If we find an IPv6 address, hold the first one in case we don't find an IPv4 address + if (servinfo->ai_family == AF_INET6) { + + // Clear and set our struct + std::memcpy(&target, servinfo->ai_addr, servinfo->ai_addrlen); + } + } + + // Found addresses, but none of them were IPv4 or IPv6 + if (target.sock.sa_family == AF_UNSPEC) { + throw std::runtime_error("Unable to find an address for " + address + ":" + std::to_string(port)); + } + + return target; + } + + } // namespace network +} // namespace util + +} // namespace NUClear diff --git a/src/util/network/resolve.hpp b/src/util/network/resolve.hpp new file mode 100644 index 000000000..460c8f650 --- /dev/null +++ b/src/util/network/resolve.hpp @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef UTILITY_NETWORK_RESOLVE_HPP +#define UTILITY_NETWORK_RESOLVE_HPP + +#include + +#include "sock_t.hpp" + +namespace NUClear { +namespace util { + namespace network { + + /** + * @brief Resolves a hostname and port into a socket address + * + * @details + * This function will resolve a hostname and port into a socket address. + * It will return a socket address that can be used to connect to the specified host and port. + * + * @param address the hostname or IP address to resolve + * @param port the port to connect to + * + * @return a socket address that can be used to connect to the specified host and port + */ + sock_t resolve(const std::string& address, const uint16_t& port); + + } // namespace network +} // namespace util +} // namespace NUClear + +#endif // UTILITY_NETWORK_RESOLVE_HPP From d620443065a7e29e9e34f5a44a31eef5b1f1af18 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 22 Aug 2023 22:37:35 +1000 Subject: [PATCH 116/176] Lots of refactoring and combining tests --- src/dsl/word/TCP.hpp | 88 ++-- src/dsl/word/UDP.hpp | 488 +++++++++++--------- src/dsl/word/emit/UDP.hpp | 192 +++----- src/util/network/if_number_from_address.cpp | 57 +++ src/util/network/if_number_from_address.hpp | 44 ++ src/util/network/resolve.cpp | 12 +- src/util/network/sock_t.hpp | 18 +- tests/dsl/TCP.cpp | 106 +++-- tests/dsl/UDP.cpp | 229 +++++++-- tests/dsl/UDPBroadcast.cpp | 143 ------ tests/dsl/UDPMulticastKnownPort.cpp | 110 ----- tests/dsl/UDPMulticastUnknownPort.cpp | 115 ----- tests/dsl/emit/UDP.cpp | 83 ---- 13 files changed, 772 insertions(+), 913 deletions(-) create mode 100644 src/util/network/if_number_from_address.cpp create mode 100644 src/util/network/if_number_from_address.hpp delete mode 100644 tests/dsl/UDPBroadcast.cpp delete mode 100644 tests/dsl/UDPMulticastKnownPort.cpp delete mode 100644 tests/dsl/UDPMulticastUnknownPort.cpp delete mode 100644 tests/dsl/emit/UDP.cpp diff --git a/src/dsl/word/TCP.hpp b/src/dsl/word/TCP.hpp index 3e9fd94e4..aafac5e1b 100644 --- a/src/dsl/word/TCP.hpp +++ b/src/dsl/word/TCP.hpp @@ -24,6 +24,8 @@ #include "../../PowerPlant.hpp" #include "../../threading/Reaction.hpp" #include "../../util/FileDescriptor.hpp" +#include "../../util/network/resolve.hpp" +#include "../../util/network/sock_t.hpp" #include "../../util/platform.hpp" #include "IO.hpp" @@ -62,46 +64,60 @@ namespace dsl { struct Connection { - struct { - uint32_t address; + struct Target { + /// @brief The address of the connection + std::string address; + /// @brief The port of the connection uint16_t port; - } remote; + }; - struct { - uint32_t address; - uint16_t port; - } local; + /// @brief The local address of the connection + Target local; + /// @brief The remote address of the connection + Target remote; + /// @brief The file descriptor for the connection fd_t fd; + /** + * @brief Casts this packet to a boolean to check if it is valid + * + * @return true if the packet is valid + */ operator bool() const { - return fd != 0; + return fd != INVALID_SOCKET; } }; template static inline std::tuple bind(const std::shared_ptr& reaction, - in_port_t port = 0) { + in_port_t port = 0, + const std::string& bind_address = "") { + + // Resolve the bind address if we have one + util::network::sock_t address{}; + + if (!bind_address.empty()) { + address = util::network::resolve(bind_address, port); + } + else { + address.ipv4.sin_family = AF_INET; + address.ipv4.sin_port = htons(port); + address.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); + } // Make our socket - util::FileDescriptor fd(::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), + util::FileDescriptor fd(::socket(address.sock.sa_family, SOCK_STREAM, IPPROTO_TCP), [](fd_t fd) { ::shutdown(fd, SHUT_RDWR); }); - if (fd < 0) { + if (fd == INVALID_SOCKET) { throw std::system_error(network_errno, std::system_category(), "We were unable to open the TCP socket"); } - // The address we will be binding to - sockaddr_in address{}; - std::memset(&address, 0, sizeof(sockaddr_in)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = htonl(INADDR_ANY); - // Bind to the address, and if we fail throw an error - if (::bind(fd, reinterpret_cast(&address), sizeof(sockaddr))) { + if (::bind(fd, &address.sock, address.size())) { throw std::system_error(network_errno, std::system_category(), "We were unable to bind the TCP socket to the port"); @@ -115,13 +131,18 @@ namespace dsl { } // Get the port we ended up listening on - socklen_t len = sizeof(sockaddr_in); - if (::getsockname(fd, reinterpret_cast(&address), &len) == -1) { + socklen_t len = sizeof(address); + if (::getsockname(fd, &address.sock, &len) == -1) { throw std::system_error(network_errno, std::system_category(), "We were unable to get the port from the TCP socket"); } - port = ntohs(address.sin_port); + if (address.ipv4.sin_family == AF_INET6) { + port = ntohs(address.ipv6.sin6_port); + } + else { + port = ntohs(address.ipv4.sin_port); + } // Generate a reaction for the IO system that closes on death const fd_t cfd = fd.release(); @@ -145,28 +166,31 @@ namespace dsl { auto event = IO::get(reaction); // If our get is being run without an fd (something else triggered) then short circuit - if (event.fd == 0) { + if (!event) { return Connection{{0, 0}, {0, 0}, 0}; } // Accept our connection - sockaddr_in local{}; - sockaddr_in remote{}; - socklen_t size = sizeof(sockaddr_in); + util::network::sock_t local{}; + util::network::sock_t remote{}; // Accept the remote connection - util::FileDescriptor fd = ::accept(event.fd, reinterpret_cast(&remote), &size); + socklen_t remote_size = sizeof(util::network::sock_t); + util::FileDescriptor fd(::accept(event.fd, &remote.sock, &remote_size), + [](fd_t fd) { ::shutdown(fd, SHUT_RDWR); }); // Get our local address - ::getsockname(fd, reinterpret_cast(&local), &size); + socklen_t local_size = sizeof(util::network::sock_t); + ::getsockname(fd, &local.sock, &local_size); - if (fd == -1) { + if (fd == INVALID_SOCKET) { return Connection{{0, 0}, {0, 0}, 0}; } - return Connection{{ntohl(remote.sin_addr.s_addr), ntohs(remote.sin_port)}, - {ntohl(local.sin_addr.s_addr), ntohs(local.sin_port)}, - fd.release()}; + auto local_s = local.address(); + auto remote_s = remote.address(); + + return Connection{{local_s.first, local_s.second}, {remote_s.first, remote_s.second}, fd.release()}; } }; diff --git a/src/dsl/word/UDP.hpp b/src/dsl/word/UDP.hpp index 76065dca4..8831f55da 100644 --- a/src/dsl/word/UDP.hpp +++ b/src/dsl/word/UDP.hpp @@ -25,6 +25,8 @@ #include "../../threading/Reaction.hpp" #include "../../util/FileDescriptor.hpp" #include "../../util/network/get_interfaces.hpp" +#include "../../util/network/if_number_from_address.hpp" +#include "../../util/network/resolve.hpp" #include "../../util/platform.hpp" #include "IO.hpp" @@ -59,39 +61,44 @@ namespace dsl { * Bind */ struct UDP : public IO { + private: + struct Target { + std::string bind_address{}; + in_port_t port = 0; + std::string target_address{}; + enum Type { UNICAST, BROADCAST, MULTICAST } type{}; + }; + public: struct Packet { Packet() = default; - /// If the packet is valid (it contains data) + /// @brief If the packet is valid (it contains data) bool valid{false}; - /// The information about this packet's source - struct Remote { - Remote() = default; - Remote(uint32_t addr, uint16_t port) : address(addr), port(port) {} + struct Target { + Target() = default; + Target(std::string address, const uint16_t& port) : address(std::move(address)), port(port) {} - /// The address that the packet is from - uint32_t address{0}; - /// The port that the packet is from + /// The address of the target + std::string address{}; + /// The port of the target uint16_t port{0}; - } remote; - - /// The information about this packet's destination - struct Local { - Local() = default; - Local(uint32_t addr, uint16_t port) : address(addr), port(port) {} + }; - /// The address that the packet is to - uint32_t address{0}; - /// The port that the packet is to - uint16_t port{0}; - } local; + /// @brief The information about this packets destination + Target local; + /// @brief The information about this packets source + Target remote; - /// The data to be sent in the packet + /// @brief The data to be sent in the packet std::vector payload{}; - /// Our validator when returned for if we are a real packet + /** + * @brief Casts this packet to a boolean to check if it is valid + * + * @return true if the packet is valid + */ operator bool() const { return valid; } @@ -105,47 +112,191 @@ namespace dsl { }; template - static inline std::tuple bind(const std::shared_ptr& reaction, - in_port_t port = 0) { + static inline std::tuple connect(const std::shared_ptr& reaction, + const Target& target) { + + // Resolve the addresses + util::network::sock_t bind_address{}; + util::network::sock_t multicast_target{}; + if (target.type == Target::MULTICAST) { + multicast_target = util::network::resolve(target.target_address, target.port); + if (target.bind_address.empty()) { + bind_address = multicast_target; + switch (bind_address.sock.sa_family) { + case AF_INET: { + bind_address.ipv4.sin_port = htons(target.port); + bind_address.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); + } break; + case AF_INET6: { + bind_address.ipv6.sin6_port = htons(target.port); + bind_address.ipv6.sin6_addr = in6addr_any; + } break; + default: throw std::runtime_error("Unknown socket family"); + } + } + else { + bind_address = util::network::resolve(target.bind_address, target.port); + if (multicast_target.sock.sa_family != bind_address.sock.sa_family) { + throw std::runtime_error("Multicast address family does not match bind address family"); + } + } + } + else { + if (target.bind_address.empty()) { + bind_address.ipv4.sin_family = AF_INET; + bind_address.ipv4.sin_port = htons(target.port); + bind_address.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); + } + else { + bind_address = util::network::resolve(target.bind_address, target.port); + } + } // Make our socket - util::FileDescriptor fd = ::socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - if (fd < 0) { + util::FileDescriptor fd = ::socket(bind_address.sock.sa_family, SOCK_DGRAM, IPPROTO_UDP); + if (!fd.valid()) { + throw std::system_error(network_errno, std::system_category(), "Unable to open the UDP socket"); + } + + int yes = 1; + // Include struct in the message "ancillary" control data + if (bind_address.sock.sa_family == AF_INET) { + if (::setsockopt(fd, IPPROTO_IP, IP_PKTINFO, reinterpret_cast(&yes), sizeof(yes)) + < 0) { + throw std::system_error(network_errno, + std::system_category(), + "We were unable to flag the socket as getting ancillary data"); + } + } + else if (bind_address.sock.sa_family == AF_INET6) { + if (::setsockopt(fd, + IPPROTO_IPV6, + IPV6_RECVPKTINFO, + reinterpret_cast(&yes), + sizeof(yes)) + < 0) { throw std::system_error(network_errno, std::system_category(), - "We were unable to open the UDP socket"); + "We were unable to flag the socket as getting ancillary data"); + } } - // The address we will be binding to - sockaddr_in address{}; - std::memset(&address, 0, sizeof(sockaddr_in)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = htonl(INADDR_ANY); + // Broadcast and multicast reuse address and port + if (target.type == Target::BROADCAST || target.type == Target::MULTICAST) { - // Bind to the address, and if we fail throw an error - if (::bind(fd, reinterpret_cast(&address), sizeof(sockaddr))) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to bind the UDP socket to the port"); + if (::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), sizeof(yes)) < 0) { + throw std::system_error(network_errno, + std::system_category(), + "Unable to reuse address on the socket"); + } + +// If SO_REUSEPORT is available set it too +#ifdef SO_REUSEPORT + if (::setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast(&yes), sizeof(yes)) < 0) { + throw std::system_error(network_errno, + std::system_category(), + "Unable to reuse port on the socket"); + } +#endif + + // We enable SO_BROADCAST since sometimes we need to send broadcast packets + if (::setsockopt(fd, SOL_SOCKET, SO_BROADCAST, reinterpret_cast(&yes), sizeof(yes)) < 0) { + throw std::system_error(network_errno, + std::system_category(), + "Unable to set broadcast on the socket"); + } } - int yes = 1; - // Include struct in_pktinfo in the message "ancilliary" control data - if (::setsockopt(fd, IPPROTO_IP, IP_PKTINFO, reinterpret_cast(&yes), sizeof(yes)) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to flag the socket as getting ancillary data"); + // Bind to the address + if (::bind(fd, &bind_address.sock, bind_address.size()) != 0) { + throw std::system_error(network_errno, std::system_category(), "Unable to bind the UDP socket"); + } + + // If we have a multicast address, then we need to join the multicast groups + if (target.type == Target::MULTICAST) { + + // Our multicast join request will depend on protocol version + if (multicast_target.sock.sa_family == AF_INET) { + + // Set the multicast address we are listening on and bind address + ip_mreq mreq{}; + mreq.imr_multiaddr = multicast_target.ipv4.sin_addr; + mreq.imr_interface = bind_address.ipv4.sin_addr; + + // Join our multicast group + if (::setsockopt(fd, + IPPROTO_IP, + IP_ADD_MEMBERSHIP, + reinterpret_cast(&mreq), + sizeof(ip_mreq)) + < 0) { + throw std::system_error(network_errno, + std::system_category(), + "There was an error while attempting to join the multicast group"); + } + + // Set our transmission interface for the multicast socket + if (::setsockopt(fd, + IPPROTO_IP, + IP_MULTICAST_IF, + reinterpret_cast(&bind_address.ipv4.sin_addr), + sizeof(bind_address.ipv4.sin_addr)) + < 0) { + throw std::system_error(network_errno, + std::system_category(), + "We were unable to use the requested interface for multicast"); + } + } + else if (multicast_target.sock.sa_family == AF_INET6) { + + // Set the multicast address we are listening on + ipv6_mreq mreq{}; + mreq.ipv6mr_multiaddr = multicast_target.ipv6.sin6_addr; + mreq.ipv6mr_interface = util::network::if_number_from_address(bind_address.ipv6); + + // Join our multicast group + if (::setsockopt(fd, + IPPROTO_IPV6, + IPV6_JOIN_GROUP, + reinterpret_cast(&mreq), + sizeof(ipv6_mreq)) + < 0) { + throw std::system_error(network_errno, + std::system_category(), + "There was an error while attempting to join the multicast group"); + } + + // Set our transmission interface for the multicast socket + if (::setsockopt(fd, + IPPROTO_IPV6, + IPV6_MULTICAST_IF, + reinterpret_cast(&mreq.ipv6mr_interface), + sizeof(mreq.ipv6mr_interface)) + < 0) { + throw std::system_error(network_errno, + std::system_category(), + "We were unable to use the requested interface for multicast"); + } + } } // Get the port we ended up listening on socklen_t len = sizeof(sockaddr_in); - if (::getsockname(fd, reinterpret_cast(&address), &len) == -1) { + if (::getsockname(fd, reinterpret_cast(&bind_address), &len) == -1) { throw std::system_error(network_errno, std::system_category(), "We were unable to get the port from the UDP socket"); } - port = ntohs(address.sin_port); + in_port_t port; + if (bind_address.sock.sa_family == AF_INET) { + port = ntohs(bind_address.ipv4.sin_port); + } + else if (bind_address.sock.sa_family == AF_INET6) { + port = ntohs(bind_address.ipv6.sin6_port); + } + else { + throw std::runtime_error("Unknown socket family"); + } // Generate a reaction for the IO system that closes on death const fd_t cfd = fd.release(); @@ -154,48 +305,46 @@ namespace dsl { // Return our handles and our bound port return std::make_tuple(port, cfd); + return std::make_tuple(in_port_t(0), fd_t(0)); + } + + template + static inline std::tuple bind(const std::shared_ptr& reaction, + in_port_t port = 0, + std::string bind_address = "") { + Target t; + t.type = Target::UNICAST; + t.bind_address = std::move(bind_address); + t.port = port; + return connect(reaction, t); } template static inline Packet get(threading::Reaction& reaction) { - // Get our filedescriptor from the magic cache + // Get our file descriptor from the magic cache auto event = IO::get(reaction); // If our get is being run without an fd (something else triggered) then short circuit - if (event.fd == 0) { - Packet p; - p.remote.address = INADDR_NONE; - p.remote.port = 0; - p.local.address = INADDR_NONE; - p.local.port = 0; - p.valid = false; - return p; + if (!event) { + return Packet{}; } - // Make a packet with 2k of storage (hopefully packets are smaller than this as most MTUs are around - // 1500) - Packet p; - p.remote.address = INADDR_NONE; - p.remote.port = 0; - p.local.address = INADDR_NONE; - p.local.port = 0; - p.valid = false; - p.payload.resize(2048); + // Allocate max size for a UDP packet + Packet p{}; + p.payload.resize(65535); // Make some variables to hold our message header information std::array cmbuff = {0}; - sockaddr_in from{}; - std::memset(&from, 0, sizeof(sockaddr_in)); + util::network::sock_t remote{}; iovec payload{}; payload.iov_base = p.payload.data(); payload.iov_len = static_cast(p.payload.size()); // Make our message header to receive with msghdr mh{}; - std::memset(&mh, 0, sizeof(msghdr)); - mh.msg_name = reinterpret_cast(&from); - mh.msg_namelen = sizeof(sockaddr_in); + mh.msg_name = &remote.sock; + mh.msg_namelen = sizeof(util::network::sock_t); mh.msg_control = cmbuff.data(); mh.msg_controllen = cmbuff.size(); mh.msg_iov = &payload; @@ -204,38 +353,49 @@ namespace dsl { // Receive our message ssize_t received = recvmsg(event.fd, &mh, 0); + // Load the socket we are listening on + util::network::sock_t local{}; + socklen_t len = sizeof(util::network::sock_t); + if (::getsockname(event.fd, &local.sock, &len) == -1) { + throw std::system_error(network_errno, + std::system_category(), + "We were unable to get the port from the UDP socket"); + } + // Iterate through control headers to get IP information - in_addr_t our_addr = 0; for (cmsghdr* cmsg = CMSG_FIRSTHDR(&mh); cmsg != nullptr; cmsg = CMSG_NXTHDR(&mh, cmsg)) { // ignore the control headers that don't match what we want - if (cmsg->cmsg_level == IPPROTO_IP && cmsg->cmsg_type == IP_PKTINFO) { + if (local.sock.sa_family == AF_INET && cmsg->cmsg_level == IPPROTO_IP + && cmsg->cmsg_type == IP_PKTINFO) { // Access the packet header information auto* pi = reinterpret_cast(reinterpret_cast(cmsg) + sizeof(*cmsg)); - our_addr = pi->ipi_addr.s_addr; + local.ipv4.sin_family = AF_INET; + local.ipv4.sin_addr = pi->ipi_addr; // We are done break; } - } + else if (local.sock.sa_family == AF_INET6 && cmsg->cmsg_level == IPPROTO_IPV6 + && cmsg->cmsg_type == IPV6_PKTINFO) { - // Get the port this socket is listening on - socklen_t len = sizeof(sockaddr_in); - sockaddr_in address{}; - if (::getsockname(event.fd, reinterpret_cast(&address), &len) == -1) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to get the port from the UDP socket"); + // Access the packet header information + auto* pi = reinterpret_cast(reinterpret_cast(cmsg) + sizeof(*cmsg)); + local.ipv6.sin6_addr = pi->ipi6_addr; + + // We are done + break; + } } + // if no error if (received > 0) { - p.valid = true; - p.remote.address = ntohl(from.sin_addr.s_addr); - p.remote.port = ntohs(from.sin_port); - p.local.address = ntohl(our_addr); - p.local.port = ntohs(address.sin_port); + p.valid = true; + std::tie(p.local.address, p.local.port) = local.address(); + std::tie(p.remote.address, p.remote.port) = remote.address(); p.payload.resize(size_t(received)); + p.payload.shrink_to_fit(); } return p; @@ -245,66 +405,13 @@ namespace dsl { template static inline std::tuple bind(const std::shared_ptr& reaction, - in_port_t port = 0) { - - // Make our socket - util::FileDescriptor fd = ::socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - if (fd < 0) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to open the UDP Broadcast socket"); - } - - // The address we will be binding to - sockaddr_in address{}; - std::memset(&address, 0, sizeof(sockaddr_in)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = htonl(INADDR_ANY); - - int yes = 1; - // We are a broadcast socket - if (::setsockopt(fd, SOL_SOCKET, SO_BROADCAST, reinterpret_cast(&yes), sizeof(yes)) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to set the socket as broadcast"); - } - // Set that we reuse the address so more than one application can bind - if (::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), sizeof(yes)) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to set reuse address on the socket"); - } - // Include struct in_pktinfo in the message "ancilliary" control data - if (::setsockopt(fd, IPPROTO_IP, IP_PKTINFO, reinterpret_cast(&yes), sizeof(yes)) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to flag the socket as getting ancillary data"); - } - - // Bind to the address, and if we fail throw an error - if (::bind(fd, reinterpret_cast(&address), sizeof(sockaddr))) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to bind the UDP socket to the port"); - } - - // Get the port we ended up listening on - socklen_t len = sizeof(sockaddr_in); - if (::getsockname(fd, reinterpret_cast(&address), &len) == -1) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to get the port from the UDP socket"); - } - port = ntohs(address.sin_port); - - // Generate a reaction for the IO system that closes on death - const fd_t cfd = fd.release(); - reaction->unbinders.push_back([cfd](const threading::Reaction&) { ::close(cfd); }); - IO::bind(reaction, cfd, IO::READ | IO::CLOSE); - - // Return our handles and our bound port - return std::make_tuple(port, cfd); + in_port_t port = 0, + std::string bind_address = "") { + Target t; + t.type = Target::BROADCAST; + t.bind_address = std::move(bind_address); + t.port = port; + return UDP::connect(reaction, t); } template @@ -318,91 +425,14 @@ namespace dsl { template static inline std::tuple bind(const std::shared_ptr& reaction, const std::string& multicast_group, - in_port_t port = 0) { - - // Our multicast group address - sockaddr_in address{}; - std::memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_addr.s_addr = INADDR_ANY; - address.sin_port = htons(port); - - // Make our socket - util::FileDescriptor fd = ::socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - if (fd < 0) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to open the UDP socket"); - } - - int yes = 1; - // Set that we reuse the address so more than one application can bind - if (::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), sizeof(yes)) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to set reuse address on the socket"); - } - // Include struct in_pktinfo in the message "ancilliary" control data - if (::setsockopt(fd, IPPROTO_IP, IP_PKTINFO, reinterpret_cast(&yes), sizeof(yes)) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to flag the socket as getting ancillary data"); - } - - // Bind to the address - if (::bind(fd, reinterpret_cast(&address), sizeof(sockaddr))) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to bind the UDP socket to the port"); - } - - // Store the port variable that was used - socklen_t len = sizeof(sockaddr_in); - if (::getsockname(fd, reinterpret_cast(&address), &len) == -1) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to get the port from the UDP socket"); - } - port = ntohs(address.sin_port); - - // Get all the network interfaces that support multicast - std::vector addresses; - for (auto& iface : util::network::get_interfaces()) { - // We receive on broadcast addresses and we don't want loopback or point to point - if (iface.flags.multicast && iface.ip.sock.sa_family == AF_INET) { - const auto& i = *reinterpret_cast(&iface.ip); - addresses.push_back(i.sin_addr.s_addr); - } - } - - for (auto& ad : addresses) { - - // Our multicast join request - ip_mreq mreq{}; - std::memset(&mreq, 0, sizeof(mreq)); - ::inet_pton(AF_INET, multicast_group.c_str(), &mreq.imr_multiaddr); - mreq.imr_interface.s_addr = ad; - - // Join our multicast group - if (::setsockopt(fd, - IPPROTO_IP, - IP_ADD_MEMBERSHIP, - reinterpret_cast(&mreq), - sizeof(ip_mreq)) - < 0) { - throw std::system_error(network_errno, - std::system_category(), - "There was an error while attempting to join the multicast group"); - } - } - - // Generate a reaction for the IO system that closes on death - const fd_t cfd = fd.release(); - reaction->unbinders.push_back([cfd](const threading::Reaction&) { close(cfd); }); - IO::bind(reaction, cfd, IO::READ | IO::CLOSE); - - // Return our handles - return std::make_tuple(port, cfd); + in_port_t port = 0, + const std::string& bind_address = "") { + Target t; + t.type = Target::MULTICAST; + t.bind_address = bind_address; + t.target_address = multicast_group; + t.port = port; + return UDP::connect(reaction, t); } template diff --git a/src/dsl/word/emit/UDP.hpp b/src/dsl/word/emit/UDP.hpp index 601e4edd7..8df945c14 100644 --- a/src/dsl/word/emit/UDP.hpp +++ b/src/dsl/word/emit/UDP.hpp @@ -21,6 +21,7 @@ #include "../../../PowerPlant.hpp" #include "../../../util/FileDescriptor.hpp" +#include "../../../util/network/if_number_from_address.hpp" #include "../../../util/platform.hpp" #include "../../../util/serialise/Serialise.hpp" #include "../../store/DataStore.hpp" @@ -46,12 +47,10 @@ namespace dsl { * Anything emitted over the UDP network must be serialisable. * * @param data the data to emit - * @param to_addr a string or host endian integer specifying the ip to send the packet to + * @param to_addr a string specifying the address to send this packet to * @param to_port the port to send this packet to in host endian - * @param from_addr Optional. A string or host endian integer specifying the local ip to send the packet - * from. Defaults to INADDR_ANY. - * @param from_port Optional. The port to send this from to in host endian or 0 to automatically choose a - * port. Defaults to 0. + * @param from_addr Optional. The address to send this from or "" to automatically choose an address. + * @param from_port Optional. The port to send this from in host endian or 0 to automatically choose a port. * @tparam DataType the datatype of the object to emit */ template @@ -59,62 +58,93 @@ namespace dsl { static inline void emit(PowerPlant& /*powerplant*/, std::shared_ptr data, - in_addr_t to_addr, + const std::string& to_addr, in_port_t to_port, - in_addr_t from_addr, - in_port_t from_port) { - - sockaddr_in src{}; - sockaddr_in target{}; - std::memset(&src, 0, sizeof(sockaddr_in)); - std::memset(&target, 0, sizeof(sockaddr_in)); - - // Get socket addresses for our source and target - src.sin_family = AF_INET; - src.sin_addr.s_addr = htonl(from_addr); - src.sin_port = htons(from_port); - - target.sin_family = AF_INET; - target.sin_addr.s_addr = htonl(to_addr); - target.sin_port = htons(to_port); - - // Work out if we are sending to a multicast address - const bool multicast = ((to_addr >> 28) == 14); + const std::string& from_addr = "", + in_port_t from_port = 0) { + + std::cout << to_addr << ":" << to_port << std::endl; + + // Resolve our addresses + const util::network::sock_t remote = util::network::resolve(to_addr, to_port); + const bool multicast = + remote.sock.sa_family == AF_INET ? ((remote.ipv4.sin_addr.s_addr >> 28) == 0xE) + : remote.sock.sa_family == AF_INET6 ? ((remote.ipv6.sin6_addr.s6_addr[0] & 0xFF) == 0xFF) + : false; + + // If we are not provided a from address, use any from address + util::network::sock_t local{}; + if (from_addr.empty()) { + local = remote; + switch (local.sock.sa_family) { + case AF_INET: { + local.ipv4.sin_port = htons(from_port); + local.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); + } break; + case AF_INET6: { + local.ipv6.sin6_port = htons(from_port); + local.ipv6.sin6_addr = IN6ADDR_ANY_INIT; + } break; + default: throw std::runtime_error("Unknown socket family"); + } + } + else { + util::network::sock_t local = util::network::resolve(from_addr, from_port); + if (local.sock.sa_family != remote.sock.sa_family) { + throw std::runtime_error("to and from addresses are not the same family"); + } + } // Open a socket to send the datagram from - util::FileDescriptor fd = ::socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + util::FileDescriptor fd = ::socket(local.sock.sa_family, SOCK_DGRAM, IPPROTO_UDP); if (fd < 0) { throw std::system_error(network_errno, std::system_category(), "We were unable to open the UDP socket"); } - // If we need to, bind to a port on our end - if (from_addr != 0 || from_port != 0) { - if (::bind(fd, reinterpret_cast(&src), sizeof(sockaddr)) != 0) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to bind the UDP socket to the port"); + // If we are using multicast and we have a specific from_addr we need to tell the system to use the + // correct interface + if (multicast && !from_addr.empty()) { + if (local.sock.sa_family = AF_INET) { + // Set our transmission interface for the multicast socket + if (::setsockopt(fd, + IPPROTO_IP, + IP_MULTICAST_IF, + reinterpret_cast(&local.ipv4.sin_addr), + sizeof(local.ipv4.sin_addr)) + < 0) { + throw std::system_error(network_errno, + std::system_category(), + "We were unable to use the requested interface for multicast"); + } + } + else if (local.sock.sa_family = AF_INET6) { + // Set our transmission interface for the multicast socket + auto if_number = util::network::if_number_from_address(local.ipv6); + if (::setsockopt(fd, + IPPROTO_IPV6, + IPV6_MULTICAST_IF, + reinterpret_cast(&local.ipv6.sin6_addr), + sizeof(local.ipv6.sin6_addr)) + < 0) { + throw std::system_error(network_errno, + std::system_category(), + "We were unable to use the requested interface for multicast"); + } } } - // If we are using multicast and we have a specific from_addr we need to tell the system to use it - if (multicast && from_addr != 0) { - // Set our transmission interface for the multicast socket - if (::setsockopt(fd, - IPPROTO_IP, - IP_MULTICAST_IF, - reinterpret_cast(&src.sin_addr), - sizeof(src.sin_addr)) - < 0) { + // Bind a local port if requested + if (!from_addr.empty() || from_port != 0) { + if (::bind(fd, &local.sock, local.size()) != 0) { throw std::system_error(network_errno, std::system_category(), - "We were unable to use the requested interface for multicast"); + "We were unable to bind the UDP socket to the port"); } } - // This isn't the greatest code, but lets assume our users don't send broadcasts they don't mean - // to... + // Assume that if the user is sending a broadcast they want to enable broadcasting int yes = 1; if (::setsockopt(fd, SOL_SOCKET, SO_BROADCAST, reinterpret_cast(&yes), sizeof(yes)) < 0) { @@ -131,84 +161,14 @@ namespace dsl { payload.data(), static_cast(payload.size()), 0, - reinterpret_cast(&target), - sizeof(sockaddr_in)) + &remote.sock, + remote.size()) < 0) { throw std::system_error(network_errno, std::system_category(), "We were unable to send the UDP message"); } } - - // String ip addresses - static inline void emit(PowerPlant& powerplant, - std::shared_ptr data, - const std::string& to_addr, - in_port_t to_port, - const std::string& from_addr, - in_port_t from_port) { - - in_addr addr{}; - - ::inet_pton(AF_INET, to_addr.c_str(), &addr); - in_addr_t to = ntohl(addr.s_addr); - - ::inet_pton(AF_INET, from_addr.c_str(), &addr); - in_addr_t from = ntohl(addr.s_addr); - - emit(powerplant, data, to, to_port, from, from_port); - } - - static inline void emit(PowerPlant& powerplant, - std::shared_ptr data, - const std::string& to_addr, - in_port_t to_port, - in_addr_t from_addr, - in_port_t from_port) { - - in_addr addr{}; - - ::inet_pton(AF_INET, to_addr.c_str(), &addr); - in_addr_t to = ntohl(addr.s_addr); - - emit(powerplant, data, to, to_port, from_addr, from_port); - } - - static inline void emit(PowerPlant& powerplant, - std::shared_ptr data, - in_addr_t to_addr, - in_port_t to_port, - const std::string& from_addr, - in_port_t from_port) { - - in_addr addr{}; - - ::inet_pton(AF_INET, from_addr.c_str(), &addr); - in_addr_t from = ntohl(addr.s_addr); - - emit(powerplant, data, to_addr, to_port, from, from_port); - } - - // No from address - static inline void emit(PowerPlant& powerplant, - std::shared_ptr data, - in_addr_t to_addr, - in_port_t to_port) { - emit(powerplant, data, to_addr, to_port, INADDR_ANY, in_port_t(0)); - } - - static inline void emit(PowerPlant& powerplant, - std::shared_ptr data, - const std::string& to_addr, - in_port_t to_port) { - - in_addr addr{}; - - ::inet_pton(AF_INET, to_addr.c_str(), &addr); - in_addr_t to = ntohl(addr.s_addr); - - emit(powerplant, data, to, to_port, INADDR_ANY, in_port_t(0)); - } }; } // namespace emit diff --git a/src/util/network/if_number_from_address.cpp b/src/util/network/if_number_from_address.cpp new file mode 100644 index 000000000..4c75acb75 --- /dev/null +++ b/src/util/network/if_number_from_address.cpp @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "if_number_from_address.hpp" + +#include +#include + +#include "../platform.hpp" +#include "get_interfaces.hpp" + +namespace NUClear { +namespace util { + namespace network { + + int if_number_from_address(const sockaddr_in6& ipv6) { + + // If the socket is referring to the any address, return 0 to use the default interface + if (std::all_of(ipv6.sin6_addr.s6_addr, ipv6.sin6_addr.s6_addr + 16, [](unsigned char i) { + return i == 0; + })) { + return 0; + } + + // Find the correct interface to join on (the one that has our bind address) + for (auto& iface : get_interfaces()) { + // iface must be, ipv6, multicast, and have the same address as our bind address + if (iface.ip.sock.sa_family == AF_INET6 && iface.flags.multicast + && ::memcmp(iface.ip.ipv6.sin6_addr.s6_addr, ipv6.sin6_addr.s6_addr, sizeof(in6_addr)) == 0) { + + // Get the interface for this + return ::if_nametoindex(iface.name.c_str()); + } + } + + // If we get here then we couldn't find an interface + throw std::runtime_error("Could not find interface for address"); + } + + } // namespace network +} // namespace util +} // namespace NUClear diff --git a/src/util/network/if_number_from_address.hpp b/src/util/network/if_number_from_address.hpp new file mode 100644 index 000000000..0ae958f61 --- /dev/null +++ b/src/util/network/if_number_from_address.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef NUCLEAR_UTIL_NETWORK_IF_NUMBER_FROM_ADDRESS_HPP +#define NUCLEAR_UTIL_NETWORK_IF_NUMBER_FROM_ADDRESS_HPP + +#include +#include + +#include "sock_t.hpp" + +namespace NUClear { +namespace util { + namespace network { + + /** + * @brief Gets the index of the interface that the given ipv6 address is on + * + * @param ipv6 the ipv6 address to check + * + * @return int the index of the interface that the address is on or 0 if the ipv6 address is the any address + */ + int if_number_from_address(const sockaddr_in6& ipv6); + + } // namespace network +} // namespace util +} // namespace NUClear + +#endif // NUCLEAR_UTIL_NETWORK_IF_NUMBER_FROM_ADDRESS_HPP diff --git a/src/util/network/resolve.cpp b/src/util/network/resolve.cpp index 9b5943bee..c51d89f7d 100644 --- a/src/util/network/resolve.cpp +++ b/src/util/network/resolve.cpp @@ -19,6 +19,7 @@ #include "resolve.hpp" #include +#include #include #include @@ -30,15 +31,16 @@ namespace util { NUClear::util::network::sock_t resolve(const std::string& address, const uint16_t& port) { addrinfo hints{}; - memset(&hints, 0, sizeof hints); // make sure the struct is empty - hints.ai_family = AF_UNSPEC; // don't care about IPv4 or IPv6 - hints.ai_socktype = SOCK_STREAM; // using a tcp stream + hints.ai_family = AF_UNSPEC; // don't care about IPv4 or IPv6 + hints.ai_socktype = AF_UNSPEC; // don't care about TCP or UDP // Get our info on this address addrinfo* servinfo_ptr = nullptr; if (::getaddrinfo(address.c_str(), std::to_string(port).c_str(), &hints, &servinfo_ptr) != 0) { - throw std::runtime_error("Failed to get address information for " + address + ":" - + std::to_string(port)); + throw std::system_error( + network_errno, + std::system_category(), + "Failed to get address information for " + address + ":" + std::to_string(port)); } // Check if we have any addresses to work with diff --git a/src/util/network/sock_t.hpp b/src/util/network/sock_t.hpp index 75ef3b1c3..e0b37e55c 100644 --- a/src/util/network/sock_t.hpp +++ b/src/util/network/sock_t.hpp @@ -19,6 +19,7 @@ #ifndef NUCLEAR_UTIL_NETWORK_SOCK_T_HPP #define NUCLEAR_UTIL_NETWORK_SOCK_T_HPP +#include #include #include "../platform.hpp" @@ -35,7 +36,7 @@ namespace util { sockaddr_in6 ipv6; }; - socklen_t size() const { + inline socklen_t size() const { switch (sock.sa_family) { case AF_INET: return sizeof(sockaddr_in); case AF_INET6: return sizeof(sockaddr_in6); @@ -44,6 +45,21 @@ namespace util { + std::to_string(sock.sa_family)); } } + + inline std::pair address() const { + std::array c = {0}; + switch (sock.sa_family) { + case AF_INET: + return std::make_pair(::inet_ntop(sock.sa_family, &ipv4.sin_addr, c.data(), c.size()), + ntohs(ipv4.sin_port)); + case AF_INET6: + return std::make_pair(::inet_ntop(sock.sa_family, &ipv6.sin6_addr, c.data(), c.size()), + ntohs(ipv6.sin6_port)); + default: + throw std::runtime_error("Cannot get address for socket address family " + + std::to_string(sock.sa_family)); + } + } }; } // namespace network diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 7709ea4e4..8437e8d4b 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -26,14 +26,16 @@ namespace { /// @brief Events that occur during the test std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -constexpr in_port_t PORT = 40009; - -// NOLINTNEXTLINE(cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) -const std::string TEST_STRING = "Hello TCP World!"; +enum TestPorts { + KNOWN_V4_PORT = 40010, + KNOWN_V6_PORT = 40011, +}; struct TestConnection { - TestConnection(std::string name, in_port_t port) : name(std::move(name)), port(port) {} + TestConnection(std::string name, std::string address, in_port_t port) + : name(std::move(name)), address(std::move(address)), port(port) {} std::string name; + std::string address; in_port_t port; }; @@ -50,7 +52,7 @@ class TestReactor : public test_util::TestBase { // Read into the buffer std::array buff{}; - const ssize_t len = ::recv(event.fd, buff.data(), socklen_t(TEST_STRING.size()), 0); + const ssize_t len = ::recv(event.fd, buff.data(), socklen_t(buff.size()), 0); if (len != 0) { events.push_back(name + " received: " + std::string(buff.data(), len)); ::send(event.fd, buff.data(), socklen_t(len), 0); @@ -65,65 +67,87 @@ class TestReactor : public test_util::TestBase { TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { - // Bind to a known port - on(PORT).then([this](const TCP::Connection& connection) { + // Bind to IPv4 and a known port + on(KNOWN_V4_PORT).then([this](const TCP::Connection& connection) { + on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { + handle_data("v4 Known", event); + }); + }); + + // Bind to IPv4 an unknown port and get the port number + auto v4 = on().then([this](const TCP::Connection& connection) { on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { - handle_data("Known Port", event); + handle_data("v4 Ephemeral", event); }); }); + in_port_t v4_port = std::get<1>(v4); - // Bind to an unknown port and get the port number - in_port_t ephemeral_port = 0; - std::tie(std::ignore, ephemeral_port, std::ignore) = on().then([this](const TCP::Connection& connection) { + // Bind to IPv6 and a known port + on(KNOWN_V6_PORT, "::1").then([this](const TCP::Connection& connection) { on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { - handle_data("Ephemeral Port", event); + handle_data("v6 Known", event); }); }); + // Bind to IPv6 an unknown port and get the port number + auto v6 = on(0, "::1").then([this](const TCP::Connection& connection) { + on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { + handle_data("v6 Ephemeral", event); + }); + }); + in_port_t v6_port = std::get<1>(v6); + // Send a test message to the known port on, Sync>().then([](const TestConnection& target) { + // Resolve the target address + NUClear::util::network::sock_t address = NUClear::util::network::resolve(target.address, target.port); + // Open a random socket - NUClear::util::FileDescriptor fd(::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), + NUClear::util::FileDescriptor fd(::socket(address.sock.sa_family, SOCK_STREAM, IPPROTO_TCP), [](NUClear::fd_t fd) { ::shutdown(fd, SHUT_RDWR); }); if (!fd.valid()) { throw std::runtime_error("Failed to create socket"); } - // Our address to our local connection - sockaddr_in address{}; - address.sin_family = AF_INET; - address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - address.sin_port = htons(target.port); - // Connect to ourself - if (::connect(fd, reinterpret_cast(&address), sizeof(address)) != 0) { + if (::connect(fd, &address.sock, address.size()) != 0) { throw std::runtime_error("Failed to connect to socket"); } // Write on our socket events.push_back(target.name + " sending"); - ::send(fd, TEST_STRING.data(), socklen_t(TEST_STRING.size()), 0); + ::send(fd, target.name.data(), socklen_t(target.name.size()), 0); // Receive the echo std::array buff{}; - const ssize_t recv = ::recv(fd, buff.data(), socklen_t(TEST_STRING.size()), 0); + const ssize_t recv = ::recv(fd, buff.data(), socklen_t(target.name.size()), 0); events.push_back(target.name + " echoed: " + std::string(buff.data(), recv)); }); - on, Sync>().then([this, ephemeral_port](const Finished& test) { - if (test.name == "Known Port") { - emit(std::make_unique("Ephemeral Port", ephemeral_port)); + on, Sync>().then([=](const Finished& test) { + if (test.name == "Startup") { + emit(std::make_unique("v4 Known", "127.0.0.1", KNOWN_V4_PORT)); + } + if (test.name == "v4 Known") { + emit(std::make_unique("v4 Ephemeral", "127.0.0.1", v4_port)); + } + else if (test.name == "v4 Ephemeral") { + std::cout << "Testing v6 known" << std::endl; + emit(std::make_unique("v6 Known", "::1", KNOWN_V6_PORT)); + } + else if (test.name == "v6 Known") { + emit(std::make_unique("v6 Ephemeral", "::1", v6_port)); } - else if (test.name == "Ephemeral Port") { + else if (test.name == "v6 Ephemeral") { events.push_back("Finishing Test"); powerplant.shutdown(); } }); on().then([this] { - // Start the first test - emit(std::make_unique("Known Port", PORT)); + // Start the first test by emitting a "finished" startup + emit(std::make_unique("Startup")); }); } @@ -142,14 +166,22 @@ TEST_CASE("Testing listening for TCP connections and receiving data messages", " plant.start(); const std::vector expected = { - "Known Port sending", - "Known Port received: Hello TCP World!", - "Known Port echoed: Hello TCP World!", - "Known Port closed", - "Ephemeral Port sending", - "Ephemeral Port received: Hello TCP World!", - "Ephemeral Port echoed: Hello TCP World!", - "Ephemeral Port closed", + "v4 Known sending", + "v4 Known received: v4 Known", + "v4 Known echoed: v4 Known", + "v4 Known closed", + "v4 Ephemeral sending", + "v4 Ephemeral received: v4 Ephemeral", + "v4 Ephemeral echoed: v4 Ephemeral", + "v4 Ephemeral closed", + "v6 Known sending", + "v6 Known received: v6 Known", + "v6 Known echoed: v6 Known", + "v6 Known closed", + "v6 Ephemeral sending", + "v6 Ephemeral received: v6 Ephemeral", + "v6 Ephemeral echoed: v6 Ephemeral", + "v6 Ephemeral closed", "Finishing Test", }; diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index f5b1c9216..06f6fcc72 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -23,67 +23,205 @@ namespace { -constexpr uint16_t PORT = 40000; -const std::string TEST_STRING = "Hello UDP World!"; // NOLINT(cert-err58-cpp) -bool received_a = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -bool received_b = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +/// @brief Events that occur during the test +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +enum TestPorts { + UNICAST_V4 = 40000, + UNICAST_V6 = 40001, + BROADCAST_V4 = 40002, + MULTICAST_V4 = 40003, + MULTICAST_V6 = 40004, +}; -struct Message {}; +constexpr in_port_t BASE_PORT = 40000; +const std::string IPV4_MULTICAST_ADDRESS = "230.12.3.22"; // NOLINT(cert-err58-cpp) +const std::string IPV6_MULTICAST_ADDRESS = "ff02::230:12:3:22"; // NOLINT(cert-err58-cpp) + +inline std::string get_broadcast_addr() { + // Get the first IPv4 broadcast address we can find + std::array buff{}; + bool found = false; + for (const auto& iface : NUClear::util::network::get_interfaces()) { + if (iface.ip.sock.sa_family == AF_INET && iface.flags.broadcast) { + ::inet_ntop(AF_INET, &iface.broadcast.ipv4.sin_addr, buff.data(), buff.size()); + found = true; + break; + } + } + if (!found) { + throw std::runtime_error("No broadcast address found"); + } + return std::string(buff.data()); +} + +struct TestUDP { + TestUDP(std::string name, std::string address, in_port_t port) + : name(std::move(name)), address(std::move(address)), port(port) {} + std::string name; + std::string address; + in_port_t port; +}; + +struct Finished { + Finished(std::string name) : name(std::move(name)) {} + std::string name; +}; class TestReactor : public test_util::TestBase { +private: + void handle_data(const std::string& name, const UDP::Packet& packet) { + std::string data(packet.payload.data(), packet.payload.size()); + + // Convert IP address to string in dotted decimal format + std::string local = packet.local.address + ":" + std::to_string(packet.local.port); + + events.push_back(name + " <- " + data + " (" + local + ")"); + + if (data == name) { + emit(std::make_unique(name)); + } + } + public: TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { - // Known port - on(PORT).then([this](const UDP::Packet& packet) { - // Check that the data we received is correct - REQUIRE(packet.remote.address == INADDR_LOOPBACK); - REQUIRE(packet.payload.size() == TEST_STRING.size()); - REQUIRE(std::memcmp(packet.payload.data(), TEST_STRING.data(), TEST_STRING.size()) == 0); + // Get the first IPv4 broadcast address we can find + std::string broadcast_addr = get_broadcast_addr(); - received_a = true; - if (received_a && received_b) { - // Shutdown we are done with the test - powerplant.shutdown(); - } + // IPv4 Unicast Known port + on(UNICAST_V4).then([this](const UDP::Packet& packet) { // + handle_data("Uv4 K", packet); }); - // Unknown port - in_port_t bound_port = 0; - std::tie(std::ignore, bound_port, std::ignore) = on().then([this](const UDP::Packet& packet) { - // Check that the data we received is correct - REQUIRE(packet.remote.address == INADDR_LOOPBACK); - REQUIRE(packet.payload.size() == TEST_STRING.size()); - REQUIRE(std::memcmp(packet.payload.data(), TEST_STRING.data(), TEST_STRING.size()) == 0); - - received_b = true; - if (received_a && received_b) { - // Shutdown we are done with the test - powerplant.shutdown(); - } + // IPv4 Unicast Ephemeral port + auto uni_v4 = on().then([this](const UDP::Packet& packet) { // + handle_data("Uv4 E", packet); + }); + in_port_t uni_v4_port = std::get<1>(uni_v4); + + // IPv6 Unicast Known port + on(UNICAST_V6, "::1").then([this](const UDP::Packet& packet) { // + handle_data("Uv6 K", packet); + }); + + // IPv6 Unicast Ephemeral port + auto uni_v6 = on(0, "::1").then([this](const UDP::Packet& packet) { // + handle_data("Uv6 E", packet); + }); + in_port_t uni_v6_port = std::get<1>(uni_v6); + + // IPv4 Broadcast Known port + on(BROADCAST_V4).then([this](const UDP::Packet& packet) { // + handle_data("Bv4 K", packet); + }); + + // IPv4 Broadcast Ephemeral port + auto broad_v4 = on().then([this](const UDP::Packet& packet) { // + handle_data("Bv4 E", packet); + }); + in_port_t broad_v4_port = std::get<1>(broad_v4); + + // No such thing as broadcast in IPv6 + + // IPv4 Multicast Known port + on(IPV4_MULTICAST_ADDRESS, MULTICAST_V4).then([this](const UDP::Packet& packet) { // + handle_data("Mv4 K", packet); + }); + + // IPv4 Multicast Ephemeral port + auto multi_v4 = on(IPV4_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { // + handle_data("Mv4 E", packet); }); + in_port_t multi_v4_port = std::get<1>(multi_v4); - // Send a test for a known port - // does not need to include the port in the lambda capture. This is a global variable to the unit test, so - // the function will have access to it. - on>().then([this] { // - emit(std::make_unique(TEST_STRING), INADDR_LOOPBACK, PORT); + // IPv6 Multicast Known port + on(IPV6_MULTICAST_ADDRESS, MULTICAST_V6).then([this](const UDP::Packet& packet) { // + handle_data("Mv6 K", packet); }); + // IPv6 Multicast Ephemeral port + auto multi_v6 = on(IPV6_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { // + handle_data("Mv6 E", packet); + }); + in_port_t multi_v6_port = std::get<1>(multi_v6); + + // Send a test message to the known port + on>().then([this](const TestUDP& target) { + events.push_back(" -> " + target.address + ":" + std::to_string(target.port)); + emit(std::make_unique(target.name), target.address, target.port); + }); - // Send a test for an unknown port - // needs to include the bound_port in the lambda capture, so that the function will have access to bound_port. - on>().then([this, bound_port] { - // Emit our UDP message - emit(std::make_unique(TEST_STRING), INADDR_LOOPBACK, bound_port); + on>().then([=](const Finished& test) { + events.push_back("--------------------"); + + if (test.name == "Startup") { + // Send some invalid packets and a valid one + emit(std::make_unique("Bv4 I", broadcast_addr, UNICAST_V4)); + emit(std::make_unique("Uv6 I", "::1", UNICAST_V4)); + emit(std::make_unique("Uv4 K", "127.0.0.1", UNICAST_V4)); + } + else if (test.name == "Uv4 K") { + // Send some invalid packets and a valid one + emit(std::make_unique("Bv4 I", broadcast_addr, uni_v4_port)); + emit(std::make_unique("Uv6 I", "::1", uni_v4_port)); + emit(std::make_unique("Uv4 E", "127.0.0.1", uni_v4_port)); + } + else if (test.name == "Uv4 E") { + emit(std::make_unique("Bv4 I", broadcast_addr, UNICAST_V6)); + emit(std::make_unique("Uv4 I", "127.0.0.1", UNICAST_V6)); + emit(std::make_unique("Uv6 K", "::1", UNICAST_V6)); + } + else if (test.name == "Uv6 K") { + emit(std::make_unique("Bv4 I", broadcast_addr, uni_v6_port)); + emit(std::make_unique("Uv4 I", "127.0.0.1", uni_v6_port)); + emit(std::make_unique("Uv6 E", "::1", uni_v6_port)); + } + else if (test.name == "Uv6 E") { + // Send a unicast and broadcast (only the broadcast should be received) + emit(std::make_unique("Uv4 I", "127.0.0.1", BROADCAST_V4)); + emit(std::make_unique("Uv6 E", "::1", BROADCAST_V4)); + emit(std::make_unique("Bv4 K", broadcast_addr, BROADCAST_V4)); + } + else if (test.name == "Bv4 K") { + emit(std::make_unique("Uv4 I", "127.0.0.1", broad_v4_port)); + emit(std::make_unique("Uv6 E", "::1", broad_v4_port)); + emit(std::make_unique("Bv4 E", broadcast_addr, broad_v4_port)); + } + else if (test.name == "Bv4 E") { + emit(std::make_unique("Uv4 I", "127.0.0.1", MULTICAST_V4)); + emit(std::make_unique("Bv4 I", broadcast_addr, MULTICAST_V4)); + emit(std::make_unique("Mv4 K", IPV4_MULTICAST_ADDRESS, MULTICAST_V4)); + } + else if (test.name == "Mv4 K") { + emit(std::make_unique("Uv4 I", "127.0.0.1", multi_v4_port)); + emit(std::make_unique("Bv4 I", broadcast_addr, multi_v4_port)); + emit(std::make_unique("Mv4 E", IPV4_MULTICAST_ADDRESS, multi_v4_port)); + } + else if (test.name == "Mv4 E") { + emit(std::make_unique("Uv6 I", "::1", MULTICAST_V6)); + emit(std::make_unique("Mv6 K", IPV6_MULTICAST_ADDRESS, MULTICAST_V6)); + } + else if (test.name == "Mv6 K") { + emit(std::make_unique("Uv6 I", "::1", multi_v6_port)); + emit(std::make_unique("Mv6 E", IPV6_MULTICAST_ADDRESS, multi_v6_port)); + } + else if (test.name == "Mv6 E") { + // We are done, so stop the reactor + powerplant.shutdown(); + } + else { + FAIL("Unknown test name"); + } }); on().then([this] { - // Emit a message just so it will be when everything is running - emit(std::make_unique()); + // Start the first test by emitting a "finished" event + emit(std::make_unique("Startup")); }); } }; + } // namespace TEST_CASE("Testing sending and receiving of UDP messages", "[api][network][udp]") { @@ -92,6 +230,13 @@ TEST_CASE("Testing sending and receiving of UDP messages", "[api][network][udp]" config.thread_count = 1; NUClear::PowerPlant plant(config); plant.install(); - plant.start(); + + const std::vector expected = {}; + + // Make an info print the diff in an easy to read way if we fail + INFO(test_util::diff_string(expected, events)); + + // Check the events fired in order and only those events + REQUIRE(events == expected); } diff --git a/tests/dsl/UDPBroadcast.cpp b/tests/dsl/UDPBroadcast.cpp deleted file mode 100644 index 2c960c1ba..000000000 --- a/tests/dsl/UDPBroadcast.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include "test_util/TestBase.hpp" - -namespace { - -constexpr uint16_t PORT = 40001; -const std::string TEST_STRING = "Hello UDP Broadcast World!"; // NOLINT(cert-err58-cpp) -std::size_t count_a = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -std::size_t count_b = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -std::size_t num_addresses = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -struct Message {}; - -class TestReactor : public test_util::TestBase { -public: - in_port_t bound_port = 0; - TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { - - // Known port - on(PORT).then([this](const UDP::Packet& packet) { - ++count_a; - - // Check that the data we received is correct - REQUIRE(packet.payload.size() == TEST_STRING.size()); - REQUIRE(std::memcmp(packet.payload.data(), TEST_STRING.data(), TEST_STRING.size()) == 0); - - // Shutdown we are done with the test - if (count_a == num_addresses && count_b == num_addresses) { - powerplant.shutdown(); - } - }); - - // Unknown port - std::tie(std::ignore, bound_port, std::ignore) = on().then([this](const UDP::Packet& packet) { - ++count_b; - - // Check that the data we received is correct - REQUIRE(packet.payload.size() == TEST_STRING.size()); - REQUIRE(std::memcmp(packet.payload.data(), TEST_STRING.data(), TEST_STRING.size()) == 0); - - // Shutdown we are done with the test - if (count_a == num_addresses && count_b == num_addresses) { - powerplant.shutdown(); - } - }); - - on>().then([this] { - // Get all the network interfaces - auto interfaces = NUClear::util::network::get_interfaces(); - - std::vector addresses; - - for (auto& iface : interfaces) { - // We send on broadcast addresses and we don't want loopback or point to point - if (iface.broadcast.sock.sa_family == AF_INET && iface.flags.broadcast) { - auto& i = *reinterpret_cast(&iface.broadcast); - - // Two broadcast ips that are the same are probably on the same network so ignore those - if (std::find(std::begin(addresses), std::end(addresses), ntohl(i.sin_addr.s_addr)) - == std::end(addresses)) { - - addresses.push_back(ntohl(i.sin_addr.s_addr)); - } - } - } - - num_addresses = addresses.size(); - - for (auto& ad : addresses) { - - // Send our message to that broadcast address - emit(std::make_unique(TEST_STRING), ad, PORT); - } - }); - - on>().then([this] { - // Get all the network interfaces - auto interfaces = NUClear::util::network::get_interfaces(); - - std::vector addresses; - - for (auto& iface : interfaces) { - // We send on broadcast addresses and we don't want loopback or point to point - if (iface.broadcast.sock.sa_family == AF_INET && iface.flags.broadcast) { - auto& i = *reinterpret_cast(&iface.broadcast); - // Two broadcast ips that are the same are probably on the same network so ignore those - if (std::find(std::begin(addresses), std::end(addresses), ntohl(i.sin_addr.s_addr)) - == std::end(addresses)) { - - addresses.push_back(ntohl(i.sin_addr.s_addr)); - } - } - } - - num_addresses = addresses.size(); - - for (auto& ad : addresses) { - - // Send our message to that broadcast address - emit(std::make_unique(TEST_STRING), ad, bound_port); - } - }); - - on().then([this] { - // Emit a message just so it will be when everything is running - emit(std::make_unique()); - }); - } -}; -} // namespace - -TEST_CASE("Testing sending and receiving of UDP Broadcast messages", "[api][network][udp][broadcast]") { - - NUClear::PowerPlant::Configuration config; - config.thread_count = 1; - NUClear::PowerPlant plant(config); - plant.install(); - - plant.start(); - - REQUIRE(count_a == num_addresses); - REQUIRE(count_b == num_addresses); -} diff --git a/tests/dsl/UDPMulticastKnownPort.cpp b/tests/dsl/UDPMulticastKnownPort.cpp deleted file mode 100644 index 7fbf82c6a..000000000 --- a/tests/dsl/UDPMulticastKnownPort.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include "test_util/TestBase.hpp" - -namespace { - -constexpr in_port_t PORT = 40002; -const std::string TEST_STRING = "Hello UDP Multicast World!"; // NOLINT(cert-err58-cpp) -const std::string MULTICAST_ADDRESS = "230.12.3.21"; // NOLINT(cert-err58-cpp) -std::size_t count = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -std::size_t num_addresses = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -struct Message {}; - -class TestReactor : public test_util::TestBase { -public: - bool shutdown_flag = false; - - TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { - - // Terminates the test if it takes too long - longer than 200 ms since this reaction first runs - on>().then([this]() { - if (shutdown_flag) { - powerplant.shutdown(); - } - shutdown_flag = true; - }); - - // Known port - on(MULTICAST_ADDRESS, PORT).then([this](const UDP::Packet& packet) { - ++count; - // Check that the data we received is correct - REQUIRE(packet.payload.size() == TEST_STRING.size()); - REQUIRE(std::memcmp(packet.payload.data(), TEST_STRING.data(), TEST_STRING.size()) == 0); - - // Shutdown if we have succeeded - if (count >= num_addresses) { - powerplant.shutdown(); - } - }); - - // Test with known port - on>().then([this] { - // Get all the network interfaces - auto interfaces = NUClear::util::network::get_interfaces(); - - std::vector addresses; - - for (auto& iface : interfaces) { - // We send on multicast capable addresses - if (iface.broadcast.sock.sa_family == AF_INET && iface.flags.multicast) { - auto& i = *reinterpret_cast(&iface.ip); - auto& b = *reinterpret_cast(&iface.broadcast); - - // Two broadcast ips that are the same are probably on the same network so ignore those - if (std::find(std::begin(addresses), std::end(addresses), ntohl(b.sin_addr.s_addr)) - == std::end(addresses)) { - addresses.push_back(ntohl(i.sin_addr.s_addr)); - } - } - } - - num_addresses = addresses.size(); - - for (auto& ad : addresses) { - - // Send our message to that broadcast address - emit(std::make_unique(TEST_STRING), MULTICAST_ADDRESS, PORT, ad, in_port_t(0)); - } - }); - - on().then([this] { - // Emit a message to start the test - emit(std::make_unique()); - }); - } -}; -} // namespace - -TEST_CASE("Testing sending and receiving of UDP Multicast messages with a known port", - "[api][network][udp][multicast][known_port]") { - - NUClear::PowerPlant::Configuration config; - config.thread_count = 1; - NUClear::PowerPlant plant(config); - plant.install(); - - plant.start(); - - REQUIRE(count == num_addresses); -} diff --git a/tests/dsl/UDPMulticastUnknownPort.cpp b/tests/dsl/UDPMulticastUnknownPort.cpp deleted file mode 100644 index 505365b34..000000000 --- a/tests/dsl/UDPMulticastUnknownPort.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include "test_util/TestBase.hpp" - -namespace { - -const std::string TEST_STRING = "Hello UDP Multicast World!"; // NOLINT(cert-err58-cpp) -const std::string MULTICAST_ADDRESS = "230.12.3.22"; // NOLINT(cert-err58-cpp) -std::size_t count = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -std::size_t num_addresses = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -struct Message {}; - -class TestReactor : public test_util::TestBase { -public: - bool shutdown_flag = false; - - TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { - - // Terminates the test if it takes too long - longer than 200 ms since this reaction first runs - on>().then([this]() { - if (shutdown_flag) { - powerplant.shutdown(); - } - shutdown_flag = true; - }); - - // Unknown port - in_port_t bound_port = 0; - std::tie(std::ignore, bound_port, std::ignore) = - on(MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { - ++count; - // Check that the data we received is correct - REQUIRE(packet.payload.size() == TEST_STRING.size()); - REQUIRE(std::memcmp(packet.payload.data(), TEST_STRING.data(), TEST_STRING.size()) == 0); - - // Shutdown if we have succeeded - if (count >= num_addresses) { - powerplant.shutdown(); - } - }); - - // Test with port given to us - on>().then([this, bound_port] { - // Get all the network interfaces - auto interfaces = NUClear::util::network::get_interfaces(); - - std::vector addresses; - - for (auto& iface : interfaces) { - // We send on multicast capable addresses - if (iface.broadcast.sock.sa_family == AF_INET && iface.flags.multicast) { - auto& i = *reinterpret_cast(&iface.ip); - auto& b = *reinterpret_cast(&iface.broadcast); - - // Two broadcast ips that are the same are probably on the same network so ignore those - if (std::find(std::begin(addresses), std::end(addresses), ntohl(b.sin_addr.s_addr)) - == std::end(addresses)) { - addresses.push_back(ntohl(i.sin_addr.s_addr)); - } - } - } - - num_addresses = addresses.size(); - - for (auto& ad : addresses) { - - // Send our message to that broadcast address - emit(std::make_unique(TEST_STRING), - MULTICAST_ADDRESS, - bound_port, - ad, - in_port_t(0)); - } - }); - - on().then([this] { - // Emit a message to start the test - emit(std::make_unique()); - }); - } -}; -} // namespace - -TEST_CASE("Testing sending and receiving of UDP Multicast messages with an unknown port", - "[api][network][udp][multicast][unknown_port]") { - - NUClear::PowerPlant::Configuration config; - config.thread_count = 1; - NUClear::PowerPlant plant(config); - plant.install(); - - plant.start(); - - REQUIRE(count == num_addresses); -} diff --git a/tests/dsl/emit/UDP.cpp b/tests/dsl/emit/UDP.cpp deleted file mode 100644 index df4e09f12..000000000 --- a/tests/dsl/emit/UDP.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (C) 2013 Trent Houliston , Jake Woods - * 2014-2017 Trent Houliston - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include "../../test_util/TestBase.hpp" - -// Anonymous namespace to keep everything file local -namespace { - -int received_messages = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) - -class TestReactor : public test_util::TestBase { -public: - in_port_t bound_port = 0; - - TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { - emit(std::make_unique(5)); - - std::tie(std::ignore, bound_port, std::ignore) = on().then([this](const UDP::Packet& packet) { - ++received_messages; - - switch (packet.payload.front()) { - case 'a': - case 'b': - REQUIRE(packet.remote.address == INADDR_LOOPBACK); - REQUIRE(packet.local.address == INADDR_LOOPBACK); - REQUIRE(packet.local.port == bound_port); - break; - case 'c': - REQUIRE(packet.remote.address == INADDR_LOOPBACK); - REQUIRE(packet.remote.port == 12345); - REQUIRE(packet.local.address == INADDR_LOOPBACK); - REQUIRE(packet.local.port == bound_port); - break; - case 'd': - REQUIRE(packet.remote.address == INADDR_LOOPBACK); - REQUIRE(packet.remote.port == 54321); - REQUIRE(packet.local.address == INADDR_LOOPBACK); - REQUIRE(packet.local.port == bound_port); - break; - } - - if (received_messages == 4) { - powerplant.shutdown(); - } - }); - - on().then([this] { - // Send using a string - emit(std::make_unique('a'), "127.0.0.1", bound_port); - emit(std::make_unique('b'), INADDR_LOOPBACK, bound_port); - emit(std::make_unique('c'), "127.0.0.1", bound_port, INADDR_ANY, in_port_t(12345)); - emit(std::make_unique('d'), INADDR_LOOPBACK, bound_port, INADDR_ANY, in_port_t(54321)); - }); - } -}; -} // namespace - -TEST_CASE("Testing UDP emits work correctly", "[api][emit][udp]") { - NUClear::PowerPlant::Configuration config; - config.thread_count = 1; - NUClear::PowerPlant plant(config); - plant.install(); - - plant.start(); -} From f23793146adcbb9bffa566e45d55b5a4f9775790 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 22 Aug 2023 22:42:13 +1000 Subject: [PATCH 117/176] add unit tests for resolve --- tests/CMakeLists.txt | 2 +- tests/util/network/resolve.cpp | 116 +++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 tests/util/network/resolve.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 491af31c0..6b3efd92d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -31,7 +31,7 @@ if(CATCH_FOUND) add_compile_definitions(CATCH_CONFIG_CONSOLE_WIDTH=120) - file(GLOB test_src test.cpp "api/*.cpp" "dsl/*.cpp" "dsl/emit/*.cpp" "log/*.cpp" "test_util/*.cpp") + file(GLOB test_src test.cpp "api/*.cpp" "dsl/*.cpp" "dsl/emit/*.cpp" "log/*.cpp" "util/network/*.cpp" "test_util/*.cpp") # Some tests must be executed as individual binaries file(GLOB individual_tests "${CMAKE_CURRENT_SOURCE_DIR}/individual/*.cpp") diff --git a/tests/util/network/resolve.cpp b/tests/util/network/resolve.cpp new file mode 100644 index 000000000..2f48ad06c --- /dev/null +++ b/tests/util/network/resolve.cpp @@ -0,0 +1,116 @@ +#include "util/network/resolve.hpp" + +#include + +TEST_CASE("resolve function returns expected socket address", "[util][network][resolve]") { + + SECTION("IPv4 address") { + std::string address = "127.0.0.1"; + uint16_t port = 80; + + auto result = NUClear::util::network::resolve(address, port); + + REQUIRE(result.sock.sa_family == AF_INET); + REQUIRE(ntohs(result.ipv4.sin_port) == port); + REQUIRE(ntohl(result.ipv4.sin_addr.s_addr) == INADDR_LOOPBACK); + } + + SECTION("IPv6 address") { + std::string address = "::1"; + uint16_t port = 80; + + auto result = NUClear::util::network::resolve(address, port); + + REQUIRE(result.sock.sa_family == AF_INET6); + REQUIRE(ntohs(result.ipv6.sin6_port) == port); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[0]) == 0); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[1]) == 0); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[2]) == 0); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[3]) == 1); + } + + SECTION("Hostname") { + std::string address = "localhost"; + uint16_t port = 80; + + auto result = NUClear::util::network::resolve(address, port); + + // Check that the returned socket address matches the expected address and port + REQUIRE((result.sock.sa_family == AF_INET || result.sock.sa_family == AF_INET6)); + + // Localhost could return an ipv4 or ipv6 address + if (result.sock.sa_family == AF_INET) { + REQUIRE(ntohs(result.ipv4.sin_port) == port); + REQUIRE(ntohl(result.ipv4.sin_addr.s_addr) == INADDR_LOOPBACK); + } + else { + REQUIRE(ntohs(result.ipv6.sin6_port) == port); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[0]) == 0); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[1]) == 0); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[2]) == 0); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[3]) == 1); + } + } + + SECTION("IPv4 address with leading zeros") { + std::string address = "127.000.000.001"; + uint16_t port = 80; + + auto result = NUClear::util::network::resolve(address, port); + + REQUIRE(result.sock.sa_family == AF_INET); + REQUIRE(ntohs(result.ipv4.sin_port) == port); + REQUIRE(ntohl(result.ipv4.sin_addr.s_addr) == INADDR_LOOPBACK); + } + + SECTION("IPv6 address with mixed case letters") { + std::string address = "2001:0DB8:Ac10:FE01:0000:0000:0000:0000"; + uint16_t port = 80; + + auto result = NUClear::util::network::resolve(address, port); + + REQUIRE(result.sock.sa_family == AF_INET6); + REQUIRE(ntohs(result.ipv6.sin6_port) == port); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[0]) == 0x20010db8); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[1]) == 0xac10fe01); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[2]) == 0); + REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[3]) == 0); + } + + SECTION("Hostname with valid IPv4 address") { + std::string address = "ipv4.google.com"; + uint16_t port = 80; + + auto result = NUClear::util::network::resolve(address, port); + + REQUIRE(result.sock.sa_family == AF_INET); + REQUIRE(ntohs(result.ipv4.sin_port) == port); + REQUIRE(ntohl(result.ipv4.sin_addr.s_addr) != 0); + } + + SECTION("Hostname with valid IPv6 address") { + std::string address = "ipv6.google.com"; + uint16_t port = 80; + + auto result = NUClear::util::network::resolve(address, port); + + REQUIRE(result.sock.sa_family == AF_INET6); + REQUIRE(ntohs(result.ipv6.sin6_port) == port); + + // Check if all compoments are zero + bool nonzero = false; + for (int i = 0; i < 4; i++) { + nonzero |= (ntohl(result.ipv6.sin6_addr.s6_addr32[i]) != 0); + } + + REQUIRE(nonzero); + } + + SECTION("Invalid address") { + std::string address = "notahost"; + uint16_t port = 12345; + + // Check that the function throws a std::runtime_error with the appropriate message + REQUIRE_THROWS(NUClear::util::network::resolve(address, port)); + } +} From a725768d2587aa09d3cacb606a29aa52554489f6 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 22 Aug 2023 22:46:14 +1000 Subject: [PATCH 118/176] Remove couts --- src/dsl/word/emit/UDP.hpp | 2 -- tests/dsl/TCP.cpp | 1 - 2 files changed, 3 deletions(-) diff --git a/src/dsl/word/emit/UDP.hpp b/src/dsl/word/emit/UDP.hpp index 8df945c14..a3a7804c5 100644 --- a/src/dsl/word/emit/UDP.hpp +++ b/src/dsl/word/emit/UDP.hpp @@ -63,8 +63,6 @@ namespace dsl { const std::string& from_addr = "", in_port_t from_port = 0) { - std::cout << to_addr << ":" << to_port << std::endl; - // Resolve our addresses const util::network::sock_t remote = util::network::resolve(to_addr, to_port); const bool multicast = diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 8437e8d4b..bb67b1513 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -133,7 +133,6 @@ class TestReactor : public test_util::TestBase { emit(std::make_unique("v4 Ephemeral", "127.0.0.1", v4_port)); } else if (test.name == "v4 Ephemeral") { - std::cout << "Testing v6 known" << std::endl; emit(std::make_unique("v6 Known", "::1", KNOWN_V6_PORT)); } else if (test.name == "v6 Known") { From 3cccf8fe4093d7d3e6cb2aac5190274314a8ec03 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 22 Aug 2023 22:57:08 +1000 Subject: [PATCH 119/176] Add expected output --- tests/dsl/UDP.cpp | 75 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index 06f6fcc72..d8ad9a5ab 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -34,6 +34,13 @@ enum TestPorts { MULTICAST_V6 = 40004, }; +// Ephemeral ports that we will use +in_port_t uni_v4_port = 0; +in_port_t uni_v6_port = 0; +in_port_t broad_v4_port = 0; +in_port_t multi_v4_port = 0; +in_port_t multi_v6_port = 0; + constexpr in_port_t BASE_PORT = 40000; const std::string IPV4_MULTICAST_ADDRESS = "230.12.3.22"; // NOLINT(cert-err58-cpp) const std::string IPV6_MULTICAST_ADDRESS = "ff02::230:12:3:22"; // NOLINT(cert-err58-cpp) @@ -95,10 +102,10 @@ class TestReactor : public test_util::TestBase { }); // IPv4 Unicast Ephemeral port - auto uni_v4 = on().then([this](const UDP::Packet& packet) { // + auto uni_v4 = on().then([this](const UDP::Packet& packet) { // handle_data("Uv4 E", packet); }); - in_port_t uni_v4_port = std::get<1>(uni_v4); + uni_v4_port = std::get<1>(uni_v4); // IPv6 Unicast Known port on(UNICAST_V6, "::1").then([this](const UDP::Packet& packet) { // @@ -106,10 +113,10 @@ class TestReactor : public test_util::TestBase { }); // IPv6 Unicast Ephemeral port - auto uni_v6 = on(0, "::1").then([this](const UDP::Packet& packet) { // + auto uni_v6 = on(0, "::1").then([this](const UDP::Packet& packet) { // handle_data("Uv6 E", packet); }); - in_port_t uni_v6_port = std::get<1>(uni_v6); + uni_v6_port = std::get<1>(uni_v6); // IPv4 Broadcast Known port on(BROADCAST_V4).then([this](const UDP::Packet& packet) { // @@ -117,10 +124,10 @@ class TestReactor : public test_util::TestBase { }); // IPv4 Broadcast Ephemeral port - auto broad_v4 = on().then([this](const UDP::Packet& packet) { // + auto broad_v4 = on().then([this](const UDP::Packet& packet) { // handle_data("Bv4 E", packet); }); - in_port_t broad_v4_port = std::get<1>(broad_v4); + broad_v4_port = std::get<1>(broad_v4); // No such thing as broadcast in IPv6 @@ -133,7 +140,7 @@ class TestReactor : public test_util::TestBase { auto multi_v4 = on(IPV4_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { // handle_data("Mv4 E", packet); }); - in_port_t multi_v4_port = std::get<1>(multi_v4); + multi_v4_port = std::get<1>(multi_v4); // IPv6 Multicast Known port on(IPV6_MULTICAST_ADDRESS, MULTICAST_V6).then([this](const UDP::Packet& packet) { // @@ -144,7 +151,7 @@ class TestReactor : public test_util::TestBase { auto multi_v6 = on(IPV6_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { // handle_data("Mv6 E", packet); }); - in_port_t multi_v6_port = std::get<1>(multi_v6); + multi_v6_port = std::get<1>(multi_v6); // Send a test message to the known port on>().then([this](const TestUDP& target) { @@ -232,7 +239,57 @@ TEST_CASE("Testing sending and receiving of UDP messages", "[api][network][udp]" plant.install(); plant.start(); - const std::vector expected = {}; + const std::vector expected = { + "--------------------", + " -> 192.168.189.255:" + std::to_string(UNICAST_V4) + "", + " -> ::1:" + std::to_string(UNICAST_V4) + "", + " -> 127.0.0.1:" + std::to_string(UNICAST_V4) + "", + "Uv4 K <- Uv4 K (127.0.0.1:" + std::to_string(UNICAST_V4) + ")", + "--------------------", + " -> 192.168.189.255:" + std::to_string(uni_v4_port) + "", + " -> ::1:" + std::to_string(uni_v4_port) + "", + " -> 127.0.0.1:" + std::to_string(uni_v4_port) + "", + "Uv4 E <- Uv4 E (127.0.0.1:" + std::to_string(uni_v4_port) + ")", + "--------------------", + " -> 192.168.189.255:" + std::to_string(UNICAST_V6) + "", + " -> 127.0.0.1:" + std::to_string(UNICAST_V6) + "", + " -> ::1:" + std::to_string(UNICAST_V6) + "", + "Uv6 K <- Uv6 K (::1:" + std::to_string(UNICAST_V6) + ")", + "--------------------", + " -> 192.168.189.255:" + std::to_string(uni_v6_port) + "", + " -> 127.0.0.1:" + std::to_string(uni_v6_port) + "", + " -> ::1:" + std::to_string(uni_v6_port) + "", + "Uv6 E <- Uv6 E (::1:" + std::to_string(uni_v6_port) + ")", + "--------------------", + " -> 127.0.0.1:" + std::to_string(BROADCAST_V4) + "", + " -> ::1:" + std::to_string(BROADCAST_V4) + "", + " -> 192.168.189.255:" + std::to_string(BROADCAST_V4) + "", + "Bv4 K <- Bv4 K (192.168.189.255:" + std::to_string(BROADCAST_V4) + ")", + "--------------------", + " -> 127.0.0.1:" + std::to_string(broad_v4_port) + "", + " -> ::1:" + std::to_string(broad_v4_port) + "", + " -> 192.168.189.255:" + std::to_string(broad_v4_port) + "", + "Bv4 E <- Bv4 E (192.168.189.255:" + std::to_string(broad_v4_port) + ")", + "--------------------", + " -> 127.0.0.1:" + std::to_string(MULTICAST_V4) + "", + " -> 192.168.189.255:" + std::to_string(MULTICAST_V4) + "", + " -> 230.12.3.22:" + std::to_string(MULTICAST_V4) + "", + "Mv4 K <- Mv4 K (230.12.3.22:" + std::to_string(MULTICAST_V4) + ")", + "--------------------", + " -> 127.0.0.1:" + std::to_string(multi_v4_port) + "", + " -> 192.168.189.255:" + std::to_string(multi_v4_port) + "", + " -> 230.12.3.22:" + std::to_string(multi_v4_port) + "", + "Mv4 E <- Mv4 E (230.12.3.22:" + std::to_string(multi_v4_port) + ")", + "--------------------", + " -> ::1:" + std::to_string(MULTICAST_V6) + "", + " -> ff02::230:12:3:22:" + std::to_string(MULTICAST_V6) + "", + "Mv6 K <- Mv6 K (ff02::230:12:3:22:" + std::to_string(MULTICAST_V6) + ")", + "--------------------", + " -> ::1:" + std::to_string(multi_v6_port) + "", + " -> ff02::230:12:3:22:" + std::to_string(multi_v6_port) + "", + "Mv6 E <- Mv6 E (ff02::230:12:3:22:" + std::to_string(multi_v6_port) + ")", + "--------------------", + }; // Make an info print the diff in an easy to read way if we fail INFO(test_util::diff_string(expected, events)); From 60584663a21e3d6f297a2f53d7f535c92ed281a6 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 22 Aug 2023 23:04:25 +1000 Subject: [PATCH 120/176] . --- tests/dsl/UDP.cpp | 56 +++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index d8ad9a5ab..889abc3c3 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -241,52 +241,52 @@ TEST_CASE("Testing sending and receiving of UDP messages", "[api][network][udp]" const std::vector expected = { "--------------------", - " -> 192.168.189.255:" + std::to_string(UNICAST_V4) + "", - " -> ::1:" + std::to_string(UNICAST_V4) + "", - " -> 127.0.0.1:" + std::to_string(UNICAST_V4) + "", + " -> 192.168.189.255:" + std::to_string(UNICAST_V4), + " -> ::1:" + std::to_string(UNICAST_V4), + " -> 127.0.0.1:" + std::to_string(UNICAST_V4), "Uv4 K <- Uv4 K (127.0.0.1:" + std::to_string(UNICAST_V4) + ")", "--------------------", - " -> 192.168.189.255:" + std::to_string(uni_v4_port) + "", - " -> ::1:" + std::to_string(uni_v4_port) + "", - " -> 127.0.0.1:" + std::to_string(uni_v4_port) + "", + " -> 192.168.189.255:" + std::to_string(uni_v4_port), + " -> ::1:" + std::to_string(uni_v4_port), + " -> 127.0.0.1:" + std::to_string(uni_v4_port), "Uv4 E <- Uv4 E (127.0.0.1:" + std::to_string(uni_v4_port) + ")", "--------------------", - " -> 192.168.189.255:" + std::to_string(UNICAST_V6) + "", - " -> 127.0.0.1:" + std::to_string(UNICAST_V6) + "", - " -> ::1:" + std::to_string(UNICAST_V6) + "", + " -> 192.168.189.255:" + std::to_string(UNICAST_V6), + " -> 127.0.0.1:" + std::to_string(UNICAST_V6), + " -> ::1:" + std::to_string(UNICAST_V6), "Uv6 K <- Uv6 K (::1:" + std::to_string(UNICAST_V6) + ")", "--------------------", - " -> 192.168.189.255:" + std::to_string(uni_v6_port) + "", - " -> 127.0.0.1:" + std::to_string(uni_v6_port) + "", - " -> ::1:" + std::to_string(uni_v6_port) + "", + " -> 192.168.189.255:" + std::to_string(uni_v6_port), + " -> 127.0.0.1:" + std::to_string(uni_v6_port), + " -> ::1:" + std::to_string(uni_v6_port), "Uv6 E <- Uv6 E (::1:" + std::to_string(uni_v6_port) + ")", "--------------------", - " -> 127.0.0.1:" + std::to_string(BROADCAST_V4) + "", - " -> ::1:" + std::to_string(BROADCAST_V4) + "", - " -> 192.168.189.255:" + std::to_string(BROADCAST_V4) + "", + " -> 127.0.0.1:" + std::to_string(BROADCAST_V4), + " -> ::1:" + std::to_string(BROADCAST_V4), + " -> 192.168.189.255:" + std::to_string(BROADCAST_V4), "Bv4 K <- Bv4 K (192.168.189.255:" + std::to_string(BROADCAST_V4) + ")", "--------------------", - " -> 127.0.0.1:" + std::to_string(broad_v4_port) + "", - " -> ::1:" + std::to_string(broad_v4_port) + "", - " -> 192.168.189.255:" + std::to_string(broad_v4_port) + "", + " -> 127.0.0.1:" + std::to_string(broad_v4_port), + " -> ::1:" + std::to_string(broad_v4_port), + " -> 192.168.189.255:" + std::to_string(broad_v4_port), "Bv4 E <- Bv4 E (192.168.189.255:" + std::to_string(broad_v4_port) + ")", "--------------------", - " -> 127.0.0.1:" + std::to_string(MULTICAST_V4) + "", - " -> 192.168.189.255:" + std::to_string(MULTICAST_V4) + "", - " -> 230.12.3.22:" + std::to_string(MULTICAST_V4) + "", + " -> 127.0.0.1:" + std::to_string(MULTICAST_V4), + " -> 192.168.189.255:" + std::to_string(MULTICAST_V4), + " -> 230.12.3.22:" + std::to_string(MULTICAST_V4), "Mv4 K <- Mv4 K (230.12.3.22:" + std::to_string(MULTICAST_V4) + ")", "--------------------", - " -> 127.0.0.1:" + std::to_string(multi_v4_port) + "", - " -> 192.168.189.255:" + std::to_string(multi_v4_port) + "", - " -> 230.12.3.22:" + std::to_string(multi_v4_port) + "", + " -> 127.0.0.1:" + std::to_string(multi_v4_port), + " -> 192.168.189.255:" + std::to_string(multi_v4_port), + " -> 230.12.3.22:" + std::to_string(multi_v4_port), "Mv4 E <- Mv4 E (230.12.3.22:" + std::to_string(multi_v4_port) + ")", "--------------------", - " -> ::1:" + std::to_string(MULTICAST_V6) + "", - " -> ff02::230:12:3:22:" + std::to_string(MULTICAST_V6) + "", + " -> ::1:" + std::to_string(MULTICAST_V6), + " -> ff02::230:12:3:22:" + std::to_string(MULTICAST_V6), "Mv6 K <- Mv6 K (ff02::230:12:3:22:" + std::to_string(MULTICAST_V6) + ")", "--------------------", - " -> ::1:" + std::to_string(multi_v6_port) + "", - " -> ff02::230:12:3:22:" + std::to_string(multi_v6_port) + "", + " -> ::1:" + std::to_string(multi_v6_port), + " -> ff02::230:12:3:22:" + std::to_string(multi_v6_port), "Mv6 E <- Mv6 E (ff02::230:12:3:22:" + std::to_string(multi_v6_port) + ")", "--------------------", }; From 77009b208b80146bfaeea851dbb50a4788f40b6c Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 24 Aug 2023 13:25:48 +1000 Subject: [PATCH 121/176] Fix filtering and make UDP tests pass --- src/dsl/word/UDP.hpp | 222 ++++++++++++++++++++++++----------- tests/dsl/UDP.cpp | 273 ++++++++++++++++++++++++++----------------- 2 files changed, 322 insertions(+), 173 deletions(-) diff --git a/src/dsl/word/UDP.hpp b/src/dsl/word/UDP.hpp index 8831f55da..6a523b272 100644 --- a/src/dsl/word/UDP.hpp +++ b/src/dsl/word/UDP.hpp @@ -45,28 +45,50 @@ namespace dsl { * request for a UDP based reaction can use a runtime argument to reference a specific port. Note that the * port reference can be changed during the systems execution phase. * + * @code on(port, bind_address) @endcode + * The `bind_address` parameter can be used to specify which interface to bind on. If `bind_address` is an + * empty string, the system will bind to any available interface. + * * @code on() @endcode * Should the port reference be omitted, then the system will bind to a currently unassigned port. * - * @code on(port, port) @endcode - * A reaction can also be triggered via activity on more than one port. - * * @code on(port) * on(multicast_address, port) @endcode * If needed, this trigger can also listen for UDP activity such as broadcast and multicast. * - * These requests currently support IPv4 addressing. + * These requests support both IPv4 and IPv6 addressing. * * @par Implements * Bind */ struct UDP : public IO { private: - struct Target { + /** + * @brief This structure is used to configure the UDP connection + */ + struct ConnectOptions { + /// @brief The type of connection we are making + enum Type { UNICAST, BROADCAST, MULTICAST } type{}; + /// @brief The address we are binding to or empty for any std::string bind_address{}; + /// @brief The port we are binding to or 0 for any in_port_t port = 0; + /// @brief The multicast address we are listening on or empty for any std::string target_address{}; - enum Type { UNICAST, BROADCAST, MULTICAST } type{}; + }; + + /** + * @brief This structure is used to return the result of a recvmsg call + */ + struct RecvResult { + /// @brief If the packet is valid + bool valid{false}; + /// @brief The data that was received + std::vector payload{}; + /// @brief The local address that the packet was received on + util::network::sock_t local{}; + /// @brief The remote address that the packet was received from + util::network::sock_t remote{}; }; public: @@ -80,9 +102,9 @@ namespace dsl { Target() = default; Target(std::string address, const uint16_t& port) : address(std::move(address)), port(port) {} - /// The address of the target + /// @brief The address of the target std::string address{}; - /// The port of the target + /// @brief The port of the target uint16_t port{0}; }; @@ -113,42 +135,44 @@ namespace dsl { template static inline std::tuple connect(const std::shared_ptr& reaction, - const Target& target) { + const ConnectOptions& options) { // Resolve the addresses util::network::sock_t bind_address{}; util::network::sock_t multicast_target{}; - if (target.type == Target::MULTICAST) { - multicast_target = util::network::resolve(target.target_address, target.port); - if (target.bind_address.empty()) { + if (options.type == ConnectOptions::MULTICAST) { + multicast_target = util::network::resolve(options.target_address, options.port); + + // If there is no bind address, make sure we bind to an address of the same family + if (options.bind_address.empty()) { bind_address = multicast_target; switch (bind_address.sock.sa_family) { case AF_INET: { - bind_address.ipv4.sin_port = htons(target.port); + bind_address.ipv4.sin_port = htons(options.port); bind_address.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); } break; case AF_INET6: { - bind_address.ipv6.sin6_port = htons(target.port); + bind_address.ipv6.sin6_port = htons(options.port); bind_address.ipv6.sin6_addr = in6addr_any; } break; default: throw std::runtime_error("Unknown socket family"); } } else { - bind_address = util::network::resolve(target.bind_address, target.port); + bind_address = util::network::resolve(options.bind_address, options.port); if (multicast_target.sock.sa_family != bind_address.sock.sa_family) { throw std::runtime_error("Multicast address family does not match bind address family"); } } } else { - if (target.bind_address.empty()) { + if (options.bind_address.empty()) { bind_address.ipv4.sin_family = AF_INET; - bind_address.ipv4.sin_port = htons(target.port); + bind_address.ipv4.sin_port = htons(options.port); bind_address.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); } else { - bind_address = util::network::resolve(target.bind_address, target.port); + bind_address = util::network::resolve(options.bind_address, options.port); } } @@ -175,14 +199,14 @@ namespace dsl { reinterpret_cast(&yes), sizeof(yes)) < 0) { - throw std::system_error(network_errno, - std::system_category(), - "We were unable to flag the socket as getting ancillary data"); - } + throw std::system_error(network_errno, + std::system_category(), + "We were unable to flag the socket as getting ancillary data"); + } } // Broadcast and multicast reuse address and port - if (target.type == Target::BROADCAST || target.type == Target::MULTICAST) { + if (options.type == ConnectOptions::BROADCAST || options.type == ConnectOptions::MULTICAST) { if (::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&yes), sizeof(yes)) < 0) { throw std::system_error(network_errno, @@ -213,7 +237,7 @@ namespace dsl { } // If we have a multicast address, then we need to join the multicast groups - if (target.type == Target::MULTICAST) { + if (options.type == ConnectOptions::MULTICAST) { // Our multicast join request will depend on protocol version if (multicast_target.sock.sa_family == AF_INET) { @@ -309,37 +333,25 @@ namespace dsl { } template - static inline std::tuple bind(const std::shared_ptr& reaction, - in_port_t port = 0, - std::string bind_address = "") { - Target t; - t.type = Target::UNICAST; - t.bind_address = std::move(bind_address); - t.port = port; - return connect(reaction, t); - } - - template - static inline Packet get(threading::Reaction& reaction) { + static inline RecvResult read(threading::Reaction& reaction) { // Get our file descriptor from the magic cache auto event = IO::get(reaction); // If our get is being run without an fd (something else triggered) then short circuit if (!event) { - return Packet{}; + return {}; } // Allocate max size for a UDP packet - Packet p{}; - p.payload.resize(65535); + std::vector buffer(65535, 0); // Make some variables to hold our message header information std::array cmbuff = {0}; util::network::sock_t remote{}; iovec payload{}; - payload.iov_base = p.payload.data(); - payload.iov_len = static_cast(p.payload.size()); + payload.iov_base = buffer.data(); + payload.iov_len = static_cast(buffer.size()); // Make our message header to receive with msghdr mh{}; @@ -352,6 +364,12 @@ namespace dsl { // Receive our message ssize_t received = recvmsg(event.fd, &mh, 0); + if (received < 0) { + return {}; + } + + buffer.resize(received); + buffer.shrink_to_fit(); // Load the socket we are listening on util::network::sock_t local{}; @@ -388,35 +406,93 @@ namespace dsl { } } + return RecvResult{true, buffer, local, remote}; + } + + template + static inline std::tuple bind(const std::shared_ptr& reaction, + const in_port_t& port = 0, + const std::string& bind_address = "") { + return connect(reaction, ConnectOptions{ConnectOptions::UNICAST, bind_address, port, ""}); + } + + template + static inline Packet get(threading::Reaction& reaction) { + RecvResult result = read(reaction); - // if no error - if (received > 0) { - p.valid = true; - std::tie(p.local.address, p.local.port) = local.address(); - std::tie(p.remote.address, p.remote.port) = remote.address(); - p.payload.resize(size_t(received)); - p.payload.shrink_to_fit(); + Packet p{}; + p.valid = result.valid; + p.payload = std::move(result.payload); + auto local_s = result.local.address(); + auto remote_s = result.remote.address(); + p.local = Packet::Target{local_s.first, local_s.second}; + p.remote = Packet::Target{remote_s.first, remote_s.second}; + + // Confirm that this packet was sent to one of our broadcast addresses + for (const auto& iface : util::network::get_interfaces()) { + if (iface.ip.sock.sa_family == result.local.sock.sa_family) { + // If the two are equal + if (iface.ip.sock.sa_family == AF_INET) { + if (iface.ip.ipv4.sin_addr.s_addr == result.local.ipv4.sin_addr.s_addr) { + return p; + } + } + else if (iface.ip.sock.sa_family == AF_INET6) { + if (std::memcmp(&iface.ip.ipv6.sin6_addr, + &result.local.ipv6.sin6_addr, + sizeof(result.local.ipv6.sin6_addr)) + == 0) { + return p; + } + } + } } - return p; + return {}; } struct Broadcast : public IO { template static inline std::tuple bind(const std::shared_ptr& reaction, - in_port_t port = 0, - std::string bind_address = "") { - Target t; - t.type = Target::BROADCAST; - t.bind_address = std::move(bind_address); - t.port = port; - return UDP::connect(reaction, t); + const in_port_t& port = 0, + const std::string& bind_address = "") { + return UDP::connect(reaction, + ConnectOptions{ConnectOptions::BROADCAST, bind_address, port, ""}); } template static inline Packet get(threading::Reaction& reaction) { - return UDP::get(reaction); + RecvResult result = read(reaction); + + // Broadcast is only IPv4 + if (result.local.sock.sa_family == AF_INET) { + + Packet p{}; + p.valid = result.valid; + p.payload = std::move(result.payload); + auto local_s = result.local.address(); + auto remote_s = result.remote.address(); + p.local = Packet::Target{local_s.first, local_s.second}; + p.remote = Packet::Target{remote_s.first, remote_s.second}; + + // 255.255.255.255 is always a valid broadcast address + if (result.local.ipv4.sin_addr.s_addr == htonl(INADDR_BROADCAST)) { + return p; + } + + // Confirm that this packet was sent to one of our broadcast addresses + for (const auto& iface : util::network::get_interfaces()) { + if (iface.broadcast.sock.sa_family == AF_INET) { + if (iface.flags.broadcast + && iface.broadcast.ipv4.sin_addr.s_addr == result.local.ipv4.sin_addr.s_addr) { + return p; + } + } + } + } + + return {}; } }; @@ -425,19 +501,35 @@ namespace dsl { template static inline std::tuple bind(const std::shared_ptr& reaction, const std::string& multicast_group, - in_port_t port = 0, + const in_port_t& port = 0, const std::string& bind_address = "") { - Target t; - t.type = Target::MULTICAST; - t.bind_address = bind_address; - t.target_address = multicast_group; - t.port = port; - return UDP::connect(reaction, t); + return UDP::connect( + reaction, + ConnectOptions{ConnectOptions::MULTICAST, bind_address, port, multicast_group}); } template static inline Packet get(threading::Reaction& reaction) { - return UDP::get(reaction); + RecvResult result = read(reaction); + + const auto& a = result.local; + const bool multicast = + (a.sock.sa_family == AF_INET && (ntohl(a.ipv4.sin_addr.s_addr) & 0xF0000000) == 0xE0000000) + || (a.sock.sa_family == AF_INET6 && a.ipv6.sin6_addr.s6_addr[0] == 0xFF); + + // Only return multicast packets + if (multicast) { + Packet p{}; + p.valid = result.valid; + p.payload = std::move(result.payload); + auto local_s = result.local.address(); + auto remote_s = result.remote.address(); + p.local = Packet::Target{local_s.first, local_s.second}; + p.remote = Packet::Target{remote_s.first, remote_s.second}; + return p; + } + + return {}; } }; }; diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index 889abc3c3..9daa92999 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -46,6 +46,12 @@ const std::string IPV4_MULTICAST_ADDRESS = "230.12.3.22"; // NOLINT(cert- const std::string IPV6_MULTICAST_ADDRESS = "ff02::230:12:3:22"; // NOLINT(cert-err58-cpp) inline std::string get_broadcast_addr() { + static std::string addr = ""; + + if (!addr.empty()) { + return addr; + } + // Get the first IPv4 broadcast address we can find std::array buff{}; bool found = false; @@ -59,7 +65,40 @@ inline std::string get_broadcast_addr() { if (!found) { throw std::runtime_error("No broadcast address found"); } - return std::string(buff.data()); + addr = std::string(buff.data()); + return addr; +} + +struct SendTarget { + std::string data; + std::string address; + in_port_t port; +}; +std::vector send_targets(const std::string& type, in_port_t port, bool include_target = false) { + std::vector results; + + // Make sure that the type we are actually after is sent last + std::string t = type.substr(0, 3); + if (type[2] == '4' && include_target == (t == "Uv4")) { + results.push_back(SendTarget{type + ":Uv4", "127.0.0.1", port}); + } + if (type[2] == '4' && include_target == (t == "Bv4")) { + results.push_back(SendTarget{type + ":Bv4", get_broadcast_addr(), port}); + } + if (type[2] == '4' && include_target == (t == "Mv4")) { + results.push_back(SendTarget{type + ":Mv4", IPV4_MULTICAST_ADDRESS, port}); + } + if (type[2] == '6' && include_target == (t == "Uv6")) { + results.push_back(SendTarget{type + ":Uv6", "::1", port}); + } + if (type[2] == '6' && include_target == (t == "Mv6")) { + results.push_back(SendTarget{type + ":Mv6", IPV6_MULTICAST_ADDRESS, port}); + } + if (!include_target) { + auto target = send_targets(type, port, true); + results.insert(results.end(), target.begin(), target.end()); + } + return results; } struct TestUDP { @@ -85,7 +124,7 @@ class TestReactor : public test_util::TestBase { events.push_back(name + " <- " + data + " (" + local + ")"); - if (data == name) { + if (data == (name + ":" + name.substr(0, 3))) { emit(std::make_unique(name)); } } @@ -93,39 +132,36 @@ class TestReactor : public test_util::TestBase { public: TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { - // Get the first IPv4 broadcast address we can find - std::string broadcast_addr = get_broadcast_addr(); - // IPv4 Unicast Known port on(UNICAST_V4).then([this](const UDP::Packet& packet) { // - handle_data("Uv4 K", packet); + handle_data("Uv4K", packet); }); // IPv4 Unicast Ephemeral port auto uni_v4 = on().then([this](const UDP::Packet& packet) { // - handle_data("Uv4 E", packet); + handle_data("Uv4E", packet); }); uni_v4_port = std::get<1>(uni_v4); // IPv6 Unicast Known port on(UNICAST_V6, "::1").then([this](const UDP::Packet& packet) { // - handle_data("Uv6 K", packet); + handle_data("Uv6K", packet); }); // IPv6 Unicast Ephemeral port auto uni_v6 = on(0, "::1").then([this](const UDP::Packet& packet) { // - handle_data("Uv6 E", packet); + handle_data("Uv6E", packet); }); uni_v6_port = std::get<1>(uni_v6); // IPv4 Broadcast Known port on(BROADCAST_V4).then([this](const UDP::Packet& packet) { // - handle_data("Bv4 K", packet); + handle_data("Bv4K", packet); }); // IPv4 Broadcast Ephemeral port auto broad_v4 = on().then([this](const UDP::Packet& packet) { // - handle_data("Bv4 E", packet); + handle_data("Bv4E", packet); }); broad_v4_port = std::get<1>(broad_v4); @@ -133,23 +169,23 @@ class TestReactor : public test_util::TestBase { // IPv4 Multicast Known port on(IPV4_MULTICAST_ADDRESS, MULTICAST_V4).then([this](const UDP::Packet& packet) { // - handle_data("Mv4 K", packet); + handle_data("Mv4K", packet); }); // IPv4 Multicast Ephemeral port auto multi_v4 = on(IPV4_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { // - handle_data("Mv4 E", packet); + handle_data("Mv4E", packet); }); multi_v4_port = std::get<1>(multi_v4); // IPv6 Multicast Known port on(IPV6_MULTICAST_ADDRESS, MULTICAST_V6).then([this](const UDP::Packet& packet) { // - handle_data("Mv6 K", packet); + handle_data("Mv6K", packet); }); // IPv6 Multicast Ephemeral port auto multi_v6 = on(IPV6_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { // - handle_data("Mv6 E", packet); + handle_data("Mv6E", packet); }); multi_v6_port = std::get<1>(multi_v6); @@ -160,60 +196,63 @@ class TestReactor : public test_util::TestBase { }); on>().then([=](const Finished& test) { - events.push_back("--------------------"); + auto send_all = [this](std::string type, in_port_t port) { + for (const auto& t : send_targets(type, port)) { + events.push_back(" -> " + t.address + ":" + std::to_string(t.port)); + emit(std::make_unique(t.data), t.address, t.port); + } + }; if (test.name == "Startup") { - // Send some invalid packets and a valid one - emit(std::make_unique("Bv4 I", broadcast_addr, UNICAST_V4)); - emit(std::make_unique("Uv6 I", "::1", UNICAST_V4)); - emit(std::make_unique("Uv4 K", "127.0.0.1", UNICAST_V4)); + events.push_back("- Known Unicast V4 Test -"); + send_all("Uv4K", UNICAST_V4); } - else if (test.name == "Uv4 K") { - // Send some invalid packets and a valid one - emit(std::make_unique("Bv4 I", broadcast_addr, uni_v4_port)); - emit(std::make_unique("Uv6 I", "::1", uni_v4_port)); - emit(std::make_unique("Uv4 E", "127.0.0.1", uni_v4_port)); + else if (test.name == "Uv4K") { + events.push_back(""); + events.push_back("- Ephemeral Unicast V4 Test -"); + send_all("Uv4E", uni_v4_port); } - else if (test.name == "Uv4 E") { - emit(std::make_unique("Bv4 I", broadcast_addr, UNICAST_V6)); - emit(std::make_unique("Uv4 I", "127.0.0.1", UNICAST_V6)); - emit(std::make_unique("Uv6 K", "::1", UNICAST_V6)); + else if (test.name == "Uv4E") { + events.push_back(""); + events.push_back("- Known Unicast V6 Test -"); + send_all("Uv6K", UNICAST_V6); } - else if (test.name == "Uv6 K") { - emit(std::make_unique("Bv4 I", broadcast_addr, uni_v6_port)); - emit(std::make_unique("Uv4 I", "127.0.0.1", uni_v6_port)); - emit(std::make_unique("Uv6 E", "::1", uni_v6_port)); + else if (test.name == "Uv6K") { + events.push_back(""); + events.push_back("- Ephemeral Unicast V6 Test -"); + send_all("Uv6E", uni_v6_port); } - else if (test.name == "Uv6 E") { - // Send a unicast and broadcast (only the broadcast should be received) - emit(std::make_unique("Uv4 I", "127.0.0.1", BROADCAST_V4)); - emit(std::make_unique("Uv6 E", "::1", BROADCAST_V4)); - emit(std::make_unique("Bv4 K", broadcast_addr, BROADCAST_V4)); + else if (test.name == "Uv6E") { + events.push_back(""); + events.push_back("- Known Broadcast V4 Test -"); + send_all("Bv4K", BROADCAST_V4); } - else if (test.name == "Bv4 K") { - emit(std::make_unique("Uv4 I", "127.0.0.1", broad_v4_port)); - emit(std::make_unique("Uv6 E", "::1", broad_v4_port)); - emit(std::make_unique("Bv4 E", broadcast_addr, broad_v4_port)); + else if (test.name == "Bv4K") { + events.push_back(""); + events.push_back("- Ephemeral Broadcast V4 Test -"); + send_all("Bv4E", broad_v4_port); } - else if (test.name == "Bv4 E") { - emit(std::make_unique("Uv4 I", "127.0.0.1", MULTICAST_V4)); - emit(std::make_unique("Bv4 I", broadcast_addr, MULTICAST_V4)); - emit(std::make_unique("Mv4 K", IPV4_MULTICAST_ADDRESS, MULTICAST_V4)); + else if (test.name == "Bv4E") { + events.push_back(""); + events.push_back("- Known Multicast V4 Test -"); + send_all("Mv4K", MULTICAST_V4); } - else if (test.name == "Mv4 K") { - emit(std::make_unique("Uv4 I", "127.0.0.1", multi_v4_port)); - emit(std::make_unique("Bv4 I", broadcast_addr, multi_v4_port)); - emit(std::make_unique("Mv4 E", IPV4_MULTICAST_ADDRESS, multi_v4_port)); + else if (test.name == "Mv4K") { + events.push_back(""); + events.push_back("- Ephemeral Multicast V4 Test -"); + send_all("Mv4E", multi_v4_port); } - else if (test.name == "Mv4 E") { - emit(std::make_unique("Uv6 I", "::1", MULTICAST_V6)); - emit(std::make_unique("Mv6 K", IPV6_MULTICAST_ADDRESS, MULTICAST_V6)); + else if (test.name == "Mv4E") { + events.push_back(""); + events.push_back("- Known Multicast V6 Test -"); + send_all("Mv6K", MULTICAST_V6); } - else if (test.name == "Mv6 K") { - emit(std::make_unique("Uv6 I", "::1", multi_v6_port)); - emit(std::make_unique("Mv6 E", IPV6_MULTICAST_ADDRESS, multi_v6_port)); + else if (test.name == "Mv6K") { + events.push_back(""); + events.push_back("- Ephemeral Multicast V6 Test -"); + send_all("Mv6E", multi_v6_port); } - else if (test.name == "Mv6 E") { + else if (test.name == "Mv6E") { // We are done, so stop the reactor powerplant.shutdown(); } @@ -239,57 +278,75 @@ TEST_CASE("Testing sending and receiving of UDP messages", "[api][network][udp]" plant.install(); plant.start(); - const std::vector expected = { - "--------------------", - " -> 192.168.189.255:" + std::to_string(UNICAST_V4), - " -> ::1:" + std::to_string(UNICAST_V4), - " -> 127.0.0.1:" + std::to_string(UNICAST_V4), - "Uv4 K <- Uv4 K (127.0.0.1:" + std::to_string(UNICAST_V4) + ")", - "--------------------", - " -> 192.168.189.255:" + std::to_string(uni_v4_port), - " -> ::1:" + std::to_string(uni_v4_port), - " -> 127.0.0.1:" + std::to_string(uni_v4_port), - "Uv4 E <- Uv4 E (127.0.0.1:" + std::to_string(uni_v4_port) + ")", - "--------------------", - " -> 192.168.189.255:" + std::to_string(UNICAST_V6), - " -> 127.0.0.1:" + std::to_string(UNICAST_V6), - " -> ::1:" + std::to_string(UNICAST_V6), - "Uv6 K <- Uv6 K (::1:" + std::to_string(UNICAST_V6) + ")", - "--------------------", - " -> 192.168.189.255:" + std::to_string(uni_v6_port), - " -> 127.0.0.1:" + std::to_string(uni_v6_port), - " -> ::1:" + std::to_string(uni_v6_port), - "Uv6 E <- Uv6 E (::1:" + std::to_string(uni_v6_port) + ")", - "--------------------", - " -> 127.0.0.1:" + std::to_string(BROADCAST_V4), - " -> ::1:" + std::to_string(BROADCAST_V4), - " -> 192.168.189.255:" + std::to_string(BROADCAST_V4), - "Bv4 K <- Bv4 K (192.168.189.255:" + std::to_string(BROADCAST_V4) + ")", - "--------------------", - " -> 127.0.0.1:" + std::to_string(broad_v4_port), - " -> ::1:" + std::to_string(broad_v4_port), - " -> 192.168.189.255:" + std::to_string(broad_v4_port), - "Bv4 E <- Bv4 E (192.168.189.255:" + std::to_string(broad_v4_port) + ")", - "--------------------", - " -> 127.0.0.1:" + std::to_string(MULTICAST_V4), - " -> 192.168.189.255:" + std::to_string(MULTICAST_V4), - " -> 230.12.3.22:" + std::to_string(MULTICAST_V4), - "Mv4 K <- Mv4 K (230.12.3.22:" + std::to_string(MULTICAST_V4) + ")", - "--------------------", - " -> 127.0.0.1:" + std::to_string(multi_v4_port), - " -> 192.168.189.255:" + std::to_string(multi_v4_port), - " -> 230.12.3.22:" + std::to_string(multi_v4_port), - "Mv4 E <- Mv4 E (230.12.3.22:" + std::to_string(multi_v4_port) + ")", - "--------------------", - " -> ::1:" + std::to_string(MULTICAST_V6), - " -> ff02::230:12:3:22:" + std::to_string(MULTICAST_V6), - "Mv6 K <- Mv6 K (ff02::230:12:3:22:" + std::to_string(MULTICAST_V6) + ")", - "--------------------", - " -> ::1:" + std::to_string(multi_v6_port), - " -> ff02::230:12:3:22:" + std::to_string(multi_v6_port), - "Mv6 E <- Mv6 E (ff02::230:12:3:22:" + std::to_string(multi_v6_port) + ")", - "--------------------", - }; + std::vector expected; + expected.push_back("- Known Unicast V4 Test -"); + for (const auto& line : send_targets("Uv4K", UNICAST_V4)) { + expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + } + expected.push_back("Uv4K <- Uv4K:Uv4 (127.0.0.1:" + std::to_string(UNICAST_V4) + ")"); + expected.push_back(""); + + expected.push_back("- Ephemeral Unicast V4 Test -"); + for (const auto& line : send_targets("Uv4E", uni_v4_port)) { + expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + } + expected.push_back("Uv4E <- Uv4E:Uv4 (127.0.0.1:" + std::to_string(uni_v4_port) + ")"); + expected.push_back(""); + + expected.push_back("- Known Unicast V6 Test -"); + for (const auto& line : send_targets("Uv6K", UNICAST_V6)) { + expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + } + expected.push_back("Uv6K <- Uv6K:Uv6 (::1:" + std::to_string(UNICAST_V6) + ")"); + expected.push_back(""); + + expected.push_back("- Ephemeral Unicast V6 Test -"); + for (const auto& line : send_targets("Uv6E", uni_v6_port)) { + expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + } + expected.push_back("Uv6E <- Uv6E:Uv6 (::1:" + std::to_string(uni_v6_port) + ")"); + expected.push_back(""); + + expected.push_back("- Known Broadcast V4 Test -"); + for (const auto& line : send_targets("Bv4K", BROADCAST_V4)) { + expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + } + expected.push_back("Bv4K <- Bv4K:Bv4 (" + get_broadcast_addr() + ":" + std::to_string(BROADCAST_V4) + ")"); + expected.push_back(""); + + expected.push_back("- Ephemeral Broadcast V4 Test -"); + for (const auto& line : send_targets("Bv4E", broad_v4_port)) { + expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + } + expected.push_back("Bv4E <- Bv4E:Bv4 (" + get_broadcast_addr() + ":" + std::to_string(broad_v4_port) + ")"); + expected.push_back(""); + + expected.push_back("- Known Multicast V4 Test -"); + for (const auto& line : send_targets("Mv4K", MULTICAST_V4)) { + expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + } + expected.push_back("Mv4K <- Mv4K:Mv4 (" + IPV4_MULTICAST_ADDRESS + ":" + std::to_string(MULTICAST_V4) + ")"); + expected.push_back(""); + + expected.push_back("- Ephemeral Multicast V4 Test -"); + for (const auto& line : send_targets("Mv4E", multi_v4_port)) { + expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + } + expected.push_back("Mv4E <- Mv4E:Mv4 (" + IPV4_MULTICAST_ADDRESS + ":" + std::to_string(multi_v4_port) + ")"); + expected.push_back(""); + + expected.push_back("- Known Multicast V6 Test -"); + for (const auto& line : send_targets("Mv6K", MULTICAST_V6)) { + expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + } + expected.push_back("Mv6K <- Mv6K:Mv6 (" + IPV6_MULTICAST_ADDRESS + ":" + std::to_string(MULTICAST_V6) + ")"); + expected.push_back(""); + + expected.push_back("- Ephemeral Multicast V6 Test -"); + for (const auto& line : send_targets("Mv6E", multi_v6_port)) { + expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + } + expected.push_back("Mv6E <- Mv6E:Mv6 (" + IPV6_MULTICAST_ADDRESS + ":" + std::to_string(multi_v6_port) + ")"); // Make an info print the diff in an easy to read way if we fail INFO(test_util::diff_string(expected, events)); From c1ba0c0df2af1ebc3f118afdd809921945fa470c Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 24 Aug 2023 14:38:07 +1000 Subject: [PATCH 122/176] Update NUClearNet based on changes to UDP --- src/extension/NetworkController.hpp | 9 +- src/extension/network/NUClearNetwork.cpp | 224 +++++++++-------------- src/extension/network/NUClearNetwork.hpp | 15 +- src/message/NetworkConfiguration.hpp | 22 ++- 4 files changed, 123 insertions(+), 147 deletions(-) diff --git a/src/extension/NetworkController.hpp b/src/extension/NetworkController.hpp index 7ae5253c6..9512a0da0 100644 --- a/src/extension/NetworkController.hpp +++ b/src/extension/NetworkController.hpp @@ -143,14 +143,11 @@ namespace extension { listen_handles.clear(); } - // Read the new configuration - const std::string name = config.name.empty() ? util::get_hostname() : config.name; - const std::string announce_address = config.announce_address; - const in_port_t announce_port = config.announce_port; - const uint16_t mtu = config.mtu; + // Name becomes hostname by default if not set + const std::string name = config.name.empty() ? util::get_hostname() : config.name; // Reset our network using this configuration - network.reset(name, announce_address, announce_port, mtu); + network.reset(name, config.announce_address, config.announce_port, config.bind_address, config.mtu); // Execution handle process_handle = diff --git a/src/extension/network/NUClearNetwork.cpp b/src/extension/network/NUClearNetwork.cpp index 58d6b866f..f1932fa1b 100644 --- a/src/extension/network/NUClearNetwork.cpp +++ b/src/extension/network/NUClearNetwork.cpp @@ -27,6 +27,8 @@ #include #include "../../util/network/get_interfaces.hpp" +#include "../../util/network/if_number_from_address.hpp" +#include "../../util/network/resolve.hpp" #include "../../util/platform.hpp" namespace NUClear { @@ -144,19 +146,17 @@ namespace extension { } - void NUClearNetwork::open_data(const sock_t& announce_target) { + void NUClearNetwork::open_data(const sock_t& bind_target) { // Create the "join any" address for this address family - sock_t address = announce_target; + sock_t address = bind_target; - // IPv4 + // Set port to 0 to get an ephemeral port if (address.sock.sa_family == AF_INET) { - address.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); - address.ipv4.sin_port = 0; + address.ipv4.sin_port = 0; } // IPv6 else if (address.sock.sa_family == AF_INET6) { - address.ipv6.sin6_addr = IN6ADDR_ANY_INIT; address.ipv6.sin6_port = 0; } @@ -166,7 +166,7 @@ namespace extension { throw std::system_error(network_errno, std::system_category(), "Unable to open the UDP socket"); } - // If we are a broadcast address we need to state we are explicitly before binding + // Set broadcast so we can send to broadcast addresses if needed int yes = 1; if (::setsockopt(data_fd, SOL_SOCKET, SO_BROADCAST, reinterpret_cast(&yes), sizeof(yes)) < 0) { throw std::system_error(network_errno, std::system_category(), "Unable to set broadcast on the socket"); @@ -181,27 +181,16 @@ namespace extension { } - void NUClearNetwork::open_announce(const sock_t& announce_target) { - - // Work out what type of announce target we are using - sock_t address = announce_target; + void NUClearNetwork::open_announce(const sock_t& announce_target, const sock_t& bind_target) { // Work out what type of announce we are doing as it will influence how we make the socket const bool multicast = - (address.sock.sa_family == AF_INET && (ntohl(address.ipv4.sin_addr.s_addr) & 0xF0000000) == 0xE0000000) - || (address.sock.sa_family == AF_INET6 && address.ipv6.sin6_addr.s6_addr[0] == 0xFF - && address.ipv6.sin6_addr.s6_addr[1] == 0x00); - - // Swap our address so the rest of the information is anys - if (address.sock.sa_family == AF_INET) { - address.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); - } - else if (address.sock.sa_family == AF_INET6) { - address.ipv6.sin6_addr = IN6ADDR_ANY_INIT; - } + (announce_target.sock.sa_family == AF_INET + && (ntohl(announce_target.ipv4.sin_addr.s_addr) & 0xF0000000) == 0xE0000000) + || (announce_target.sock.sa_family == AF_INET6 && announce_target.ipv6.sin6_addr.s6_addr[0] == 0xFF); // Make our socket - announce_fd = ::socket(address.sock.sa_family, SOCK_DGRAM, IPPROTO_UDP); + announce_fd = ::socket(bind_target.sock.sa_family, SOCK_DGRAM, IPPROTO_UDP); if (announce_fd < 0) { throw std::system_error(network_errno, std::system_category(), "Unable to open the UDP socket"); } @@ -225,49 +214,43 @@ namespace extension { } // Bind to the address - if (::bind(announce_fd, &address.sock, address.size()) != 0) { + if (::bind(announce_fd, &bind_target.sock, bind_target.size()) != 0) { throw std::system_error(network_errno, std::system_category(), "Unable to bind the UDP socket"); } // If we have a multicast address, then we need to join the multicast groups if (multicast) { + // Our multicast join request will depend on protocol version if (announce_target.sock.sa_family == AF_INET) { - // Set the multicast address we are listening on + // Set the multicast address we are listening on and bind address ip_mreq mreq{}; mreq.imr_multiaddr = announce_target.ipv4.sin_addr; - - int connected_count = 0; - int last_network_errno = 0; - - // Join the multicast group on all the interfaces that support it - for (auto& iface : util::network::get_interfaces()) { - - if (iface.ip.sock.sa_family == AF_INET && iface.flags.multicast) { - - // Set our interface address - mreq.imr_interface = iface.ip.ipv4.sin_addr; - - // Join our multicast group - if (::setsockopt(announce_fd, - IPPROTO_IP, - IP_ADD_MEMBERSHIP, - reinterpret_cast(&mreq), - sizeof(ip_mreq)) - < 0) { - last_network_errno = network_errno; - } - else { - connected_count++; - } - } + mreq.imr_interface = bind_target.ipv4.sin_addr; + + // Join our multicast group + if (::setsockopt(announce_fd, + IPPROTO_IP, + IP_ADD_MEMBERSHIP, + reinterpret_cast(&mreq), + sizeof(ip_mreq)) + < 0) { + throw std::system_error(network_errno, + std::system_category(), + "There was an error while attempting to join the multicast group"); } - if (connected_count == 0) { - throw std::system_error(last_network_errno, + // Set our transmission interface for the multicast socket + if (::setsockopt(announce_fd, + IPPROTO_IP, + IP_MULTICAST_IF, + reinterpret_cast(&bind_target.ipv4.sin_addr), + sizeof(bind_target.ipv4.sin_addr)) + < 0) { + throw std::system_error(network_errno, std::system_category(), - "There was an error while attempting to join the multicast group"); + "We were unable to use the requested interface for multicast"); } } else if (announce_target.sock.sa_family == AF_INET6) { @@ -275,41 +258,35 @@ namespace extension { // Set the multicast address we are listening on ipv6_mreq mreq{}; mreq.ipv6mr_multiaddr = announce_target.ipv6.sin6_addr; + mreq.ipv6mr_interface = util::network::if_number_from_address(bind_target.ipv6); + + // Join our multicast group + if (::setsockopt(announce_fd, + IPPROTO_IPV6, + IPV6_JOIN_GROUP, + reinterpret_cast(&mreq), + sizeof(ipv6_mreq)) + < 0) { + throw std::system_error(network_errno, + std::system_category(), + "There was an error while attempting to join the multicast group"); + } - std::set added_interfaces; - - // Join the multicast group on all the interfaces that support it - for (auto& iface : util::network::get_interfaces()) { - - if (iface.ip.sock.sa_family == AF_INET6 && iface.flags.multicast) { - - // Get the interface for this - mreq.ipv6mr_interface = if_nametoindex(iface.name.c_str()); - - // Only add each interface index once - if (added_interfaces.count(mreq.ipv6mr_interface) == 0) { - added_interfaces.insert(mreq.ipv6mr_interface); - - // Join our multicast group - if (::setsockopt(announce_fd, - IPPROTO_IPV6, - IPV6_JOIN_GROUP, - reinterpret_cast(&mreq), - sizeof(ipv6_mreq)) - < 0) { - throw std::system_error( - network_errno, - std::system_category(), - "There was an error while attempting to join the multicast group"); - } - } - } + // Set our transmission interface for the multicast socket + if (::setsockopt(announce_fd, + IPPROTO_IPV6, + IPV6_MULTICAST_IF, + reinterpret_cast(&mreq.ipv6mr_interface), + sizeof(mreq.ipv6mr_interface)) + < 0) { + throw std::system_error(network_errno, + std::system_category(), + "We were unable to use the requested interface for multicast"); } } } } - void NUClearNetwork::shutdown() { // If we have an fd, send a shutdown message @@ -341,10 +318,10 @@ namespace extension { } } - void NUClearNetwork::reset(const std::string& name, const std::string& address, in_port_t port, + const std::string& bind_address, uint16_t network_mtu) { // Close our existing FDs if they exist @@ -360,57 +337,29 @@ namespace extension { targets.clear(); udp_target.clear(); - // Setup some hints for what our address is - addrinfo hints{}; - std::memset(&hints, 0, sizeof hints); // make sure the struct is empty - hints.ai_family = AF_UNSPEC; // don't care about IPv4 or IPv6 - hints.ai_socktype = SOCK_DGRAM; // using udp datagrams - - // Get our info on this address - addrinfo* servinfo = nullptr; - const int status = ::getaddrinfo(address.c_str(), std::to_string(port).c_str(), &hints, &servinfo); - - // Check if we have any addresses to work with - if (status < 0 || servinfo == nullptr) { - throw std::runtime_error(std::string("The multicast address provided (") + address - + std::string(") was invalid (error ") + std::to_string(status) - + std::string(")")); - } - - // Clear our struct - sock_t announce_target{}; - std::memset(&announce_target, 0, sizeof(announce_target)); - - // The list is actually a linked list of valid addresses - // The address we choose is in the following priority, IPv4, IPv6, Other - for (addrinfo* p = servinfo; p != nullptr; p = p->ai_next) { - - // If we find an IPv4 address, prefer that - if (servinfo->ai_family == AF_INET) { - - // Clear and set our struct - std::memset(&announce_target, 0, sizeof(announce_target)); - std::memcpy(&announce_target, servinfo->ai_addr, servinfo->ai_addrlen); + // Resolve the announce address and port into a sockaddr + const util::network::sock_t announce_target = util::network::resolve(address, port); - // We prefer IPv4 so use it and stop looking - break; + // If we have a bind address, resolve it otherwise use the announce address family with any address + sock_t bind_target{}; + if (bind_address.empty()) { + bind_target = announce_target; + if (announce_target.sock.sa_family == AF_INET) { + bind_target.ipv4.sin_addr.s_addr = htonl(INADDR_ANY); } - // If we find an IPv6 address we set that for now - if (servinfo->ai_family == AF_INET6) { - - // Clear and set our struct - std::memset(&announce_target, 0, sizeof(announce_target)); - std::memcpy(&announce_target, servinfo->ai_addr, servinfo->ai_addrlen); + else if (announce_target.sock.sa_family == AF_INET6) { + bind_target.ipv6.sin6_addr = IN6ADDR_ANY_INIT; + } + else { + throw std::runtime_error("Unknown address family"); } } - - // Clean up our memory - ::freeaddrinfo(servinfo); - - // If we couldn't find a useable address die - if (announce_target.sock.sa_family != AF_INET && announce_target.sock.sa_family != AF_INET6) { - throw std::runtime_error(std::string("The network address provided (") + address - + ") was not a valid multicast address"); + else { + bind_target = util::network::resolve(bind_address, port); + // If the family doesn't match, throw an error + if (bind_target.sock.sa_family != announce_target.sock.sa_family) { + throw std::runtime_error("Bind address family does not match announce address family"); + } } // Add the target for our multicast packets @@ -433,12 +382,17 @@ namespace extension { pkt = AnnouncePacket(); std::memcpy(&pkt.name, name.c_str(), name.size()); - // Open our data socket and then our multicast one - - open_data(announce_target); - open_announce(announce_target); + // Open the data and announce sockets + open_data(bind_target); + open_announce(announce_target, bind_target); } + void NUClearNetwork::reset(const std::string& name, + const std::string& address, + in_port_t port, + uint16_t network_mtu) { + reset(name, address, port, "", network_mtu); + } void NUClearNetwork::process() { @@ -568,7 +522,6 @@ namespace extension { } } - void NUClearNetwork::announce() { // Get all our targets that are global targets @@ -590,7 +543,6 @@ namespace extension { } } - void NUClearNetwork::process_packet(const sock_t& address, std::vector&& payload) { // First validate this is a NUClear network packet we can read (a version 2 NUClear packet) diff --git a/src/extension/network/NUClearNetwork.hpp b/src/extension/network/NUClearNetwork.hpp index a83828df2..32c10b400 100644 --- a/src/extension/network/NUClearNetwork.hpp +++ b/src/extension/network/NUClearNetwork.hpp @@ -171,8 +171,14 @@ namespace extension { * @param name the name of this node in the network * @param address the address to announce on * @param port the port to use for announcement + * @param bind_address the address to bind to (if blank will bind to all interfaces) * @param network_mtu the mtu of the network we operate on */ + void reset(const std::string& name, + const std::string& address, + in_port_t port, + const std::string& bind_address, + uint16_t network_mtu = 1500); void reset(const std::string& name, const std::string& address, in_port_t port, @@ -223,13 +229,18 @@ namespace extension { /** * @brief Open our data udp socket + * + * @param bind_address the address to bind to or any to bind to all interfaces */ - void open_data(const sock_t& announce_target); + void open_data(const sock_t& bind_address); /** * @brief Open our announce udp socket + * + * @param announce_target the target to announce to + * @param bind_address the address to bind to or any to bind to all interfaces */ - void open_announce(const sock_t& announce_target); + void open_announce(const sock_t& announce_target, const sock_t& bind_address); /** * @brief Processes the given packet and calls the callback if a packet was completed diff --git a/src/message/NetworkConfiguration.hpp b/src/message/NetworkConfiguration.hpp index 4ff3514a5..da2ce81d7 100644 --- a/src/message/NetworkConfiguration.hpp +++ b/src/message/NetworkConfiguration.hpp @@ -19,6 +19,8 @@ #ifndef NUCLEAR_MESSAGE_NETWORKCONFIGURATION_HPP #define NUCLEAR_MESSAGE_NETWORKCONFIGURATION_HPP +#include + namespace NUClear { namespace message { @@ -26,12 +28,26 @@ namespace message { NetworkConfiguration() = default; - NetworkConfiguration(std::string name, std::string address, uint16_t port, uint16_t mtu = 1500) - : name(std::move(name)), announce_address(std::move(address)), announce_port(port), mtu(mtu) {} - + NetworkConfiguration(std::string name, + std::string address, + uint16_t port, + std::string bind_address = "", + uint16_t mtu = 1500) + : name(std::move(name)) + , announce_address(std::move(address)) + , announce_port(port) + , bind_address(std::move(bind_address)) + , mtu(mtu) {} + + /// @brief The name of this node when connecting to the NUClear network std::string name{}; + /// @brief The address to announce to the NUClear network std::string announce_address{}; + /// @brief The port to announce to the NUClear network uint16_t announce_port{0}; + /// @brief The address of the interface to bind to when connecting to the NUClear network + std::string bind_address{}; + /// @brief The maximum transmission unit for this node uint16_t mtu{1500}; }; From b0170d570a1e3e53544894eb7a7c9e73c5356b38 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 24 Aug 2023 16:16:19 +1000 Subject: [PATCH 123/176] fixes from osx --- src/dsl/word/emit/UDP.hpp | 8 +++--- src/util/platform.hpp | 4 +++ tests/dsl/UDP.cpp | 7 +++--- tests/util/network/resolve.cpp | 46 +++++++++++++++++++++++----------- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/dsl/word/emit/UDP.hpp b/src/dsl/word/emit/UDP.hpp index a3a7804c5..8d57d828e 100644 --- a/src/dsl/word/emit/UDP.hpp +++ b/src/dsl/word/emit/UDP.hpp @@ -104,7 +104,7 @@ namespace dsl { // If we are using multicast and we have a specific from_addr we need to tell the system to use the // correct interface if (multicast && !from_addr.empty()) { - if (local.sock.sa_family = AF_INET) { + if (local.sock.sa_family == AF_INET) { // Set our transmission interface for the multicast socket if (::setsockopt(fd, IPPROTO_IP, @@ -117,14 +117,14 @@ namespace dsl { "We were unable to use the requested interface for multicast"); } } - else if (local.sock.sa_family = AF_INET6) { + else if (local.sock.sa_family == AF_INET6) { // Set our transmission interface for the multicast socket auto if_number = util::network::if_number_from_address(local.ipv6); if (::setsockopt(fd, IPPROTO_IPV6, IPV6_MULTICAST_IF, - reinterpret_cast(&local.ipv6.sin6_addr), - sizeof(local.ipv6.sin6_addr)) + reinterpret_cast(&if_number), + sizeof(if_number)) < 0) { throw std::system_error(network_errno, std::system_category(), diff --git a/src/util/platform.hpp b/src/util/platform.hpp index f06a6389e..da25d4cd7 100644 --- a/src/util/platform.hpp +++ b/src/util/platform.hpp @@ -99,6 +99,10 @@ /******************************************* * SHIM FOR NETWORKING * *******************************************/ +#if defined(__APPLE__) && defined(__MACH__) + // On OSX the IPV6_RECVPKTINFO and IPV6_PKTINFO must be enabled using this define + #define __APPLE_USE_RFC_3542 +#endif #ifdef _WIN32 #include diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index 9daa92999..f62722c20 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -34,6 +34,9 @@ enum TestPorts { MULTICAST_V6 = 40004, }; +const std::string IPV4_MULTICAST_ADDRESS = "230.12.3.22"; // NOLINT(cert-err58-cpp) +const std::string IPV6_MULTICAST_ADDRESS = "ff02::230:12:3:22"; // NOLINT(cert-err58-cpp) + // Ephemeral ports that we will use in_port_t uni_v4_port = 0; in_port_t uni_v6_port = 0; @@ -41,10 +44,6 @@ in_port_t broad_v4_port = 0; in_port_t multi_v4_port = 0; in_port_t multi_v6_port = 0; -constexpr in_port_t BASE_PORT = 40000; -const std::string IPV4_MULTICAST_ADDRESS = "230.12.3.22"; // NOLINT(cert-err58-cpp) -const std::string IPV6_MULTICAST_ADDRESS = "ff02::230:12:3:22"; // NOLINT(cert-err58-cpp) - inline std::string get_broadcast_addr() { static std::string addr = ""; diff --git a/tests/util/network/resolve.cpp b/tests/util/network/resolve.cpp index 2f48ad06c..b67e7ea31 100644 --- a/tests/util/network/resolve.cpp +++ b/tests/util/network/resolve.cpp @@ -23,10 +23,12 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r REQUIRE(result.sock.sa_family == AF_INET6); REQUIRE(ntohs(result.ipv6.sin6_port) == port); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[0]) == 0); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[1]) == 0); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[2]) == 0); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[3]) == 1); + // Check each byte of the address except the last one is 0 + for (int i = 0; i < 15; i++) { + REQUIRE(result.ipv6.sin6_addr.s6_addr[i] == 0); + } + // Last byte should be 1 + REQUIRE(result.ipv6.sin6_addr.s6_addr[15] == 1); } SECTION("Hostname") { @@ -45,10 +47,12 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r } else { REQUIRE(ntohs(result.ipv6.sin6_port) == port); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[0]) == 0); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[1]) == 0); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[2]) == 0); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[3]) == 1); + // Check each byte of the address except the last one is 0 + for (int i = 0; i < 15; i++) { + REQUIRE(result.ipv6.sin6_addr.s6_addr[i] == 0); + } + // Last byte should be 1 + REQUIRE(result.ipv6.sin6_addr.s6_addr[15] == 1); } } @@ -71,10 +75,22 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r REQUIRE(result.sock.sa_family == AF_INET6); REQUIRE(ntohs(result.ipv6.sin6_port) == port); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[0]) == 0x20010db8); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[1]) == 0xac10fe01); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[2]) == 0); - REQUIRE(ntohl(result.ipv6.sin6_addr.s6_addr32[3]) == 0); + REQUIRE(result.ipv6.sin6_addr.s6_addr[0] == 0x20); + REQUIRE(result.ipv6.sin6_addr.s6_addr[1] == 0x01); + REQUIRE(result.ipv6.sin6_addr.s6_addr[2] == 0x0d); + REQUIRE(result.ipv6.sin6_addr.s6_addr[3] == 0xb8); + REQUIRE(result.ipv6.sin6_addr.s6_addr[4] == 0xac); + REQUIRE(result.ipv6.sin6_addr.s6_addr[5] == 0x10); + REQUIRE(result.ipv6.sin6_addr.s6_addr[6] == 0xfe); + REQUIRE(result.ipv6.sin6_addr.s6_addr[7] == 0x01); + REQUIRE(result.ipv6.sin6_addr.s6_addr[8] == 0x00); + REQUIRE(result.ipv6.sin6_addr.s6_addr[9] == 0x00); + REQUIRE(result.ipv6.sin6_addr.s6_addr[10] == 0x00); + REQUIRE(result.ipv6.sin6_addr.s6_addr[11] == 0x00); + REQUIRE(result.ipv6.sin6_addr.s6_addr[12] == 0x00); + REQUIRE(result.ipv6.sin6_addr.s6_addr[13] == 0x00); + REQUIRE(result.ipv6.sin6_addr.s6_addr[14] == 0x00); + REQUIRE(result.ipv6.sin6_addr.s6_addr[15] == 0x00); } SECTION("Hostname with valid IPv4 address") { @@ -97,10 +113,10 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r REQUIRE(result.sock.sa_family == AF_INET6); REQUIRE(ntohs(result.ipv6.sin6_port) == port); - // Check if all compoments are zero + // Check if all components are zero bool nonzero = false; - for (int i = 0; i < 4; i++) { - nonzero |= (ntohl(result.ipv6.sin6_addr.s6_addr32[i]) != 0); + for (int i = 0; i < 15; i++) { + nonzero |= result.ipv6.sin6_addr.s6_addr[i] != 0; } REQUIRE(nonzero); From 238409e54f3479aa6e29bde52148520a1e5f0b24 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 24 Aug 2023 16:24:51 +1000 Subject: [PATCH 124/176] clang-tidy --- src/dsl/word/TCP.hpp | 2 +- src/dsl/word/UDP.hpp | 10 ++++++---- src/dsl/word/emit/UDP.hpp | 2 +- src/extension/network/NUClearNetwork.cpp | 18 +++++++++--------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/dsl/word/TCP.hpp b/src/dsl/word/TCP.hpp index aafac5e1b..78044189c 100644 --- a/src/dsl/word/TCP.hpp +++ b/src/dsl/word/TCP.hpp @@ -167,7 +167,7 @@ namespace dsl { // If our get is being run without an fd (something else triggered) then short circuit if (!event) { - return Connection{{0, 0}, {0, 0}, 0}; + return {}; } // Accept our connection diff --git a/src/dsl/word/UDP.hpp b/src/dsl/word/UDP.hpp index 6a523b272..4b3e643d0 100644 --- a/src/dsl/word/UDP.hpp +++ b/src/dsl/word/UDP.hpp @@ -311,7 +311,7 @@ namespace dsl { std::system_category(), "We were unable to get the port from the UDP socket"); } - in_port_t port; + in_port_t port = 0; if (bind_address.sock.sa_family == AF_INET) { port = ntohs(bind_address.ipv4.sin_port); } @@ -382,7 +382,7 @@ namespace dsl { // Iterate through control headers to get IP information for (cmsghdr* cmsg = CMSG_FIRSTHDR(&mh); cmsg != nullptr; cmsg = CMSG_NXTHDR(&mh, cmsg)) { - // ignore the control headers that don't match what we want + // If we find an ipv4 packet info header if (local.sock.sa_family == AF_INET && cmsg->cmsg_level == IPPROTO_IP && cmsg->cmsg_type == IP_PKTINFO) { @@ -394,8 +394,10 @@ namespace dsl { // We are done break; } - else if (local.sock.sa_family == AF_INET6 && cmsg->cmsg_level == IPPROTO_IPV6 - && cmsg->cmsg_type == IPV6_PKTINFO) { + + // If we find a ipv6 packet info header + if (local.sock.sa_family == AF_INET6 && cmsg->cmsg_level == IPPROTO_IPV6 + && cmsg->cmsg_type == IPV6_PKTINFO) { // Access the packet header information auto* pi = reinterpret_cast(reinterpret_cast(cmsg) + sizeof(*cmsg)); diff --git a/src/dsl/word/emit/UDP.hpp b/src/dsl/word/emit/UDP.hpp index 8d57d828e..7d7df595a 100644 --- a/src/dsl/word/emit/UDP.hpp +++ b/src/dsl/word/emit/UDP.hpp @@ -87,7 +87,7 @@ namespace dsl { } } else { - util::network::sock_t local = util::network::resolve(from_addr, from_port); + local = util::network::resolve(from_addr, from_port); if (local.sock.sa_family != remote.sock.sa_family) { throw std::runtime_error("to and from addresses are not the same family"); } diff --git a/src/extension/network/NUClearNetwork.cpp b/src/extension/network/NUClearNetwork.cpp index f1932fa1b..9d2b023ab 100644 --- a/src/extension/network/NUClearNetwork.cpp +++ b/src/extension/network/NUClearNetwork.cpp @@ -146,10 +146,10 @@ namespace extension { } - void NUClearNetwork::open_data(const sock_t& bind_target) { + void NUClearNetwork::open_data(const sock_t& bind_address) { // Create the "join any" address for this address family - sock_t address = bind_target; + sock_t address = bind_address; // Set port to 0 to get an ephemeral port if (address.sock.sa_family == AF_INET) { @@ -181,7 +181,7 @@ namespace extension { } - void NUClearNetwork::open_announce(const sock_t& announce_target, const sock_t& bind_target) { + void NUClearNetwork::open_announce(const sock_t& announce_target, const sock_t& bind_address) { // Work out what type of announce we are doing as it will influence how we make the socket const bool multicast = @@ -190,7 +190,7 @@ namespace extension { || (announce_target.sock.sa_family == AF_INET6 && announce_target.ipv6.sin6_addr.s6_addr[0] == 0xFF); // Make our socket - announce_fd = ::socket(bind_target.sock.sa_family, SOCK_DGRAM, IPPROTO_UDP); + announce_fd = ::socket(bind_address.sock.sa_family, SOCK_DGRAM, IPPROTO_UDP); if (announce_fd < 0) { throw std::system_error(network_errno, std::system_category(), "Unable to open the UDP socket"); } @@ -214,7 +214,7 @@ namespace extension { } // Bind to the address - if (::bind(announce_fd, &bind_target.sock, bind_target.size()) != 0) { + if (::bind(announce_fd, &bind_address.sock, bind_address.size()) != 0) { throw std::system_error(network_errno, std::system_category(), "Unable to bind the UDP socket"); } @@ -227,7 +227,7 @@ namespace extension { // Set the multicast address we are listening on and bind address ip_mreq mreq{}; mreq.imr_multiaddr = announce_target.ipv4.sin_addr; - mreq.imr_interface = bind_target.ipv4.sin_addr; + mreq.imr_interface = bind_address.ipv4.sin_addr; // Join our multicast group if (::setsockopt(announce_fd, @@ -245,8 +245,8 @@ namespace extension { if (::setsockopt(announce_fd, IPPROTO_IP, IP_MULTICAST_IF, - reinterpret_cast(&bind_target.ipv4.sin_addr), - sizeof(bind_target.ipv4.sin_addr)) + reinterpret_cast(&bind_address.ipv4.sin_addr), + sizeof(bind_address.ipv4.sin_addr)) < 0) { throw std::system_error(network_errno, std::system_category(), @@ -258,7 +258,7 @@ namespace extension { // Set the multicast address we are listening on ipv6_mreq mreq{}; mreq.ipv6mr_multiaddr = announce_target.ipv6.sin6_addr; - mreq.ipv6mr_interface = util::network::if_number_from_address(bind_target.ipv6); + mreq.ipv6mr_interface = util::network::if_number_from_address(bind_address.ipv6); // Join our multicast group if (::setsockopt(announce_fd, From ce31581b0c0df6d5f7d12ca48e3e17eb5d5df6f2 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 24 Aug 2023 16:39:32 +1000 Subject: [PATCH 125/176] clang-tidy --- src/dsl/word/TCP.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dsl/word/TCP.hpp b/src/dsl/word/TCP.hpp index 78044189c..d60003170 100644 --- a/src/dsl/word/TCP.hpp +++ b/src/dsl/word/TCP.hpp @@ -183,8 +183,8 @@ namespace dsl { socklen_t local_size = sizeof(util::network::sock_t); ::getsockname(fd, &local.sock, &local_size); - if (fd == INVALID_SOCKET) { - return Connection{{0, 0}, {0, 0}, 0}; + if (!fd.valid()) { + return Connection{}; } auto local_s = local.address(); From 967c202cd5a5d5b63012c187b7b5cbeca857835a Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 24 Aug 2023 16:49:58 +1000 Subject: [PATCH 126/176] Bind IPv6 tests to any rather than localhost --- tests/dsl/TCP.cpp | 4 ++-- tests/dsl/UDP.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index bb67b1513..34e507262 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -83,14 +83,14 @@ class TestReactor : public test_util::TestBase { in_port_t v4_port = std::get<1>(v4); // Bind to IPv6 and a known port - on(KNOWN_V6_PORT, "::1").then([this](const TCP::Connection& connection) { + on(KNOWN_V6_PORT, "::").then([this](const TCP::Connection& connection) { on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { handle_data("v6 Known", event); }); }); // Bind to IPv6 an unknown port and get the port number - auto v6 = on(0, "::1").then([this](const TCP::Connection& connection) { + auto v6 = on(0, "::").then([this](const TCP::Connection& connection) { on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { handle_data("v6 Ephemeral", event); }); diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index f62722c20..f36183406 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -143,12 +143,12 @@ class TestReactor : public test_util::TestBase { uni_v4_port = std::get<1>(uni_v4); // IPv6 Unicast Known port - on(UNICAST_V6, "::1").then([this](const UDP::Packet& packet) { // + on(UNICAST_V6, "::").then([this](const UDP::Packet& packet) { // handle_data("Uv6K", packet); }); // IPv6 Unicast Ephemeral port - auto uni_v6 = on(0, "::1").then([this](const UDP::Packet& packet) { // + auto uni_v6 = on(0, "::").then([this](const UDP::Packet& packet) { // handle_data("Uv6E", packet); }); uni_v6_port = std::get<1>(uni_v6); From aa9260f29e3a1e8219dd64f4812be3d40385fd6f Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 31 Aug 2023 22:24:18 +1000 Subject: [PATCH 127/176] Fix IPv6 problems on osx --- src/dsl/word/UDP.hpp | 1 - src/util/network/get_interfaces.cpp | 57 ++++++++++---------------- src/util/network/get_interfaces.hpp | 20 +++++++-- src/util/network/resolve.cpp | 4 +- tests/dsl/UDP.cpp | 63 ++++++++++++++++++----------- tests/util/network/resolve.cpp | 2 +- 6 files changed, 79 insertions(+), 68 deletions(-) diff --git a/src/dsl/word/UDP.hpp b/src/dsl/word/UDP.hpp index 4b3e643d0..b5b10c221 100644 --- a/src/dsl/word/UDP.hpp +++ b/src/dsl/word/UDP.hpp @@ -329,7 +329,6 @@ namespace dsl { // Return our handles and our bound port return std::make_tuple(port, cfd); - return std::make_tuple(in_port_t(0), fd_t(0)); } template diff --git a/src/util/network/get_interfaces.cpp b/src/util/network/get_interfaces.cpp index d7952a25c..fcf90ecbe 100644 --- a/src/util/network/get_interfaces.cpp +++ b/src/util/network/get_interfaces.cpp @@ -62,8 +62,7 @@ namespace util { for (auto uaddr = addr->FirstUnicastAddress; uaddr != nullptr; uaddr = uaddr->Next) { - Interface iface; - std::memset(&iface, 0, sizeof(iface)); + Interface iface{}; iface.name = addr->AdapterName; @@ -159,53 +158,39 @@ namespace util { } // Loop through our interfaces - for (ifaddrs* cursor = addrs; cursor != nullptr; cursor = cursor->ifa_next) { + for (ifaddrs* it = addrs; it != nullptr; it = it->ifa_next) { // Sometimes we find an interface with no IP (like a CAN bus) this is not what we're after - if (cursor->ifa_addr != nullptr) { + if (it->ifa_addr != nullptr) { - Interface iface; - iface.name = cursor->ifa_name; + Interface iface{}; + iface.name = it->ifa_name; // Copy across our various addresses - switch (cursor->ifa_addr->sa_family) { - case AF_INET: std::memcpy(&iface.ip, cursor->ifa_addr, sizeof(sockaddr_in)); break; - - case AF_INET6: std::memcpy(&iface.ip, cursor->ifa_addr, sizeof(sockaddr_in6)); break; + switch (it->ifa_addr->sa_family) { + case AF_INET: std::memcpy(&iface.ip, it->ifa_addr, sizeof(sockaddr_in)); break; + case AF_INET6: std::memcpy(&iface.ip, it->ifa_addr, sizeof(sockaddr_in6)); break; + default: continue; } - if (cursor->ifa_netmask != nullptr) { - switch (cursor->ifa_addr->sa_family) { - case AF_INET: std::memcpy(&iface.netmask, cursor->ifa_netmask, sizeof(sockaddr_in)); break; - - case AF_INET6: - std::memcpy(&iface.netmask, cursor->ifa_netmask, sizeof(sockaddr_in6)); - break; + if (it->ifa_netmask != nullptr) { + switch (it->ifa_addr->sa_family) { + case AF_INET: std::memcpy(&iface.netmask, it->ifa_netmask, sizeof(sockaddr_in)); break; + case AF_INET6: std::memcpy(&iface.netmask, it->ifa_netmask, sizeof(sockaddr_in6)); break; } } - else { - std::memset(&iface.netmask, 0, sizeof(iface.netmask)); - } - if (cursor->ifa_dstaddr != nullptr) { - switch (cursor->ifa_addr->sa_family) { - case AF_INET: - std::memcpy(&iface.broadcast, cursor->ifa_dstaddr, sizeof(sockaddr_in)); - break; - - case AF_INET6: - std::memcpy(&iface.broadcast, cursor->ifa_dstaddr, sizeof(sockaddr_in6)); - break; + if (it->ifa_dstaddr != nullptr) { + switch (it->ifa_addr->sa_family) { + case AF_INET: std::memcpy(&iface.broadcast, it->ifa_dstaddr, sizeof(sockaddr_in)); break; + case AF_INET6: std::memcpy(&iface.broadcast, it->ifa_dstaddr, sizeof(sockaddr_in6)); break; } } - else { - std::memset(&iface.broadcast, 0, sizeof(iface.broadcast)); - } - iface.flags.broadcast = (cursor->ifa_flags & IFF_BROADCAST) != 0; - iface.flags.loopback = (cursor->ifa_flags & IFF_LOOPBACK) != 0; - iface.flags.pointtopoint = (cursor->ifa_flags & IFF_POINTOPOINT) != 0; - iface.flags.multicast = (cursor->ifa_flags & IFF_MULTICAST) != 0; + iface.flags.broadcast = (it->ifa_flags & IFF_BROADCAST) != 0; + iface.flags.loopback = (it->ifa_flags & IFF_LOOPBACK) != 0; + iface.flags.pointtopoint = (it->ifa_flags & IFF_POINTOPOINT) != 0; + iface.flags.multicast = (it->ifa_flags & IFF_MULTICAST) != 0; ifaces.push_back(iface); } diff --git a/src/util/network/get_interfaces.hpp b/src/util/network/get_interfaces.hpp index 5e89a34a7..6f8985c46 100644 --- a/src/util/network/get_interfaces.hpp +++ b/src/util/network/get_interfaces.hpp @@ -28,25 +28,37 @@ namespace NUClear { namespace util { namespace network { + /** + * @brief A structure that contains information about a network interface + */ struct Interface { - Interface() = default; - + /// @brief The name of the interface std::string name{}; + /// @brief The address that is bound to the interface sock_t ip{}; + /// @brief The netmask of the interface sock_t netmask{}; + /// @brief The broadcast address of the interface or point to point address sock_t broadcast{}; struct Flags { - Flags() = default; - + /// @brief True if the interface is a broadcast interface bool broadcast{false}; + /// @brief True if the interface is a loopback interface bool loopback{false}; + /// @brief True if the interface is a point to point interface bool pointtopoint{false}; + /// @brief True if the interface is a multicast interface bool multicast{false}; } flags; }; + /** + * @brief Gets a list of all the network interfaces on the system with the addresses they are bound to + * + * @return a list of all the interfaces on the system + */ std::vector get_interfaces(); } // namespace network diff --git a/src/util/network/resolve.cpp b/src/util/network/resolve.cpp index c51d89f7d..be19f8f7e 100644 --- a/src/util/network/resolve.cpp +++ b/src/util/network/resolve.cpp @@ -33,6 +33,7 @@ namespace util { addrinfo hints{}; hints.ai_family = AF_UNSPEC; // don't care about IPv4 or IPv6 hints.ai_socktype = AF_UNSPEC; // don't care about TCP or UDP + hints.ai_flags = AI_ALL; // Get all addresses even if we don't have an interface for them // Get our info on this address addrinfo* servinfo_ptr = nullptr; @@ -50,9 +51,8 @@ namespace util { std::unique_ptr servinfo(servinfo_ptr, ::freeaddrinfo); - // Clear our struct + // Empty sock_t struct NUClear::util::network::sock_t target{}; - std::memset(&target, 0, sizeof(target)); // The list is actually a linked list of valid addresses // The address we choose is in the following priority, IPv4, IPv6, Other diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index f36183406..1ac11af3d 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -69,9 +69,13 @@ inline std::string get_broadcast_addr() { } struct SendTarget { - std::string data; - std::string address; - in_port_t port; + std::string data{}; + struct Target { + std::string address = ""; + in_port_t port = 0; + }; + Target to{}; + Target from{}; }; std::vector send_targets(const std::string& type, in_port_t port, bool include_target = false) { std::vector results; @@ -79,19 +83,20 @@ std::vector send_targets(const std::string& type, in_port_t port, bo // Make sure that the type we are actually after is sent last std::string t = type.substr(0, 3); if (type[2] == '4' && include_target == (t == "Uv4")) { - results.push_back(SendTarget{type + ":Uv4", "127.0.0.1", port}); + results.push_back(SendTarget{type + ":Uv4", {"127.0.0.1", port}, {}}); } if (type[2] == '4' && include_target == (t == "Bv4")) { - results.push_back(SendTarget{type + ":Bv4", get_broadcast_addr(), port}); + results.push_back(SendTarget{type + ":Bv4", {get_broadcast_addr(), port}, {}}); } if (type[2] == '4' && include_target == (t == "Mv4")) { - results.push_back(SendTarget{type + ":Mv4", IPV4_MULTICAST_ADDRESS, port}); + results.push_back(SendTarget{type + ":Mv4", {IPV4_MULTICAST_ADDRESS, port}, {}}); } if (type[2] == '6' && include_target == (t == "Uv6")) { - results.push_back(SendTarget{type + ":Uv6", "::1", port}); + results.push_back(SendTarget{type + ":Uv6", {"::1", port}, {}}); } if (type[2] == '6' && include_target == (t == "Mv6")) { - results.push_back(SendTarget{type + ":Mv6", IPV6_MULTICAST_ADDRESS, port}); + // For multicast v6 send from localhost so that it works on OSX + results.push_back(SendTarget{type + ":Mv6", {IPV6_MULTICAST_ADDRESS, port}, {"::1", 0}}); } if (!include_target) { auto target = send_targets(type, port, true); @@ -167,23 +172,29 @@ class TestReactor : public test_util::TestBase { // No such thing as broadcast in IPv6 // IPv4 Multicast Known port - on(IPV4_MULTICAST_ADDRESS, MULTICAST_V4).then([this](const UDP::Packet& packet) { // + on(IPV4_MULTICAST_ADDRESS, MULTICAST_V4).then([this](const UDP::Packet& packet) { handle_data("Mv4K", packet); }); // IPv4 Multicast Ephemeral port - auto multi_v4 = on(IPV4_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { // + auto multi_v4 = on(IPV4_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { handle_data("Mv4E", packet); }); multi_v4_port = std::get<1>(multi_v4); + // For the IPv6 test we need to bind to the IPv6 localhost address and send from it when using udp emit + // This is because on OSX by default there is no default route for IPv6 multicast packets (see `netstat -nr`) + // As a result if you don't specify an interface to use when sending and receiving IPv6 multicast packets + // the send/bind fails which makes the tests not work. Linux does not care about this extra step so it doesn't + // break the tests + // IPv6 Multicast Known port - on(IPV6_MULTICAST_ADDRESS, MULTICAST_V6).then([this](const UDP::Packet& packet) { // + on(IPV6_MULTICAST_ADDRESS, MULTICAST_V6, "::1").then([this](const UDP::Packet& packet) { handle_data("Mv6K", packet); }); // IPv6 Multicast Ephemeral port - auto multi_v6 = on(IPV6_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { // + auto multi_v6 = on(IPV6_MULTICAST_ADDRESS, 0, "::1").then([this](const UDP::Packet& packet) { handle_data("Mv6E", packet); }); multi_v6_port = std::get<1>(multi_v6); @@ -197,8 +208,12 @@ class TestReactor : public test_util::TestBase { on>().then([=](const Finished& test) { auto send_all = [this](std::string type, in_port_t port) { for (const auto& t : send_targets(type, port)) { - events.push_back(" -> " + t.address + ":" + std::to_string(t.port)); - emit(std::make_unique(t.data), t.address, t.port); + events.push_back(" -> " + t.to.address + ":" + std::to_string(t.to.port)); + emit(std::make_unique(t.data), + t.to.address, + t.to.port, + t.from.address, + t.from.port); } }; @@ -280,70 +295,70 @@ TEST_CASE("Testing sending and receiving of UDP messages", "[api][network][udp]" std::vector expected; expected.push_back("- Known Unicast V4 Test -"); for (const auto& line : send_targets("Uv4K", UNICAST_V4)) { - expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); } expected.push_back("Uv4K <- Uv4K:Uv4 (127.0.0.1:" + std::to_string(UNICAST_V4) + ")"); expected.push_back(""); expected.push_back("- Ephemeral Unicast V4 Test -"); for (const auto& line : send_targets("Uv4E", uni_v4_port)) { - expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); } expected.push_back("Uv4E <- Uv4E:Uv4 (127.0.0.1:" + std::to_string(uni_v4_port) + ")"); expected.push_back(""); expected.push_back("- Known Unicast V6 Test -"); for (const auto& line : send_targets("Uv6K", UNICAST_V6)) { - expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); } expected.push_back("Uv6K <- Uv6K:Uv6 (::1:" + std::to_string(UNICAST_V6) + ")"); expected.push_back(""); expected.push_back("- Ephemeral Unicast V6 Test -"); for (const auto& line : send_targets("Uv6E", uni_v6_port)) { - expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); } expected.push_back("Uv6E <- Uv6E:Uv6 (::1:" + std::to_string(uni_v6_port) + ")"); expected.push_back(""); expected.push_back("- Known Broadcast V4 Test -"); for (const auto& line : send_targets("Bv4K", BROADCAST_V4)) { - expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); } expected.push_back("Bv4K <- Bv4K:Bv4 (" + get_broadcast_addr() + ":" + std::to_string(BROADCAST_V4) + ")"); expected.push_back(""); expected.push_back("- Ephemeral Broadcast V4 Test -"); for (const auto& line : send_targets("Bv4E", broad_v4_port)) { - expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); } expected.push_back("Bv4E <- Bv4E:Bv4 (" + get_broadcast_addr() + ":" + std::to_string(broad_v4_port) + ")"); expected.push_back(""); expected.push_back("- Known Multicast V4 Test -"); for (const auto& line : send_targets("Mv4K", MULTICAST_V4)) { - expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); } expected.push_back("Mv4K <- Mv4K:Mv4 (" + IPV4_MULTICAST_ADDRESS + ":" + std::to_string(MULTICAST_V4) + ")"); expected.push_back(""); expected.push_back("- Ephemeral Multicast V4 Test -"); for (const auto& line : send_targets("Mv4E", multi_v4_port)) { - expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); } expected.push_back("Mv4E <- Mv4E:Mv4 (" + IPV4_MULTICAST_ADDRESS + ":" + std::to_string(multi_v4_port) + ")"); expected.push_back(""); expected.push_back("- Known Multicast V6 Test -"); for (const auto& line : send_targets("Mv6K", MULTICAST_V6)) { - expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); } expected.push_back("Mv6K <- Mv6K:Mv6 (" + IPV6_MULTICAST_ADDRESS + ":" + std::to_string(MULTICAST_V6) + ")"); expected.push_back(""); expected.push_back("- Ephemeral Multicast V6 Test -"); for (const auto& line : send_targets("Mv6E", multi_v6_port)) { - expected.push_back(" -> " + line.address + ":" + std::to_string(line.port)); + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); } expected.push_back("Mv6E <- Mv6E:Mv6 (" + IPV6_MULTICAST_ADDRESS + ":" + std::to_string(multi_v6_port) + ")"); diff --git a/tests/util/network/resolve.cpp b/tests/util/network/resolve.cpp index b67e7ea31..b73488b60 100644 --- a/tests/util/network/resolve.cpp +++ b/tests/util/network/resolve.cpp @@ -123,7 +123,7 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r } SECTION("Invalid address") { - std::string address = "notahost"; + std::string address = "this.url.is.invalid"; uint16_t port = 12345; // Check that the function throws a std::runtime_error with the appropriate message From 6389d8ba87604b89d6824830abdd163e760c8d9e Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 31 Aug 2023 22:30:40 +1000 Subject: [PATCH 128/176] Remove unused code --- src/util/network/get_interfaces.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/util/network/get_interfaces.cpp b/src/util/network/get_interfaces.cpp index fcf90ecbe..a4eeb9a5a 100644 --- a/src/util/network/get_interfaces.cpp +++ b/src/util/network/get_interfaces.cpp @@ -145,10 +145,6 @@ namespace util { std::vector ifaces; - addrinfo hints{}; - std::memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_INET; - // Query our interfaces ifaddrs* addrs{}; if (::getifaddrs(&addrs) < 0) { From 2f11c3f049e3f06cc5a7d9935a0b60af31feb49d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 31 Aug 2023 22:30:51 +1000 Subject: [PATCH 129/176] No need for memset when doing {} initialisation --- src/extension/network/NUClearNetwork.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/extension/network/NUClearNetwork.cpp b/src/extension/network/NUClearNetwork.cpp index 9d2b023ab..cd96cab24 100644 --- a/src/extension/network/NUClearNetwork.cpp +++ b/src/extension/network/NUClearNetwork.cpp @@ -52,11 +52,9 @@ namespace extension { // Who we are receiving from util::network::sock_t from{}; - std::memset(&from, 0, sizeof(from)); // Setup our message header to receive msghdr mh{}; - std::memset(&mh, 0, sizeof(msghdr)); mh.msg_name = &from.sock; mh.msg_namelen = sizeof(from); mh.msg_iov = &iov; @@ -998,7 +996,6 @@ namespace extension { // Our packet we are sending msghdr message{}; - std::memset(&message, 0, sizeof(msghdr)); iovec data[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) message.msg_iov = static_cast(data); From 27afc807a3004c2a95c6f6adcc0f1ce1244a316e Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 31 Aug 2023 22:33:40 +1000 Subject: [PATCH 130/176] clang-tidy --- src/util/network/if_number_from_address.cpp | 2 +- src/util/network/if_number_from_address.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/network/if_number_from_address.cpp b/src/util/network/if_number_from_address.cpp index 4c75acb75..ac77c945f 100644 --- a/src/util/network/if_number_from_address.cpp +++ b/src/util/network/if_number_from_address.cpp @@ -28,7 +28,7 @@ namespace NUClear { namespace util { namespace network { - int if_number_from_address(const sockaddr_in6& ipv6) { + unsigned int if_number_from_address(const sockaddr_in6& ipv6) { // If the socket is referring to the any address, return 0 to use the default interface if (std::all_of(ipv6.sin6_addr.s6_addr, ipv6.sin6_addr.s6_addr + 16, [](unsigned char i) { diff --git a/src/util/network/if_number_from_address.hpp b/src/util/network/if_number_from_address.hpp index 0ae958f61..06232cb5e 100644 --- a/src/util/network/if_number_from_address.hpp +++ b/src/util/network/if_number_from_address.hpp @@ -35,7 +35,7 @@ namespace util { * * @return int the index of the interface that the address is on or 0 if the ipv6 address is the any address */ - int if_number_from_address(const sockaddr_in6& ipv6); + unsigned int if_number_from_address(const sockaddr_in6& ipv6); } // namespace network } // namespace util From 7b7c99fda4e5554907d2e71a90b3743654611e0b Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Thu, 31 Aug 2023 22:41:08 +1000 Subject: [PATCH 131/176] Try to fix windows build --- src/util/platform.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/util/platform.hpp b/src/util/platform.hpp index da25d4cd7..3e1a659ac 100644 --- a/src/util/platform.hpp +++ b/src/util/platform.hpp @@ -83,6 +83,11 @@ #define SHUT_WR SD_RECEIVE #define SHUT_RDWR SD_BOTH + // Windows always wanting to be different + #ifndef IPV6_RECVPKTINFO + #define IPV6_RECVPKTINFO IPV6_PKTINFO + #endif + #endif // _WIN32 /******************************************* @@ -126,7 +131,6 @@ using fd_t = SOCKET; using socklen_t = int; - // Network errors come from WSAGetLastError() #define network_errno WSAGetLastError() From 41cc1a8995fe70385e7f89c171fac184c4b91e9f Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 1 Sep 2023 12:08:38 +1000 Subject: [PATCH 132/176] clang-tidy --- tests/dsl/TCP.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 34e507262..584b1581b 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -75,12 +75,12 @@ class TestReactor : public test_util::TestBase { }); // Bind to IPv4 an unknown port and get the port number - auto v4 = on().then([this](const TCP::Connection& connection) { + auto v4 = on().then([this](const TCP::Connection& connection) { on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { handle_data("v4 Ephemeral", event); }); }); - in_port_t v4_port = std::get<1>(v4); + const in_port_t& v4_port = std::get<1>(v4); // Bind to IPv6 and a known port on(KNOWN_V6_PORT, "::").then([this](const TCP::Connection& connection) { @@ -90,17 +90,17 @@ class TestReactor : public test_util::TestBase { }); // Bind to IPv6 an unknown port and get the port number - auto v6 = on(0, "::").then([this](const TCP::Connection& connection) { + auto v6 = on(0, "::").then([this](const TCP::Connection& connection) { on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { handle_data("v6 Ephemeral", event); }); }); - in_port_t v6_port = std::get<1>(v6); + const in_port_t& v6_port = std::get<1>(v6); // Send a test message to the known port on, Sync>().then([](const TestConnection& target) { // Resolve the target address - NUClear::util::network::sock_t address = NUClear::util::network::resolve(target.address, target.port); + const NUClear::util::network::sock_t address = NUClear::util::network::resolve(target.address, target.port); // Open a random socket NUClear::util::FileDescriptor fd(::socket(address.sock.sa_family, SOCK_STREAM, IPPROTO_TCP), From 236522d21b70b45695f872980b43f8e94994370e Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 4 Sep 2023 10:27:35 +1000 Subject: [PATCH 133/176] clang-tidy --- tests/dsl/UDP.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index 1ac11af3d..ded6d3bfb 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -38,14 +38,14 @@ const std::string IPV4_MULTICAST_ADDRESS = "230.12.3.22"; // NOLINT(cert- const std::string IPV6_MULTICAST_ADDRESS = "ff02::230:12:3:22"; // NOLINT(cert-err58-cpp) // Ephemeral ports that we will use -in_port_t uni_v4_port = 0; -in_port_t uni_v6_port = 0; -in_port_t broad_v4_port = 0; -in_port_t multi_v4_port = 0; -in_port_t multi_v6_port = 0; +in_port_t uni_v4_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +in_port_t uni_v6_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +in_port_t broad_v4_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +in_port_t multi_v4_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +in_port_t multi_v6_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) inline std::string get_broadcast_addr() { - static std::string addr = ""; + static std::string addr{}; if (!addr.empty()) { return addr; @@ -81,7 +81,7 @@ std::vector send_targets(const std::string& type, in_port_t port, bo std::vector results; // Make sure that the type we are actually after is sent last - std::string t = type.substr(0, 3); + const std::string t = type.substr(0, 3); if (type[2] == '4' && include_target == (t == "Uv4")) { results.push_back(SendTarget{type + ":Uv4", {"127.0.0.1", port}, {}}); } @@ -121,10 +121,10 @@ struct Finished { class TestReactor : public test_util::TestBase { private: void handle_data(const std::string& name, const UDP::Packet& packet) { - std::string data(packet.payload.data(), packet.payload.size()); + const std::string data(packet.payload.data(), packet.payload.size()); // Convert IP address to string in dotted decimal format - std::string local = packet.local.address + ":" + std::to_string(packet.local.port); + const std::string local = packet.local.address + ":" + std::to_string(packet.local.port); events.push_back(name + " <- " + data + " (" + local + ")"); @@ -206,7 +206,7 @@ class TestReactor : public test_util::TestBase { }); on>().then([=](const Finished& test) { - auto send_all = [this](std::string type, in_port_t port) { + auto send_all = [this](const std::string& type, const in_port_t& port) { for (const auto& t : send_targets(type, port)) { events.push_back(" -> " + t.to.address + ":" + std::to_string(t.to.port)); emit(std::make_unique(t.data), From 36e9567603042ca463a9730735b8720f386030a7 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 4 Sep 2023 10:29:37 +1000 Subject: [PATCH 134/176] Test for ipv6 to see if the runners have it --- tests/dsl/UDP.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index ded6d3bfb..eb97d6414 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -44,6 +44,16 @@ in_port_t broad_v4_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global in_port_t multi_v4_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) in_port_t multi_v6_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +inline bool has_ipv6() { + // Get the first IPv6 address we can find + for (const auto& iface : NUClear::util::network::get_interfaces()) { + if (iface.ip.sock.sa_family == AF_INET6) { + return true; + } + } + return false; +} + inline std::string get_broadcast_addr() { static std::string addr{}; @@ -286,6 +296,8 @@ class TestReactor : public test_util::TestBase { TEST_CASE("Testing sending and receiving of UDP messages", "[api][network][udp]") { + REQUIRE(has_ipv6()); + NUClear::PowerPlant::Configuration config; config.thread_count = 1; NUClear::PowerPlant plant(config); From c48e91a31be0d7cd4da5df3b663a6144c386bfdf Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 4 Sep 2023 13:54:21 +1000 Subject: [PATCH 135/176] add a test utility to check if any interface has ipv6 --- tests/test_util/has_ipv6.cpp | 34 ++++++++++++++++++++++++++++++++++ tests/test_util/has_ipv6.hpp | 27 +++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 tests/test_util/has_ipv6.cpp create mode 100644 tests/test_util/has_ipv6.hpp diff --git a/tests/test_util/has_ipv6.cpp b/tests/test_util/has_ipv6.cpp new file mode 100644 index 000000000..ae6ba40f6 --- /dev/null +++ b/tests/test_util/has_ipv6.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "has_ipv6.hpp" + +#include "nuclear" + +namespace test_util { + +bool has_ipv6() { + // Get the first IPv6 address we can find + for (const auto& iface : NUClear::util::network::get_interfaces()) { + if (iface.ip.sock.sa_family == AF_INET6) { + return true; + } + } + return false; +} + +} // namespace test_util diff --git a/tests/test_util/has_ipv6.hpp b/tests/test_util/has_ipv6.hpp new file mode 100644 index 000000000..eb852fc46 --- /dev/null +++ b/tests/test_util/has_ipv6.hpp @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef TEST_UTIL_HAS_IPV6_HPP +#define TEST_UTIL_HAS_IPV6_HPP + +namespace test_util { + +bool has_ipv6(); + +} // namespace test_util + +#endif // TEST_UTIL_HAS_IPV6_HPP From 77c184ef06667e140cc2cefb64a569fe265290d0 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 4 Sep 2023 13:54:30 +1000 Subject: [PATCH 136/176] use locally built nuclear in tests --- tests/test_util/TestBase.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_util/TestBase.hpp b/tests/test_util/TestBase.hpp index eef2eb6aa..e2dfd1449 100644 --- a/tests/test_util/TestBase.hpp +++ b/tests/test_util/TestBase.hpp @@ -20,10 +20,10 @@ #define TEST_UTIL_TESTBASE_HPP #include -#include #include #include +#include "nuclear" #include "test_util/diff_string.hpp" namespace test_util { From 547a66463b473809ffd6b6c3213f034af5fc7f28 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 4 Sep 2023 13:55:07 +1000 Subject: [PATCH 137/176] Make the TCP test only do IPv6 tests if there is IPv6 --- tests/dsl/TCP.cpp | 180 +++++++++++++++++++++++++++++----------------- 1 file changed, 113 insertions(+), 67 deletions(-) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 584b1581b..26d5956c8 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -20,6 +20,7 @@ #include #include "test_util/TestBase.hpp" +#include "test_util/has_ipv6.hpp" namespace { @@ -31,6 +32,17 @@ enum TestPorts { KNOWN_V6_PORT = 40011, }; +enum TestType { + V4_KNOWN, + V4_EPHEMERAL, + V6_KNOWN, + V6_EPHEMERAL, +}; +std::vector active_tests; + +in_port_t v4_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +in_port_t v6_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + struct TestConnection { TestConnection(std::string name, std::string address, in_port_t port) : name(std::move(name)), address(std::move(address)), port(port) {} @@ -39,10 +51,7 @@ struct TestConnection { in_port_t port; }; -struct Finished { - Finished(std::string name) : name(std::move(name)) {} - std::string name; -}; +struct Finished {}; class TestReactor : public test_util::TestBase { public: @@ -61,41 +70,52 @@ class TestReactor : public test_util::TestBase { if ((event.events & IO::CLOSE) != 0) { events.push_back(name + " closed"); - emit(std::make_unique(name)); + emit(std::make_unique()); } } TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { // Bind to IPv4 and a known port - on(KNOWN_V4_PORT).then([this](const TCP::Connection& connection) { - on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { - handle_data("v4 Known", event); - }); - }); - - // Bind to IPv4 an unknown port and get the port number - auto v4 = on().then([this](const TCP::Connection& connection) { - on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { - handle_data("v4 Ephemeral", event); - }); - }); - const in_port_t& v4_port = std::get<1>(v4); - - // Bind to IPv6 and a known port - on(KNOWN_V6_PORT, "::").then([this](const TCP::Connection& connection) { - on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { - handle_data("v6 Known", event); - }); - }); - - // Bind to IPv6 an unknown port and get the port number - auto v6 = on(0, "::").then([this](const TCP::Connection& connection) { - on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { - handle_data("v6 Ephemeral", event); - }); - }); - const in_port_t& v6_port = std::get<1>(v6); + for (const auto& t : active_tests) { + switch (t) { + case V4_KNOWN: { + on(KNOWN_V4_PORT).then([this](const TCP::Connection& connection) { + on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { + handle_data("v4 Known", event); + }); + }); + } break; + case V4_EPHEMERAL: { + // Bind to IPv4 an unknown port and get the port number + auto v4 = on().then([this](const TCP::Connection& connection) { + on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { + handle_data("v4 Ephemeral", event); + }); + }); + v4_port = std::get<1>(v4); + } break; + + // Bind to IPv6 and a known port + case V6_KNOWN: { + on(KNOWN_V6_PORT, "::").then([this](const TCP::Connection& connection) { + on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { + handle_data("v6 Known", event); + }); + }); + } break; + + // Bind to IPv6 an unknown port and get the port number + case V6_EPHEMERAL: { + auto v6 = on(0, "::").then([this](const TCP::Connection& connection) { + on(connection.fd, IO::READ | IO::CLOSE).then([this](IO::Event event) { + handle_data("v6 Ephemeral", event); + }); + }); + v6_port = std::get<1>(v6); + } break; + } + } // Send a test message to the known port on, Sync>().then([](const TestConnection& target) { @@ -125,64 +145,90 @@ class TestReactor : public test_util::TestBase { events.push_back(target.name + " echoed: " + std::string(buff.data(), recv)); }); - on, Sync>().then([=](const Finished& test) { - if (test.name == "Startup") { - emit(std::make_unique("v4 Known", "127.0.0.1", KNOWN_V4_PORT)); - } - if (test.name == "v4 Known") { - emit(std::make_unique("v4 Ephemeral", "127.0.0.1", v4_port)); - } - else if (test.name == "v4 Ephemeral") { - emit(std::make_unique("v6 Known", "::1", KNOWN_V6_PORT)); - } - else if (test.name == "v6 Known") { - emit(std::make_unique("v6 Ephemeral", "::1", v6_port)); + on, Sync>().then([this](const Finished&) { + if (test_no < active_tests.size()) { + switch (active_tests[test_no++]) { + case V4_KNOWN: + emit(std::make_unique("v4 Known", "127.0.0.1", KNOWN_V4_PORT)); + break; + case V4_EPHEMERAL: + emit(std::make_unique("v4 Ephemeral", "127.0.0.1", v4_port)); + break; + case V6_KNOWN: emit(std::make_unique("v6 Known", "::1", KNOWN_V6_PORT)); break; + case V6_EPHEMERAL: emit(std::make_unique("v6 Ephemeral", "::1", v6_port)); break; + default: + events.push_back("Unexpected test"); + powerplant.shutdown(); + break; + } } - else if (test.name == "v6 Ephemeral") { + else { events.push_back("Finishing Test"); powerplant.shutdown(); } }); on().then([this] { - // Start the first test by emitting a "finished" startup - emit(std::make_unique("Startup")); + // Start the first test by emitting a "finished" event + emit(std::make_unique()); }); } private: + size_t test_no = 0; NUClear::util::FileDescriptor known_port_fd; NUClear::util::FileDescriptor ephemeral_port_fd; }; + } // namespace TEST_CASE("Testing listening for TCP connections and receiving data messages", "[api][network][tcp]") { + // First work out what tests will be active + active_tests.push_back(V4_KNOWN); + active_tests.push_back(V4_EPHEMERAL); + if (test_util::has_ipv6()) { + active_tests.push_back(V6_KNOWN); + active_tests.push_back(V6_EPHEMERAL); + } + NUClear::PowerPlant::Configuration config; config.thread_count = 2; NUClear::PowerPlant plant(config); plant.install(); plant.start(); - const std::vector expected = { - "v4 Known sending", - "v4 Known received: v4 Known", - "v4 Known echoed: v4 Known", - "v4 Known closed", - "v4 Ephemeral sending", - "v4 Ephemeral received: v4 Ephemeral", - "v4 Ephemeral echoed: v4 Ephemeral", - "v4 Ephemeral closed", - "v6 Known sending", - "v6 Known received: v6 Known", - "v6 Known echoed: v6 Known", - "v6 Known closed", - "v6 Ephemeral sending", - "v6 Ephemeral received: v6 Ephemeral", - "v6 Ephemeral echoed: v6 Ephemeral", - "v6 Ephemeral closed", - "Finishing Test", - }; + // Get the results for the tests we expect + std::vector expected{}; + for (const auto& t : active_tests) { + switch (t) { + case V4_KNOWN: + expected.push_back("v4 Known sending"); + expected.push_back("v4 Known received: v4 Known"); + expected.push_back("v4 Known echoed: v4 Known"); + expected.push_back("v4 Known closed"); + break; + case V4_EPHEMERAL: + expected.push_back("v4 Ephemeral sending"); + expected.push_back("v4 Ephemeral received: v4 Ephemeral"); + expected.push_back("v4 Ephemeral echoed: v4 Ephemeral"); + expected.push_back("v4 Ephemeral closed"); + break; + case V6_KNOWN: + expected.push_back("v6 Known sending"); + expected.push_back("v6 Known received: v6 Known"); + expected.push_back("v6 Known echoed: v6 Known"); + expected.push_back("v6 Known closed"); + break; + case V6_EPHEMERAL: + expected.push_back("v6 Ephemeral sending"); + expected.push_back("v6 Ephemeral received: v6 Ephemeral"); + expected.push_back("v6 Ephemeral echoed: v6 Ephemeral"); + expected.push_back("v6 Ephemeral closed"); + break; + } + } + expected.push_back("Finishing Test"); // Make an info print the diff in an easy to read way if we fail INFO(test_util::diff_string(expected, events)); From d2c425a84e25457d31dfc9faf552d8d918bff16d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 4 Sep 2023 15:25:23 +1000 Subject: [PATCH 138/176] Allow the UDP test to dynamically test for IPv6 if available --- tests/dsl/TCP.cpp | 2 +- tests/dsl/UDP.cpp | 514 +++++++++++++++++++++++++++------------------- 2 files changed, 305 insertions(+), 211 deletions(-) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 26d5956c8..9a0b1b70b 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -38,7 +38,7 @@ enum TestType { V6_KNOWN, V6_EPHEMERAL, }; -std::vector active_tests; +std::vector active_tests; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) in_port_t v4_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) in_port_t v6_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index eb97d6414..aa3b4a44a 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -20,6 +20,7 @@ #include #include "test_util/TestBase.hpp" +#include "test_util/has_ipv6.hpp" namespace { @@ -44,15 +45,19 @@ in_port_t broad_v4_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global in_port_t multi_v4_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) in_port_t multi_v6_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -inline bool has_ipv6() { - // Get the first IPv6 address we can find - for (const auto& iface : NUClear::util::network::get_interfaces()) { - if (iface.ip.sock.sa_family == AF_INET6) { - return true; - } - } - return false; -} +enum TestType { + UNICAST_V4_KNOWN, + UNICAST_V4_EPHEMERAL, + UNICAST_V6_KNOWN, + UNICAST_V6_EPHEMERAL, + BROADCAST_V4_KNOWN, + BROADCAST_V4_EPHEMERAL, + MULTICAST_V4_KNOWN, + MULTICAST_V4_EPHEMERAL, + MULTICAST_V6_KNOWN, + MULTICAST_V6_EPHEMERAL, +}; +std::vector active_tests; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) inline std::string get_broadcast_addr() { static std::string addr{}; @@ -87,31 +92,54 @@ struct SendTarget { Target to{}; Target from{}; }; -std::vector send_targets(const std::string& type, in_port_t port, bool include_target = false) { +std::vector send_targets(const std::string& type, in_port_t port) { std::vector results; + // Loop through the active tests and add the send targets // Make sure that the type we are actually after is sent last - const std::string t = type.substr(0, 3); - if (type[2] == '4' && include_target == (t == "Uv4")) { - results.push_back(SendTarget{type + ":Uv4", {"127.0.0.1", port}, {}}); - } - if (type[2] == '4' && include_target == (t == "Bv4")) { - results.push_back(SendTarget{type + ":Bv4", {get_broadcast_addr(), port}, {}}); - } - if (type[2] == '4' && include_target == (t == "Mv4")) { - results.push_back(SendTarget{type + ":Mv4", {IPV4_MULTICAST_ADDRESS, port}, {}}); - } - if (type[2] == '6' && include_target == (t == "Uv6")) { - results.push_back(SendTarget{type + ":Uv6", {"::1", port}, {}}); - } - if (type[2] == '6' && include_target == (t == "Mv6")) { - // For multicast v6 send from localhost so that it works on OSX - results.push_back(SendTarget{type + ":Mv6", {IPV6_MULTICAST_ADDRESS, port}, {"::1", 0}}); - } - if (!include_target) { - auto target = send_targets(type, port, true); - results.insert(results.end(), target.begin(), target.end()); + for (const auto& t : active_tests) { + switch (t) { + case UNICAST_V4_KNOWN: + case UNICAST_V4_EPHEMERAL: { + results.push_back(SendTarget{type + ":Uv4", {"127.0.0.1", port}, {}}); + } break; + case UNICAST_V6_KNOWN: + case UNICAST_V6_EPHEMERAL: { + results.push_back(SendTarget{type + ":Uv6", {"::1", port}, {}}); + } break; + case BROADCAST_V4_KNOWN: + case BROADCAST_V4_EPHEMERAL: { + results.push_back(SendTarget{type + ":Bv4", {get_broadcast_addr(), port}, {}}); + } break; + case MULTICAST_V4_KNOWN: + case MULTICAST_V4_EPHEMERAL: { + results.push_back(SendTarget{type + ":Mv4", {IPV4_MULTICAST_ADDRESS, port}, {}}); + } break; + case MULTICAST_V6_KNOWN: + case MULTICAST_V6_EPHEMERAL: { + results.push_back(SendTarget{type + ":Mv6", {IPV6_MULTICAST_ADDRESS, port}, {"::1", 0}}); + } break; + } } + + // remove duplicates + results.erase(std::unique(results.begin(), + results.end(), + [](const SendTarget& a, const SendTarget& b) { + return a.to.address == b.to.address && a.to.port == b.to.port && a.data == b.data + && a.from.address == b.from.address && a.from.port == b.from.port; + }), + results.end()); + + // Stable sort so that the type we are after is last + std::stable_sort(results.begin(), results.end(), [](const SendTarget& /*a*/, const SendTarget& b) { + // We want to sort such that the one we are after is last and everything else is unmodified + // That means that every comparision except one should be false + // This is because equality is implied if a < b == false and b < a == false + // The only time we should return true is when b is our target (which would make a less than it) + return b.data.substr(0, 3) == b.data.substr(5, 8); + }); + return results; } @@ -146,68 +174,91 @@ class TestReactor : public test_util::TestBase { public: TestReactor(std::unique_ptr environment) : TestBase(std::move(environment), false) { - // IPv4 Unicast Known port - on(UNICAST_V4).then([this](const UDP::Packet& packet) { // - handle_data("Uv4K", packet); - }); - - // IPv4 Unicast Ephemeral port - auto uni_v4 = on().then([this](const UDP::Packet& packet) { // - handle_data("Uv4E", packet); - }); - uni_v4_port = std::get<1>(uni_v4); - - // IPv6 Unicast Known port - on(UNICAST_V6, "::").then([this](const UDP::Packet& packet) { // - handle_data("Uv6K", packet); - }); - - // IPv6 Unicast Ephemeral port - auto uni_v6 = on(0, "::").then([this](const UDP::Packet& packet) { // - handle_data("Uv6E", packet); - }); - uni_v6_port = std::get<1>(uni_v6); - - // IPv4 Broadcast Known port - on(BROADCAST_V4).then([this](const UDP::Packet& packet) { // - handle_data("Bv4K", packet); - }); - - // IPv4 Broadcast Ephemeral port - auto broad_v4 = on().then([this](const UDP::Packet& packet) { // - handle_data("Bv4E", packet); - }); - broad_v4_port = std::get<1>(broad_v4); - - // No such thing as broadcast in IPv6 - - // IPv4 Multicast Known port - on(IPV4_MULTICAST_ADDRESS, MULTICAST_V4).then([this](const UDP::Packet& packet) { - handle_data("Mv4K", packet); - }); - - // IPv4 Multicast Ephemeral port - auto multi_v4 = on(IPV4_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { - handle_data("Mv4E", packet); - }); - multi_v4_port = std::get<1>(multi_v4); - - // For the IPv6 test we need to bind to the IPv6 localhost address and send from it when using udp emit - // This is because on OSX by default there is no default route for IPv6 multicast packets (see `netstat -nr`) - // As a result if you don't specify an interface to use when sending and receiving IPv6 multicast packets - // the send/bind fails which makes the tests not work. Linux does not care about this extra step so it doesn't - // break the tests - - // IPv6 Multicast Known port - on(IPV6_MULTICAST_ADDRESS, MULTICAST_V6, "::1").then([this](const UDP::Packet& packet) { - handle_data("Mv6K", packet); - }); - - // IPv6 Multicast Ephemeral port - auto multi_v6 = on(IPV6_MULTICAST_ADDRESS, 0, "::1").then([this](const UDP::Packet& packet) { - handle_data("Mv6E", packet); - }); - multi_v6_port = std::get<1>(multi_v6); + for (const auto& t : active_tests) { + switch (t) { + case UNICAST_V4_KNOWN: { + on(UNICAST_V4).then([this](const UDP::Packet& packet) { // + handle_data("Uv4K", packet); + }); + } break; + + // IPv4 Unicast Ephemeral port + case UNICAST_V4_EPHEMERAL: { + auto uni_v4 = on().then([this](const UDP::Packet& packet) { // + handle_data("Uv4E", packet); + }); + uni_v4_port = std::get<1>(uni_v4); + } break; + + // IPv6 Unicast Known port + case UNICAST_V6_KNOWN: { + on(UNICAST_V6, "::").then([this](const UDP::Packet& packet) { // + handle_data("Uv6K", packet); + }); + } break; + + // IPv6 Unicast Ephemeral port + case UNICAST_V6_EPHEMERAL: { + auto uni_v6 = on(0, "::").then([this](const UDP::Packet& packet) { // + handle_data("Uv6E", packet); + }); + uni_v6_port = std::get<1>(uni_v6); + } break; + + // IPv4 Broadcast Known port + case BROADCAST_V4_KNOWN: { + on(BROADCAST_V4).then([this](const UDP::Packet& packet) { // + handle_data("Bv4K", packet); + }); + } break; + + // IPv4 Broadcast Ephemeral port + case BROADCAST_V4_EPHEMERAL: { + auto broad_v4 = on().then([this](const UDP::Packet& packet) { // + handle_data("Bv4E", packet); + }); + broad_v4_port = std::get<1>(broad_v4); + } break; + + // No such thing as broadcast in IPv6 + + // IPv4 Multicast Known port + case MULTICAST_V4_KNOWN: { + on(IPV4_MULTICAST_ADDRESS, MULTICAST_V4).then([this](const UDP::Packet& packet) { + handle_data("Mv4K", packet); + }); + } break; + + // IPv4 Multicast Ephemeral port + case MULTICAST_V4_EPHEMERAL: { + auto multi_v4 = on(IPV4_MULTICAST_ADDRESS).then([this](const UDP::Packet& packet) { + handle_data("Mv4E", packet); + }); + multi_v4_port = std::get<1>(multi_v4); + } break; + + // For the IPv6 test we need to bind to the IPv6 localhost address and send from it when using udp + // emit This is because on OSX by default there is no default route for IPv6 multicast packets (see + // `netstat -nr`) As a result if you don't specify an interface to use when sending and receiving + // IPv6 multicast packets the send/bind fails which makes the tests not work. Linux does not care + // about this extra step so it doesn't break the tests + + // IPv6 Multicast Known port + case MULTICAST_V6_KNOWN: { + on(IPV6_MULTICAST_ADDRESS, MULTICAST_V6, "::1") + .then([this](const UDP::Packet& packet) { handle_data("Mv6K", packet); }); + } break; + + // IPv6 Multicast Ephemeral port + case MULTICAST_V6_EPHEMERAL: { + auto multi_v6 = + on(IPV6_MULTICAST_ADDRESS, 0, "::1").then([this](const UDP::Packet& packet) { + handle_data("Mv6E", packet); + }); + multi_v6_port = std::get<1>(multi_v6); + } break; + } + } // Send a test message to the known port on>().then([this](const TestUDP& target) { @@ -215,7 +266,7 @@ class TestReactor : public test_util::TestBase { emit(std::make_unique(target.name), target.address, target.port); }); - on>().then([=](const Finished& test) { + on>().then([this](const Finished&) { auto send_all = [this](const std::string& type, const in_port_t& port) { for (const auto& t : send_targets(type, port)) { events.push_back(" -> " + t.to.address + ":" + std::to_string(t.to.port)); @@ -227,61 +278,66 @@ class TestReactor : public test_util::TestBase { } }; - if (test.name == "Startup") { - events.push_back("- Known Unicast V4 Test -"); - send_all("Uv4K", UNICAST_V4); - } - else if (test.name == "Uv4K") { - events.push_back(""); - events.push_back("- Ephemeral Unicast V4 Test -"); - send_all("Uv4E", uni_v4_port); - } - else if (test.name == "Uv4E") { - events.push_back(""); - events.push_back("- Known Unicast V6 Test -"); - send_all("Uv6K", UNICAST_V6); - } - else if (test.name == "Uv6K") { - events.push_back(""); - events.push_back("- Ephemeral Unicast V6 Test -"); - send_all("Uv6E", uni_v6_port); - } - else if (test.name == "Uv6E") { - events.push_back(""); - events.push_back("- Known Broadcast V4 Test -"); - send_all("Bv4K", BROADCAST_V4); - } - else if (test.name == "Bv4K") { - events.push_back(""); - events.push_back("- Ephemeral Broadcast V4 Test -"); - send_all("Bv4E", broad_v4_port); - } - else if (test.name == "Bv4E") { - events.push_back(""); - events.push_back("- Known Multicast V4 Test -"); - send_all("Mv4K", MULTICAST_V4); - } - else if (test.name == "Mv4K") { - events.push_back(""); - events.push_back("- Ephemeral Multicast V4 Test -"); - send_all("Mv4E", multi_v4_port); - } - else if (test.name == "Mv4E") { - events.push_back(""); - events.push_back("- Known Multicast V6 Test -"); - send_all("Mv6K", MULTICAST_V6); - } - else if (test.name == "Mv6K") { - events.push_back(""); - events.push_back("- Ephemeral Multicast V6 Test -"); - send_all("Mv6E", multi_v6_port); - } - else if (test.name == "Mv6E") { - // We are done, so stop the reactor - powerplant.shutdown(); + if (test_no < active_tests.size()) { + switch (active_tests[test_no++]) { + case UNICAST_V4_KNOWN: { + events.push_back("- Known Unicast V4 Test -"); + send_all("Uv4K", UNICAST_V4); + } break; + + case UNICAST_V4_EPHEMERAL: { + events.push_back("- Ephemeral Unicast V4 Test -"); + send_all("Uv4E", uni_v4_port); + } break; + + case UNICAST_V6_KNOWN: { + events.push_back("- Known Unicast V6 Test -"); + send_all("Uv6K", UNICAST_V6); + } break; + + case UNICAST_V6_EPHEMERAL: { + events.push_back("- Ephemeral Unicast V6 Test -"); + send_all("Uv6E", uni_v6_port); + } break; + + case BROADCAST_V4_KNOWN: { + events.push_back("- Known Broadcast V4 Test -"); + send_all("Bv4K", BROADCAST_V4); + } break; + + case BROADCAST_V4_EPHEMERAL: { + events.push_back("- Ephemeral Broadcast V4 Test -"); + send_all("Bv4E", broad_v4_port); + } break; + + case MULTICAST_V4_KNOWN: { + events.push_back("- Known Multicast V4 Test -"); + send_all("Mv4K", MULTICAST_V4); + } break; + + case MULTICAST_V4_EPHEMERAL: { + events.push_back("- Ephemeral Multicast V4 Test -"); + send_all("Mv4E", multi_v4_port); + } break; + + case MULTICAST_V6_KNOWN: { + events.push_back("- Known Multicast V6 Test -"); + send_all("Mv6K", MULTICAST_V6); + } break; + + case MULTICAST_V6_EPHEMERAL: { + events.push_back("- Ephemeral Multicast V6 Test -"); + send_all("Mv6E", multi_v6_port); + } break; + + default: { + events.push_back("Unknown test type"); + powerplant.shutdown(); + } break; + } } else { - FAIL("Unknown test name"); + powerplant.shutdown(); } }); @@ -290,13 +346,30 @@ class TestReactor : public test_util::TestBase { emit(std::make_unique("Startup")); }); } + +private: + size_t test_no = 0; }; } // namespace TEST_CASE("Testing sending and receiving of UDP messages", "[api][network][udp]") { - REQUIRE(has_ipv6()); + // Build up the list of active tests based on what we have available + active_tests.push_back(UNICAST_V4_KNOWN); + active_tests.push_back(UNICAST_V4_EPHEMERAL); + if (test_util::has_ipv6()) { + active_tests.push_back(UNICAST_V6_KNOWN); + active_tests.push_back(UNICAST_V6_EPHEMERAL); + } + active_tests.push_back(BROADCAST_V4_KNOWN); + active_tests.push_back(BROADCAST_V4_EPHEMERAL); + active_tests.push_back(MULTICAST_V4_KNOWN); + active_tests.push_back(MULTICAST_V4_EPHEMERAL); + if (test_util::has_ipv6()) { + active_tests.push_back(MULTICAST_V6_KNOWN); + active_tests.push_back(MULTICAST_V6_EPHEMERAL); + } NUClear::PowerPlant::Configuration config; config.thread_count = 1; @@ -305,74 +378,95 @@ TEST_CASE("Testing sending and receiving of UDP messages", "[api][network][udp]" plant.start(); std::vector expected; - expected.push_back("- Known Unicast V4 Test -"); - for (const auto& line : send_targets("Uv4K", UNICAST_V4)) { - expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); - } - expected.push_back("Uv4K <- Uv4K:Uv4 (127.0.0.1:" + std::to_string(UNICAST_V4) + ")"); - expected.push_back(""); - - expected.push_back("- Ephemeral Unicast V4 Test -"); - for (const auto& line : send_targets("Uv4E", uni_v4_port)) { - expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); - } - expected.push_back("Uv4E <- Uv4E:Uv4 (127.0.0.1:" + std::to_string(uni_v4_port) + ")"); - expected.push_back(""); - - expected.push_back("- Known Unicast V6 Test -"); - for (const auto& line : send_targets("Uv6K", UNICAST_V6)) { - expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); - } - expected.push_back("Uv6K <- Uv6K:Uv6 (::1:" + std::to_string(UNICAST_V6) + ")"); - expected.push_back(""); - - expected.push_back("- Ephemeral Unicast V6 Test -"); - for (const auto& line : send_targets("Uv6E", uni_v6_port)) { - expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); - } - expected.push_back("Uv6E <- Uv6E:Uv6 (::1:" + std::to_string(uni_v6_port) + ")"); - expected.push_back(""); - - expected.push_back("- Known Broadcast V4 Test -"); - for (const auto& line : send_targets("Bv4K", BROADCAST_V4)) { - expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); - } - expected.push_back("Bv4K <- Bv4K:Bv4 (" + get_broadcast_addr() + ":" + std::to_string(BROADCAST_V4) + ")"); - expected.push_back(""); - - expected.push_back("- Ephemeral Broadcast V4 Test -"); - for (const auto& line : send_targets("Bv4E", broad_v4_port)) { - expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); - } - expected.push_back("Bv4E <- Bv4E:Bv4 (" + get_broadcast_addr() + ":" + std::to_string(broad_v4_port) + ")"); - expected.push_back(""); + for (const auto& t : active_tests) { + switch (t) { + case UNICAST_V4_KNOWN: { + expected.push_back("- Known Unicast V4 Test -"); + for (const auto& line : send_targets("Uv4K", UNICAST_V4)) { + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + } + expected.push_back("Uv4K <- Uv4K:Uv4 (127.0.0.1:" + std::to_string(UNICAST_V4) + ")"); + } break; - expected.push_back("- Known Multicast V4 Test -"); - for (const auto& line : send_targets("Mv4K", MULTICAST_V4)) { - expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); - } - expected.push_back("Mv4K <- Mv4K:Mv4 (" + IPV4_MULTICAST_ADDRESS + ":" + std::to_string(MULTICAST_V4) + ")"); - expected.push_back(""); + case UNICAST_V4_EPHEMERAL: { + expected.push_back("- Ephemeral Unicast V4 Test -"); + for (const auto& line : send_targets("Uv4E", uni_v4_port)) { + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + } + expected.push_back("Uv4E <- Uv4E:Uv4 (127.0.0.1:" + std::to_string(uni_v4_port) + ")"); + } break; - expected.push_back("- Ephemeral Multicast V4 Test -"); - for (const auto& line : send_targets("Mv4E", multi_v4_port)) { - expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); - } - expected.push_back("Mv4E <- Mv4E:Mv4 (" + IPV4_MULTICAST_ADDRESS + ":" + std::to_string(multi_v4_port) + ")"); - expected.push_back(""); + case UNICAST_V6_KNOWN: { + expected.push_back("- Known Unicast V6 Test -"); + for (const auto& line : send_targets("Uv6K", UNICAST_V6)) { + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + } + expected.push_back("Uv6K <- Uv6K:Uv6 (::1:" + std::to_string(UNICAST_V6) + ")"); + } break; - expected.push_back("- Known Multicast V6 Test -"); - for (const auto& line : send_targets("Mv6K", MULTICAST_V6)) { - expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); - } - expected.push_back("Mv6K <- Mv6K:Mv6 (" + IPV6_MULTICAST_ADDRESS + ":" + std::to_string(MULTICAST_V6) + ")"); - expected.push_back(""); + case UNICAST_V6_EPHEMERAL: { + expected.push_back("- Ephemeral Unicast V6 Test -"); + for (const auto& line : send_targets("Uv6E", uni_v6_port)) { + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + } + expected.push_back("Uv6E <- Uv6E:Uv6 (::1:" + std::to_string(uni_v6_port) + ")"); + } break; - expected.push_back("- Ephemeral Multicast V6 Test -"); - for (const auto& line : send_targets("Mv6E", multi_v6_port)) { - expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + case BROADCAST_V4_KNOWN: { + expected.push_back("- Known Broadcast V4 Test -"); + for (const auto& line : send_targets("Bv4K", BROADCAST_V4)) { + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + } + expected.push_back("Bv4K <- Bv4K:Bv4 (" + get_broadcast_addr() + ":" + std::to_string(BROADCAST_V4) + + ")"); + } break; + + case BROADCAST_V4_EPHEMERAL: { + expected.push_back("- Ephemeral Broadcast V4 Test -"); + for (const auto& line : send_targets("Bv4E", broad_v4_port)) { + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + } + expected.push_back("Bv4E <- Bv4E:Bv4 (" + get_broadcast_addr() + ":" + std::to_string(broad_v4_port) + + ")"); + } break; + + case MULTICAST_V4_KNOWN: { + expected.push_back("- Known Multicast V4 Test -"); + for (const auto& line : send_targets("Mv4K", MULTICAST_V4)) { + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + } + expected.push_back("Mv4K <- Mv4K:Mv4 (" + IPV4_MULTICAST_ADDRESS + ":" + std::to_string(MULTICAST_V4) + + ")"); + } break; + + case MULTICAST_V4_EPHEMERAL: { + expected.push_back("- Ephemeral Multicast V4 Test -"); + for (const auto& line : send_targets("Mv4E", multi_v4_port)) { + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + } + expected.push_back("Mv4E <- Mv4E:Mv4 (" + IPV4_MULTICAST_ADDRESS + ":" + std::to_string(multi_v4_port) + + ")"); + } break; + + case MULTICAST_V6_KNOWN: { + expected.push_back("- Known Multicast V6 Test -"); + for (const auto& line : send_targets("Mv6K", MULTICAST_V6)) { + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + } + expected.push_back("Mv6K <- Mv6K:Mv6 (" + IPV6_MULTICAST_ADDRESS + ":" + std::to_string(MULTICAST_V6) + + ")"); + } break; + + case MULTICAST_V6_EPHEMERAL: { + expected.push_back("- Ephemeral Multicast V6 Test -"); + for (const auto& line : send_targets("Mv6E", multi_v6_port)) { + expected.push_back(" -> " + line.to.address + ":" + std::to_string(line.to.port)); + } + expected.push_back("Mv6E <- Mv6E:Mv6 (" + IPV6_MULTICAST_ADDRESS + ":" + std::to_string(multi_v6_port) + + ")"); + } break; + } } - expected.push_back("Mv6E <- Mv6E:Mv6 (" + IPV6_MULTICAST_ADDRESS + ":" + std::to_string(multi_v6_port) + ")"); // Make an info print the diff in an easy to read way if we fail INFO(test_util::diff_string(expected, events)); From 5b5185b763e644eeb3eaa5bd980b1f51772fe515 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 4 Sep 2023 16:07:10 +1000 Subject: [PATCH 139/176] Print out on success --- .github/workflows/main.yaml | 42 ++++++++++++------------------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c1d21cc35..122871d26 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -24,7 +24,8 @@ jobs: strategy: matrix: - container: ['gcc:5', 'gcc:7', 'gcc:9', 'gcc:10', 'gcc:11', 'gcc:12', 'gcc:13'] + container: + ["gcc:5", "gcc:7", "gcc:9", "gcc:10", "gcc:11", "gcc:12", "gcc:13"] # The type of runner that the job will run on runs-on: ubuntu-latest @@ -33,9 +34,7 @@ jobs: # Use the container for this specific version of gcc container: ${{ matrix.container }} - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Checkout Code uses: actions/checkout@v2 @@ -58,18 +57,14 @@ jobs: # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | - build/tests/test_nuclear - for f in build/tests/individual/*; do echo "Testing $f"; ./$f; done + build/tests/test_nuclear -s + for f in build/tests/individual/*; do echo "Testing $f"; ./$f -s; done build-osx: name: MacOS Clang - - # The type of runner that the job will run on runs-on: macos-latest - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Checkout Code uses: actions/checkout@v2 @@ -88,8 +83,8 @@ jobs: # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | - build/tests/test_nuclear - for f in build/tests/individual/*; do echo "Testing $f"; ./$f; done + build/tests/test_nuclear -s + for f in build/tests/individual/*; do echo "Testing $f"; ./$f -s; done build-windows: name: Windows MSVC @@ -97,9 +92,7 @@ jobs: # The type of runner that the job will run on runs-on: windows-latest - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Checkout Code uses: actions/checkout@v2 @@ -118,34 +111,29 @@ jobs: # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | - build/tests/Release/test_nuclear.exe - for f in build/tests/individual/Release/*; do echo "Testing $f"; ./$f; done + build/tests/Release/test_nuclear.exe -s + for f in build/tests/individual/Release/*; do echo "Testing $f"; ./$f -s; done shell: bash check-clang-tidy-linux: name: Clang-Tidy Linux - - # The type of runner that the job will run on runs-on: ubuntu-latest - # Steps represent a sequence of tasks that will be executed as part of the job steps: - name: Install clang-tidy-15 run: | - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" | sudo tee /etc/apt/sources.list.d/llvm-15 - echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" | sudo tee -a /etc/apt/sources.list.d/llvm-15 - sudo apt-get update - sudo apt-get install -y clang-tidy-15 + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" | sudo tee /etc/apt/sources.list.d/llvm-15 + echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" | sudo tee -a /etc/apt/sources.list.d/llvm-15 + sudo apt-get update + sudo apt-get install -y clang-tidy-15 - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Checkout Code uses: actions/checkout@v2 # Download and install cmake - name: Install CMake v3.19.1 uses: lukka/get-cmake@v3.19.1 - - name: Configure CMake run: | cmake -E make_directory build @@ -158,13 +146,9 @@ jobs: check-clang-tidy-msvc: name: Clang-Tidy MSVC - - # The type of runner that the job will run on runs-on: windows-latest - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Checkout Code uses: actions/checkout@v2 From fc6262fac6c24151ab47b9fc677899e39bc6e8c9 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Mon, 4 Sep 2023 16:16:23 +1000 Subject: [PATCH 140/176] clang-tidy --- tests/dsl/UDP.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index aa3b4a44a..3fa7039aa 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -86,8 +86,8 @@ inline std::string get_broadcast_addr() { struct SendTarget { std::string data{}; struct Target { - std::string address = ""; - in_port_t port = 0; + std::string address{}; + in_port_t port = 0; }; Target to{}; Target from{}; From c717de9c231aab33018c965164509c2d26a09758 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 14:03:17 +1000 Subject: [PATCH 141/176] Fix return before close --- src/extension/IOController_Windows.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index dd8cca5e9..5763c8cbb 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -173,12 +173,14 @@ namespace extension { WSAEVENT event = it->first; // Remove the task - return tasks.erase(it); + auto new_it = tasks.erase(it); // Try to close the WSA event if (!WSACloseEvent(event)) { throw std::system_error(WSAGetLastError(), std::system_category(), "WSACloseEvent() failed"); } + + return new_it; } From 5674920fadb71675756e5d847d8902bacb08c6fb Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 14:17:32 +1000 Subject: [PATCH 142/176] Some cmake fixes --- CMakeLists.txt | 21 +++++++++++++-------- tests/CMakeLists.txt | 6 ------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a57b4372f..c24d9a60b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,9 +13,15 @@ # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - cmake_minimum_required(VERSION 3.15.0) +# Set the project after the build type as the Project command can change the build type +project( + NUClear + VERSION 1.0.0 + LANGUAGES C CXX +) + # We use additional modules that cmake needs to know about set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") @@ -35,19 +41,18 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") endif() -# Set the project after the build type as the Project command can change the build type -project( - NUClear - VERSION 1.0.0 - LANGUAGES C CXX -) - # Determine if NUClear is built as a subproject (using add_subdirectory) or if it is the master project. set(MASTER_PROJECT OFF) if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) set(MASTER_PROJECT ON) endif() +if(MSVC) + add_compile_options(/W4) +else() + add_compile_options(-Wall -Wextra -pedantic) +endif(MSVC) + # If this option is set we are building using continous integration option(CI_BUILD "Enable build options for building in the CI server" OFF) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6b3efd92d..78634ebde 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -23,12 +23,6 @@ if(CATCH_FOUND) if(BUILD_TESTS) enable_testing() - if(MSVC) - add_compile_options(/W4 /WX) - else() - add_compile_options(-Wall -Wextra -pedantic -Werror) - endif(MSVC) - add_compile_definitions(CATCH_CONFIG_CONSOLE_WIDTH=120) file(GLOB test_src test.cpp "api/*.cpp" "dsl/*.cpp" "dsl/emit/*.cpp" "log/*.cpp" "util/network/*.cpp" "test_util/*.cpp") From 6adbd4d02a908b03c767a424e531471298c40609 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 14:29:53 +1000 Subject: [PATCH 143/176] Make the sync order test check the sequence --- tests/dsl/SyncOrder.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/dsl/SyncOrder.cpp b/tests/dsl/SyncOrder.cpp index 15a457101..9e9b2424d 100644 --- a/tests/dsl/SyncOrder.cpp +++ b/tests/dsl/SyncOrder.cpp @@ -35,14 +35,14 @@ struct Message { struct ShutdownOnIdle {}; /// @brief Events that occur during the test -std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +std::vector events; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) class TestReactor : public test_util::TestBase { public: TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { on, Sync>().then([](const Message& m) { // - events.push_back("Received value " + std::to_string(m.val)); + events.push_back(m.val); }); on().then("Startup", [this] { @@ -62,8 +62,10 @@ TEST_CASE("Sync events execute in order", "[api][sync][priority]") { plant.install(); plant.start(); + REQUIRE(events.size() == N_EVENTS); - for (int i = 0; i < N_EVENTS; ++i) { - CHECK(events[i] == "Received value " + std::to_string(i)); - } + + std::vector expected(events.size()); + std::iota(expected.begin(), expected.end(), 0); + REQUIRE(events == expected); } From deb553b3a154f14d0c99eedc277e8096956dc753 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 14:31:46 +1000 Subject: [PATCH 144/176] Add numeric header --- tests/dsl/SyncOrder.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/dsl/SyncOrder.cpp b/tests/dsl/SyncOrder.cpp index 9e9b2424d..7cd3a3fae 100644 --- a/tests/dsl/SyncOrder.cpp +++ b/tests/dsl/SyncOrder.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include From aa6060b209bc518cdd5da7db115e8efa0d46236c Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 14:34:41 +1000 Subject: [PATCH 145/176] Just add a check on timeout so the rest of the checks print still --- tests/test_util/TestBase.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_util/TestBase.hpp b/tests/test_util/TestBase.hpp index e2dfd1449..d04c24cf2 100644 --- a/tests/test_util/TestBase.hpp +++ b/tests/test_util/TestBase.hpp @@ -58,8 +58,9 @@ class TestBase : public NUClear::Reactor { // Timeout if the test doesn't complete in time on, MainThread>().then([this] { + INFO("Test timed out"); + CHECK(false); powerplant.shutdown(); - FAIL("Test timed out"); }); } }; From e2cd44e034c755df30048c8808b1e1d816519616 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 16:00:41 +1000 Subject: [PATCH 146/176] Move WSA to a static holder so I don't have to think about it --- src/extension/IOController_Windows.hpp | 14 -------------- src/util/platform.cpp | 23 +++++++++++++++++++++++ src/util/platform.hpp | 13 +++++++++++++ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index 5763c8cbb..56d8299a2 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -187,15 +187,6 @@ namespace extension { public: explicit IOController(std::unique_ptr environment) : Reactor(std::move(environment)) { - // Startup WSA for IO - WORD version = MAKEWORD(2, 2); - WSADATA wsa_data; - - int startup_status = WSAStartup(version, &wsa_data); - if (startup_status != 0) { - throw std::system_error(startup_status, std::system_category(), "WSAStartup() failed"); - } - // Create an event to use for the notifier (used for getting out of WSAWaitForMultipleEvents()) notifier = WSACreateEvent(); if (notifier == WSA_INVALID_EVENT) { @@ -306,11 +297,6 @@ namespace extension { }); } - // We need a destructor to cleanup WSA stuff - virtual ~IOController() { - WSACleanup(); - } - private: /// @brief The event that is used to wake up the WaitForMultipleEvents call WSAEVENT notifier; diff --git a/src/util/platform.cpp b/src/util/platform.cpp index 4f97dba42..bfc6a5753 100644 --- a/src/util/platform.cpp +++ b/src/util/platform.cpp @@ -70,6 +70,29 @@ int sendmsg(fd_t fd, msghdr* msg, int flags) { return v == 0 ? sent : v; } + +WSAHolder::WSAHolder() { + WSADATA wsa_data{}; + WORD version = MAKEWORD(2, 2); + int startup_status = WSAStartup(MAKEWORD(2, 2), &wsaData); + + + WORD version = MAKEWORD(2, 2); + WSADATA wsa_data; + + int startup_status = WSAStartup(version, &wsa_data); + if (startup_status != 0) { + throw std::system_error(startup_status, std::system_category(), "WSAStartup() failed"); + } + WSACleanup(); +} + +WSAHolder::~WSAHolder() { + WSACleanup(); +} + +WSAHolder::WSAHolder instance{}; + } // namespace NUClear #endif diff --git a/src/util/platform.hpp b/src/util/platform.hpp index 191af7bfb..cb177cc82 100644 --- a/src/util/platform.hpp +++ b/src/util/platform.hpp @@ -153,6 +153,19 @@ using socklen_t = int; int recvmsg(fd_t fd, msghdr* msg, int flags); int sendmsg(fd_t fd, msghdr* msg, int flags); + +/** + * @brief This struct is used to setup WSA on windows in a single place so we don't have to worry about it + * + * A single instance of this struct will be created statically at startup which will ensure that WSA is setup for the + * lifetime of the program and torn down as it exists. + */ +struct WSAHolder { + static WSAHolder instance; + WSAHolder(); + ~WSAHolder(); +}; + } // namespace NUClear #else From 64de873fa2893c6927065a88e3204434a2973344 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 16:01:40 +1000 Subject: [PATCH 147/176] Make TCP timeout on error --- tests/dsl/TCP.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 9a0b1b70b..4084fe3e0 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -130,6 +130,16 @@ class TestReactor : public test_util::TestBase { throw std::runtime_error("Failed to create socket"); } + // Set a timeout so we don't hang forever if something goes wrong +#ifdef _WIN32 + DWORD timeout = 100; +#else + struct timeval timeout; + timeout.tv_sec = 0; + timeout.tv_usec = 100000; +#endif + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast(&timeout), sizeof(timeout)); + // Connect to ourself if (::connect(fd, &address.sock, address.size()) != 0) { throw std::runtime_error("Failed to connect to socket"); From b550ac50e5416da3b52ffbd72303f5d2525a2cd2 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 16:01:51 +1000 Subject: [PATCH 148/176] . --- tests/dsl/TCP.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 4084fe3e0..483e98065 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -138,7 +138,7 @@ class TestReactor : public test_util::TestBase { timeout.tv_sec = 0; timeout.tv_usec = 100000; #endif - setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast(&timeout), sizeof(timeout)); + ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast(&timeout), sizeof(timeout)); // Connect to ourself if (::connect(fd, &address.sock, address.size()) != 0) { From 580c8f30c2f71029b31d308e0f2933272251cc38 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 16:02:36 +1000 Subject: [PATCH 149/176] Shutdown before catch does stuff --- tests/test_util/TestBase.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_util/TestBase.hpp b/tests/test_util/TestBase.hpp index d04c24cf2..00630e65e 100644 --- a/tests/test_util/TestBase.hpp +++ b/tests/test_util/TestBase.hpp @@ -58,9 +58,9 @@ class TestBase : public NUClear::Reactor { // Timeout if the test doesn't complete in time on, MainThread>().then([this] { + powerplant.shutdown(); INFO("Test timed out"); CHECK(false); - powerplant.shutdown(); }); } }; From 5b3d043f46182dac4c36d8e826dd2c00d24b9545 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 16:10:43 +1000 Subject: [PATCH 150/176] ... --- src/util/platform.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/util/platform.cpp b/src/util/platform.cpp index bfc6a5753..a8982f6d9 100644 --- a/src/util/platform.cpp +++ b/src/util/platform.cpp @@ -72,11 +72,6 @@ int sendmsg(fd_t fd, msghdr* msg, int flags) { } WSAHolder::WSAHolder() { - WSADATA wsa_data{}; - WORD version = MAKEWORD(2, 2); - int startup_status = WSAStartup(MAKEWORD(2, 2), &wsaData); - - WORD version = MAKEWORD(2, 2); WSADATA wsa_data; @@ -84,7 +79,6 @@ WSAHolder::WSAHolder() { if (startup_status != 0) { throw std::system_error(startup_status, std::system_category(), "WSAStartup() failed"); } - WSACleanup(); } WSAHolder::~WSAHolder() { From 69d55ed661a92c323baa205ae30f16db5b6a0e38 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 16:13:14 +1000 Subject: [PATCH 151/176] . --- src/util/platform.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util/platform.cpp b/src/util/platform.cpp index a8982f6d9..c8a970afd 100644 --- a/src/util/platform.cpp +++ b/src/util/platform.cpp @@ -21,6 +21,7 @@ #ifdef _WIN32 #include + #include LPFN_WSARECVMSG WSARecvMsg = nullptr; // Go get that WSARecvMsg function from stupid land From 22a36407a2c69a8ef25314dab3730109d3aba1da Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Tue, 5 Sep 2023 16:28:01 +1000 Subject: [PATCH 152/176] Fix instance --- src/util/platform.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/platform.cpp b/src/util/platform.cpp index c8a970afd..e05b69c16 100644 --- a/src/util/platform.cpp +++ b/src/util/platform.cpp @@ -86,7 +86,7 @@ WSAHolder::~WSAHolder() { WSACleanup(); } -WSAHolder::WSAHolder instance{}; +WSAHolder WSAHolder::instance{}; } // namespace NUClear From 19f41f89e4a8d9d94439b48e30e005d5604c5f89 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 6 Sep 2023 10:10:43 +1000 Subject: [PATCH 153/176] Windows can't do leading 0s and it's not that important --- tests/util/network/resolve.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/util/network/resolve.cpp b/tests/util/network/resolve.cpp index b73488b60..cc632da09 100644 --- a/tests/util/network/resolve.cpp +++ b/tests/util/network/resolve.cpp @@ -56,17 +56,6 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r } } - SECTION("IPv4 address with leading zeros") { - std::string address = "127.000.000.001"; - uint16_t port = 80; - - auto result = NUClear::util::network::resolve(address, port); - - REQUIRE(result.sock.sa_family == AF_INET); - REQUIRE(ntohs(result.ipv4.sin_port) == port); - REQUIRE(ntohl(result.ipv4.sin_addr.s_addr) == INADDR_LOOPBACK); - } - SECTION("IPv6 address with mixed case letters") { std::string address = "2001:0DB8:Ac10:FE01:0000:0000:0000:0000"; uint16_t port = 80; From 0811e5f04df2505d68adfbd10bc53c2c3a03e86a Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Wed, 6 Sep 2023 16:30:07 +1000 Subject: [PATCH 154/176] Refactor the iocontroller to make it handle event timing better --- src/dsl/word/UDP.hpp | 3 +- src/extension/IOController_Posix.hpp | 127 ++++++++++++------------ src/extension/IOController_Windows.hpp | 128 ++++++++++++++----------- tests/dsl/IO.cpp | 5 +- 4 files changed, 142 insertions(+), 121 deletions(-) diff --git a/src/dsl/word/UDP.hpp b/src/dsl/word/UDP.hpp index 3deb2f9e5..5fd0e7134 100644 --- a/src/dsl/word/UDP.hpp +++ b/src/dsl/word/UDP.hpp @@ -333,7 +333,6 @@ namespace dsl { template static inline RecvResult read(threading::Reaction& reaction) { - // Get our file descriptor from the magic cache auto event = IO::get(reaction); @@ -429,7 +428,7 @@ namespace dsl { p.local = Packet::Target{local_s.first, local_s.second}; p.remote = Packet::Target{remote_s.first, remote_s.second}; - // Confirm that this packet was sent to one of our broadcast addresses + // Confirm that this packet was sent to one of our local addresses for (const auto& iface : util::network::get_interfaces()) { if (iface.ip.sock.sa_family == result.local.sock.sa_family) { // If the two are equal diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index 0e772a36c..b1cc6d04d 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -35,21 +35,26 @@ namespace extension { class IOController : public Reactor { private: + /// @brief The type that poll uses for events + using event_t = short; // NOLINT(google-runtime-int) + /** * @brief A task that is waiting for an IO event */ struct Task { Task() = default; // NOLINTNEXTLINE(google-runtime-int) - Task(const fd_t& fd, short events, std::shared_ptr reaction) - : fd(fd), events(events), reaction(std::move(reaction)) {} + Task(const fd_t& fd, event_t listening_events, std::shared_ptr reaction) + : fd(fd), listening_events(listening_events), reaction(std::move(reaction)) {} /// @brief The file descriptor we are waiting on fd_t fd{-1}; - /// @brief The events that are waiting to be fired - short waiting_events{0}; // NOLINT(google-runtime-int) /// @brief The events that the task is interested in - short events{0}; // NOLINT(google-runtime-int) + event_t listening_events{0}; + /// @brief The events that are waiting to be fired + event_t waiting_events{0}; + /// @brief The events that are currently being processed + event_t processing_events{0}; /// @brief The reaction that is waiting for this event std::shared_ptr reaction{nullptr}; @@ -66,7 +71,7 @@ namespace extension { * @return false if this task is greater than or equal to the other */ bool operator<(const Task& other) const { - return fd == other.fd ? events < other.events : fd < other.fd; + return fd == other.fd ? listening_events < other.listening_events : fd < other.fd; } }; @@ -89,11 +94,11 @@ namespace extension { for (const auto& r : tasks) { // If we are the same fd, then add our interest set if (r.fd == watches.back().fd) { - watches.back().events = short(watches.back().events | r.events); // NOLINT(google-runtime-int) + watches.back().events |= r.listening_events; } // Otherwise add a new one else { - watches.push_back(pollfd{r.fd, r.events, 0}); + watches.push_back(pollfd{r.fd, r.listening_events, 0}); } } @@ -102,9 +107,43 @@ namespace extension { } /** - * @brief Collects the events that have happened and stores them on the reactions to be fired + * @brief Fires the event for the task if it is ready + * + * @param task the task to try to fire the event for + * + * @return the iterator to the next task in the list + */ + void fire_event(Task& task) { + if (task.processing_events == 0 && task.waiting_events != 0) { + + // Make our event to pass through and store it in the local cache + IO::Event e{}; + e.fd = task.fd; + e.events = task.waiting_events; + + // Clear the waiting events, we are now processing them + task.processing_events = task.waiting_events; + task.waiting_events = 0; + + // Submit the task (which should run the get) + IO::ThreadEventStore::value = &e; + std::unique_ptr r = task.reaction->get_task(); + IO::ThreadEventStore::value = nullptr; + + if (r != nullptr) { + powerplant.submit(std::move(r)); + } + else { + task.waiting_events |= task.processing_events; + task.processing_events = 0; + } + } + } + + /** + * @brief Collects the events that have happened and sets them up to fire */ - void collect_events() { + void process_events() { // Get the lock so we don't concurrently modify the list const std::lock_guard lock(tasks_mutex); @@ -126,17 +165,6 @@ namespace extension { } // It's a regular handle else { - // Check how many bytes are available to read, if it's 0 and we have a read event the - // descriptor is sending EOF and we should fire a CLOSE event too and stop watching - if ((fd.revents & IO::READ) != 0) { - int bytes_available = 0; - const bool valid = ::ioctl(fd.fd, FIONREAD, &bytes_available) == 0; - if (valid && bytes_available == 0) { - // NOLINTNEXTLINE(google-runtime-int) - fd.revents = short(fd.revents | IO::CLOSE); - } - } - // Find our relevant tasks auto range = std::equal_range(tasks.begin(), tasks.end(), @@ -151,10 +179,10 @@ namespace extension { else { // Loop through our values for (auto it = range.first; it != range.second; ++it) { - // Load in the relevant events that happened into the waiting events - // NOLINTNEXTLINE(google-runtime-int) - it->waiting_events = short(it->waiting_events | (it->events & fd.revents)); + it->waiting_events |= event_t(it->listening_events & fd.revents); + + fire_event(*it); } } } @@ -165,37 +193,6 @@ namespace extension { } } - /** - * @brief Fires the events that have been collected when the reactions are ready - */ - void fire_events() { - const std::lock_guard lock(tasks_mutex); - - // Go through every reaction and if it has events and isn't already running then run it - for (auto it = tasks.begin(); it != tasks.end();) { - if (it->reaction->active_tasks == 0 && it->waiting_events != 0) { - - // Make our event to pass through and store it in the local cache - IO::Event e{}; - e.fd = it->fd; - e.events = it->waiting_events; - - // Submit the task (which should run the get) - IO::ThreadEventStore::value = &e; - powerplant.submit(it->reaction->get_task()); - IO::ThreadEventStore::value = nullptr; - - // Remove if we received a close event - const bool closed = (it->waiting_events & IO::CLOSE) != 0; - dirty |= closed; - it = closed ? tasks.erase(it) : std::next(it); - } - else { - ++it; - } - } - } - /** * @brief Bumps the notification pipe to wake up the poll command * @@ -236,7 +233,7 @@ namespace extension { const std::lock_guard lock(tasks_mutex); // NOLINTNEXTLINE(google-runtime-int) - tasks.emplace_back(config.fd, short(config.events), config.reaction); + tasks.emplace_back(config.fd, event_t(config.events), config.reaction); // Resort our list std::sort(tasks.begin(), tasks.end()); @@ -257,7 +254,18 @@ namespace extension { // If we found it then clear the waiting events if (task != tasks.end()) { - task->waiting_events = 0; + // If the events we were processing included close remove it from the list + if (task->processing_events & IO::CLOSE) { + dirty = true; + tasks.erase(task); + } + else { + // We have finished processing events + task->processing_events = 0; + + // Try to fire again which will check if there are any waiting events + fire_event(*task); + } } }); @@ -305,10 +313,7 @@ namespace extension { } // Collect the events that happened into the tasks list - collect_events(); - - // Fire the events that happened if we can - fire_events(); + process_events(); } }); } diff --git a/src/extension/IOController_Windows.hpp b/src/extension/IOController_Windows.hpp index 56d8299a2..19393c2c6 100644 --- a/src/extension/IOController_Windows.hpp +++ b/src/extension/IOController_Windows.hpp @@ -29,21 +29,25 @@ namespace extension { class IOController : public Reactor { private: + /// @brief The type that poll uses for events + using event_t = long; // NOLINT(google-runtime-int) + /** * @brief A task that is waiting for an IO event */ struct Task { Task() = default; - // NOLINTNEXTLINE(google-runtime-int) - Task(const fd_t& fd, long events, std::shared_ptr reaction) - : fd(fd), events(events), reaction(std::move(reaction)) {} + Task(const fd_t& fd, event_t listening_events, std::shared_ptr reaction) + : fd(fd), listening_events(listening_events), reaction(std::move(reaction)) {} /// @brief The socket we are waiting on fd_t fd; /// @brief The events that the task is interested in - long events{0}; // NOLINT(google-runtime-int) + event_t listening_events{0}; /// @brief The events that are waiting to be fired - long waiting_events{0}; // NOLINT(google-runtime-int) + event_t waiting_events{0}; + /// @brief The events that are currently being processed + event_t processing_events{0}; /// @brief The reaction that is waiting for this event std::shared_ptr reaction{nullptr}; }; @@ -73,9 +77,44 @@ namespace extension { } /** - * @brief Collects the events that have happened and stores them on the reactions to be fired + * @brief Fires the event for the task if it is ready + * + * @param task the task to try to fire the event for + * + * @return the iterator to the next task in the list + */ + void fire_event(Task& task) { + if (task.processing_events == 0 && task.waiting_events != 0) { + + // Make our event to pass through and store it in the local cache + IO::Event e{}; + e.fd = task.fd; + e.events = task.waiting_events; + + // Clear the waiting events, we are now processing them + task.processing_events = task.waiting_events; + task.waiting_events = 0; + + // Submit the task (which should run the get) + IO::ThreadEventStore::value = &e; + std::unique_ptr r = task.reaction->get_task(); + IO::ThreadEventStore::value = nullptr; + + if (r != nullptr) { + powerplant.submit(std::move(r)); + } + else { + // Waiting events are still waiting + task.waiting_events |= task.processing_events; + task.processing_events = 0; + } + } + } + + /** + * @brief Collects the events that have happened and sets them up to fire */ - void collect_events(const WSAEVENT& event) { + void process_event(const WSAEVENT& event) { // Get the lock so we don't concurrently modify the list const std::lock_guard lock(tasks_mutex); @@ -102,18 +141,8 @@ namespace extension { "WSAEnumNetworkEvents() failed"); } - // Check how many bytes are available to read, if it's 0 and we have a read event the - // descriptor is sending EOF and we should fire a CLOSE event too and stop watching - long events = wsae.lNetworkEvents; - if ((events & IO::READ) != 0) { - u_long bytes_available = 0; - bool valid = ::ioctlsocket(r->second.fd, FIONREAD, &bytes_available) == 0; - if (valid && bytes_available == 0) { - events = events | IO::CLOSE; - } - } - - r->second.waiting_events = r->second.waiting_events | events; + r->second.waiting_events |= wsae.lNetworkEvents; + fire_event(r->second); } // If we can't find the event then our list is dirty else { @@ -122,37 +151,6 @@ namespace extension { } } - /** - * @brief Fires the events that have been collected when the reactions are ready - */ - void fire_events() { - const std::lock_guard lock(tasks_mutex); - - // Go through every reaction and if it has events and isn't already running then run it - for (auto it = tasks.begin(); it != tasks.end();) { - auto& task = it->second; - - if (task.reaction->active_tasks == 0 && task.waiting_events != 0) { - - // Make our event to pass through and store it in the local cache - IO::Event e{}; - e.fd = task.fd; - e.events = task.waiting_events; - - // Submit the task (which should run the get) - IO::ThreadEventStore::value = &e; - powerplant.submit(task.reaction->get_task()); - IO::ThreadEventStore::value = nullptr; - - // Reset our value - it = ((task.waiting_events & IO::CLOSE) != 0) ? remove_task(it) : std::next(it); - } - else { - ++it; - } - } - } - /** * @brief Bumps the notification pipe to wake up the poll command * @@ -168,6 +166,13 @@ namespace extension { } } + /** + * @brief Removes a task from the list and closes the event + * + * @param it the iterator to the task to remove + * + * @return the iterator to the next task + */ std::map::iterator remove_task(std::map::iterator it) { // Close the event WSAEVENT event = it->first; @@ -229,13 +234,25 @@ namespace extension { const std::lock_guard lock(tasks_mutex); // Find the reaction that finished processing - auto task = std::find_if(tasks.begin(), tasks.end(), [&event](const std::pair& t) { + auto it = std::find_if(tasks.begin(), tasks.end(), [&event](const std::pair& t) { return t.second.reaction->id == event.id; }); // If we found it then clear the waiting events - if (task != tasks.end()) { - task->second.waiting_events = 0; + if (it != tasks.end()) { + auto& task = it->second; + // If the events we were processing included close remove it from the list + if (task.processing_events & IO::CLOSE) { + dirty = true; + remove_task(it); + } + else { + // We have finished processing events + task.processing_events = 0; + + // Try to fire again which will check if there are any waiting events + fire_event(task); + } } }); @@ -288,10 +305,7 @@ namespace extension { auto& event = watches[event_index - WSA_WAIT_EVENT_0]; // Collect the events that happened into the tasks list - collect_events(event); - - // Fire the events that happened if we can - fire_events(); + process_event(event); } } }); diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index 085a89899..c7566896b 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -70,7 +70,10 @@ class TestReactor : public test_util::TestBase { const char c = "Hello"[char_no++]; const ssize_t sent = ::write(e.fd, &c, 1); - write_events.push_back("Wrote " + std::to_string(sent) + " bytes (" + c + ") to pipe"); + // If we wrote something, log it + if (sent > 0) { + write_events.push_back("Wrote " + std::to_string(sent) + " bytes (" + c + ") to pipe"); + } if (char_no == 5) { ::close(e.fd); From 4a78e2b1983a0e9d2f745aa82c0275b0ddc7755b Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 8 Sep 2023 15:37:18 +1000 Subject: [PATCH 155/176] Fix closes on linux --- src/extension/IOController_Posix.hpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index b1cc6d04d..a5e957786 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -165,6 +165,19 @@ namespace extension { } // It's a regular handle else { + // Check if we have a read event but 0 bytes to read, this can happen when a socket is closed + // On linux we don't get a close event, we just keep getting read events with 0 bytes + // To make the close event happen if we get a read event with 0 bytes we will check if there are + // any currently processing reads and if not, then close + bool maybe_eof = false; + if ((fd.revents & IO::READ) != 0) { + int bytes_available = 0; + const bool valid = ::ioctl(fd.fd, FIONREAD, &bytes_available) == 0; + if (valid && bytes_available == 0) { + maybe_eof = true; + } + } + // Find our relevant tasks auto range = std::equal_range(tasks.begin(), tasks.end(), @@ -182,6 +195,10 @@ namespace extension { // Load in the relevant events that happened into the waiting events it->waiting_events |= event_t(it->listening_events & fd.revents); + if (maybe_eof && (it->processing_events & IO::READ) == 0) { + it->waiting_events |= IO::CLOSE; + } + fire_event(*it); } } From 7e9d1c96299a392ec8811590d70dd461cc490626 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 8 Sep 2023 16:07:35 +1000 Subject: [PATCH 156/176] clang-tidy --- src/extension/IOController_Posix.hpp | 10 +++---- tests/dsl/TCP.cpp | 2 +- tests/dsl/emit/Delay.cpp | 2 +- tests/test_util/has_ipv6.cpp | 11 +++----- tests/util/network/resolve.cpp | 40 ++++++++++++++-------------- 5 files changed, 31 insertions(+), 34 deletions(-) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index a5e957786..b8cd1dc05 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -36,7 +36,7 @@ namespace extension { class IOController : public Reactor { private: /// @brief The type that poll uses for events - using event_t = short; // NOLINT(google-runtime-int) + using event_t = decltype(pollfd::events); // NOLINT(google-runtime-int) /** * @brief A task that is waiting for an IO event @@ -94,7 +94,7 @@ namespace extension { for (const auto& r : tasks) { // If we are the same fd, then add our interest set if (r.fd == watches.back().fd) { - watches.back().events |= r.listening_events; + watches.back().events = event_t(watches.back().events | r.listening_events); } // Otherwise add a new one else { @@ -134,7 +134,7 @@ namespace extension { powerplant.submit(std::move(r)); } else { - task.waiting_events |= task.processing_events; + task.waiting_events = event_t(task.waiting_events | task.processing_events); task.processing_events = 0; } } @@ -193,7 +193,7 @@ namespace extension { // Loop through our values for (auto it = range.first; it != range.second; ++it) { // Load in the relevant events that happened into the waiting events - it->waiting_events |= event_t(it->listening_events & fd.revents); + it->waiting_events = event_t(it->waiting_events | (it->listening_events & fd.revents)); if (maybe_eof && (it->processing_events & IO::READ) == 0) { it->waiting_events |= IO::CLOSE; @@ -272,7 +272,7 @@ namespace extension { // If we found it then clear the waiting events if (task != tasks.end()) { // If the events we were processing included close remove it from the list - if (task->processing_events & IO::CLOSE) { + if ((task->processing_events & IO::CLOSE) != 0) { dirty = true; tasks.erase(task); } diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 483e98065..33b2502be 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -134,7 +134,7 @@ class TestReactor : public test_util::TestBase { #ifdef _WIN32 DWORD timeout = 100; #else - struct timeval timeout; + timeval timeout{}; timeout.tv_sec = 0; timeout.tv_usec = 100000; #endif diff --git a/tests/dsl/emit/Delay.cpp b/tests/dsl/emit/Delay.cpp index 952dd80e7..b38fbcd2e 100644 --- a/tests/dsl/emit/Delay.cpp +++ b/tests/dsl/emit/Delay.cpp @@ -77,7 +77,7 @@ class TestReactor : public test_util::TestBase { on().then([this] { // Get our jump size in milliseconds - int jump_unit = (TestUnits::period::num * 1000) / TestUnits::period::den; + const int jump_unit = (TestUnits::period::num * 1000) / TestUnits::period::den; // Delay with consistent jumps for (int i = 0; i < test_loops; ++i) { auto delay = std::chrono::milliseconds(jump_unit * i); diff --git a/tests/test_util/has_ipv6.cpp b/tests/test_util/has_ipv6.cpp index ae6ba40f6..a99ee6166 100644 --- a/tests/test_util/has_ipv6.cpp +++ b/tests/test_util/has_ipv6.cpp @@ -22,13 +22,10 @@ namespace test_util { bool has_ipv6() { - // Get the first IPv6 address we can find - for (const auto& iface : NUClear::util::network::get_interfaces()) { - if (iface.ip.sock.sa_family == AF_INET6) { - return true; - } - } - return false; + // See if any interface has an ipv6 address + return std::any_of(NUClear::util::network::get_interfaces().begin(), + NUClear::util::network::get_interfaces().end(), + [](const auto& iface) { return iface.ip.sock.sa_family == AF_INET6; }); } } // namespace test_util diff --git a/tests/util/network/resolve.cpp b/tests/util/network/resolve.cpp index cc632da09..55371b9e0 100644 --- a/tests/util/network/resolve.cpp +++ b/tests/util/network/resolve.cpp @@ -5,10 +5,10 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][resolve]") { SECTION("IPv4 address") { - std::string address = "127.0.0.1"; - uint16_t port = 80; + const std::string address = "127.0.0.1"; + const uint16_t port = 80; - auto result = NUClear::util::network::resolve(address, port); + const auto result = NUClear::util::network::resolve(address, port); REQUIRE(result.sock.sa_family == AF_INET); REQUIRE(ntohs(result.ipv4.sin_port) == port); @@ -16,10 +16,10 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r } SECTION("IPv6 address") { - std::string address = "::1"; - uint16_t port = 80; + const std::string address = "::1"; + const uint16_t port = 80; - auto result = NUClear::util::network::resolve(address, port); + const auto result = NUClear::util::network::resolve(address, port); REQUIRE(result.sock.sa_family == AF_INET6); REQUIRE(ntohs(result.ipv6.sin6_port) == port); @@ -32,10 +32,10 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r } SECTION("Hostname") { - std::string address = "localhost"; - uint16_t port = 80; + const std::string address = "localhost"; + const uint16_t port = 80; - auto result = NUClear::util::network::resolve(address, port); + const auto result = NUClear::util::network::resolve(address, port); // Check that the returned socket address matches the expected address and port REQUIRE((result.sock.sa_family == AF_INET || result.sock.sa_family == AF_INET6)); @@ -57,10 +57,10 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r } SECTION("IPv6 address with mixed case letters") { - std::string address = "2001:0DB8:Ac10:FE01:0000:0000:0000:0000"; - uint16_t port = 80; + const std::string address = "2001:0DB8:Ac10:FE01:0000:0000:0000:0000"; + const uint16_t port = 80; - auto result = NUClear::util::network::resolve(address, port); + const auto result = NUClear::util::network::resolve(address, port); REQUIRE(result.sock.sa_family == AF_INET6); REQUIRE(ntohs(result.ipv6.sin6_port) == port); @@ -83,10 +83,10 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r } SECTION("Hostname with valid IPv4 address") { - std::string address = "ipv4.google.com"; - uint16_t port = 80; + const std::string address = "ipv4.google.com"; + const uint16_t port = 80; - auto result = NUClear::util::network::resolve(address, port); + const auto result = NUClear::util::network::resolve(address, port); REQUIRE(result.sock.sa_family == AF_INET); REQUIRE(ntohs(result.ipv4.sin_port) == port); @@ -94,10 +94,10 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r } SECTION("Hostname with valid IPv6 address") { - std::string address = "ipv6.google.com"; - uint16_t port = 80; + const std::string address = "ipv6.google.com"; + const uint16_t port = 80; - auto result = NUClear::util::network::resolve(address, port); + const auto result = NUClear::util::network::resolve(address, port); REQUIRE(result.sock.sa_family == AF_INET6); REQUIRE(ntohs(result.ipv6.sin6_port) == port); @@ -112,8 +112,8 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r } SECTION("Invalid address") { - std::string address = "this.url.is.invalid"; - uint16_t port = 12345; + const std::string address = "this.url.is.invalid"; + const uint16_t port = 12345; // Check that the function throws a std::runtime_error with the appropriate message REQUIRE_THROWS(NUClear::util::network::resolve(address, port)); From 45113236e70b205c75ef5ef31f915f861242321a Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 8 Sep 2023 16:19:51 +1000 Subject: [PATCH 157/176] Make the IO read read as much as it can --- tests/dsl/IO.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index c7566896b..c544d6467 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -51,10 +51,11 @@ class TestReactor : public test_util::TestBase { if ((e.events & IO::READ) != 0) { // Read from our fd char c{0}; - auto bytes = ::read(e.fd, &c, 1); - - if (bytes > 0) { - read_events.push_back("Read " + std::to_string(bytes) + " bytes (" + c + ") from pipe"); + for (int bytes = ::read(e.fd, &c, 1); bytes > 0; bytes = ::read(e.fd, &c, 1)) { + // If we read something, log it + if (bytes > 0) { + read_events.push_back("Read " + std::to_string(bytes) + " bytes (" + c + ") from pipe"); + } } } From 49a21cca0c546d51c0b2a7bbdf0272dc28b1c9d2 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 8 Sep 2023 16:41:57 +1000 Subject: [PATCH 158/176] More time for tests --- .github/workflows/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e31c27b81..224663fca 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -56,7 +56,7 @@ jobs: run: cmake --build build --config Release --parallel 2 - name: Test - timeout-minutes: 5 + timeout-minutes: 10 # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | @@ -82,7 +82,7 @@ jobs: run: cmake --build build --config Release --parallel 2 - name: Test - timeout-minutes: 5 + timeout-minutes: 10 # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | @@ -110,7 +110,7 @@ jobs: run: cmake --build build --config Release --parallel 2 - name: Test - timeout-minutes: 5 + timeout-minutes: 10 # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | From 7a62355a033a187cca648fecbda38067177d6297 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 21:16:45 +1000 Subject: [PATCH 159/176] Make the IO nonblocking to avoid hanging --- tests/dsl/IO.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index c544d6467..b8b5bc947 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -47,6 +47,10 @@ class TestReactor : public test_util::TestBase { in = fds[0]; out = fds[1]; + // Set the pipe to non-blocking + ::fcntl(in.get(), F_SETFL, ::fcntl(in.get(), F_GETFL) | O_NONBLOCK); + ::fcntl(out.get(), F_SETFL, ::fcntl(out.get(), F_GETFL) | O_NONBLOCK); + on(in.get(), IO::READ | IO::CLOSE).then([this](const IO::Event& e) { if ((e.events & IO::READ) != 0) { // Read from our fd @@ -68,11 +72,12 @@ class TestReactor : public test_util::TestBase { writer = on(out.get(), IO::WRITE).then([this](const IO::Event& e) { // Send data into our fd - const char c = "Hello"[char_no++]; + const char c = "Hello"[char_no]; const ssize_t sent = ::write(e.fd, &c, 1); - // If we wrote something, log it + // If we wrote something, log it and move to the next character if (sent > 0) { + ++char_no; write_events.push_back("Wrote " + std::to_string(sent) + " bytes (" + c + ") to pipe"); } From b9a5131c2aeac670920198bafa53a1687ce8ea30 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 21:34:58 +1000 Subject: [PATCH 160/176] clang-tidy --- tests/dsl/IO.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dsl/IO.cpp b/tests/dsl/IO.cpp index b8b5bc947..7e7b9ae28 100644 --- a/tests/dsl/IO.cpp +++ b/tests/dsl/IO.cpp @@ -55,7 +55,7 @@ class TestReactor : public test_util::TestBase { if ((e.events & IO::READ) != 0) { // Read from our fd char c{0}; - for (int bytes = ::read(e.fd, &c, 1); bytes > 0; bytes = ::read(e.fd, &c, 1)) { + for (auto bytes = ::read(e.fd, &c, 1); bytes > 0; bytes = ::read(e.fd, &c, 1)) { // If we read something, log it if (bytes > 0) { read_events.push_back("Read " + std::to_string(bytes) + " bytes (" + c + ") from pipe"); From d0155be8171491200d68837b7865cee80a4130f1 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 21:35:07 +1000 Subject: [PATCH 161/176] lint rule no longer violated --- src/extension/IOController_Posix.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extension/IOController_Posix.hpp b/src/extension/IOController_Posix.hpp index b8cd1dc05..aeebaa8b0 100644 --- a/src/extension/IOController_Posix.hpp +++ b/src/extension/IOController_Posix.hpp @@ -36,7 +36,7 @@ namespace extension { class IOController : public Reactor { private: /// @brief The type that poll uses for events - using event_t = decltype(pollfd::events); // NOLINT(google-runtime-int) + using event_t = decltype(pollfd::events); /** * @brief A task that is waiting for an IO event From 13b4602b706f3c3479fecd43f294c6c7f8187f3d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 22:00:17 +1000 Subject: [PATCH 162/176] Add a test for demangle --- tests/CMakeLists.txt | 2 +- tests/util/demangle.cpp | 109 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 tests/util/demangle.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 78634ebde..a08231576 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,7 +25,7 @@ if(CATCH_FOUND) add_compile_definitions(CATCH_CONFIG_CONSOLE_WIDTH=120) - file(GLOB test_src test.cpp "api/*.cpp" "dsl/*.cpp" "dsl/emit/*.cpp" "log/*.cpp" "util/network/*.cpp" "test_util/*.cpp") + file(GLOB test_src test.cpp "api/*.cpp" "dsl/*.cpp" "dsl/emit/*.cpp" "log/*.cpp" "util/*.cpp" "util/network/*.cpp" "test_util/*.cpp") # Some tests must be executed as individual binaries file(GLOB individual_tests "${CMAKE_CURRENT_SOURCE_DIR}/individual/*.cpp") diff --git a/tests/util/demangle.cpp b/tests/util/demangle.cpp new file mode 100644 index 000000000..721ffa2b1 --- /dev/null +++ b/tests/util/demangle.cpp @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include + +#include "nuclear" + +struct TestSymbol {}; + +template +struct TestTemplate {}; + +SCENARIO("Test the demangle function correctly demangles symbols", "[util][demangle]") { + + GIVEN("A valid mangled symbol") { + const char* symbol = typeid(int).name(); + const std::string expected = "int"; + + WHEN("Demangle is called") { + const std::string result = NUClear::util::demangle(symbol); + + THEN("It should return the demangled symbol") { + REQUIRE(result == expected); + } + } + } + + GIVEN("An empty symbol") { + const char* symbol = ""; + const std::string expected{}; + + WHEN("Demangle is called") { + const std::string result = NUClear::util::demangle(symbol); + + THEN("It should return an empty string") { + REQUIRE(result == expected); + } + } + } + + GIVEN("An invalid symbol") { + const char* symbol = "InvalidSymbol"; + const std::string expected = "InvalidSymbol"; + + WHEN("Demangle is called") { + const std::string result = NUClear::util::demangle(symbol); + + THEN("It should return the original symbol") { + REQUIRE(result == expected); + } + } + } + + GIVEN("A symbol from a struct") { + const char* symbol = typeid(TestSymbol).name(); + const std::string expected = "TestSymbol"; + + WHEN("Demangle is called") { + const std::string result = NUClear::util::demangle(symbol); + + THEN("It should return the demangled symbol") { + REQUIRE(result == expected); + } + } + } + + GIVEN("A symbol in a namespace") { + const char* symbol = typeid(NUClear::message::CommandLineArguments).name(); + const std::string expected = "NUClear::message::CommandLineArguments"; + + WHEN("Demangle is called") { + const std::string result = NUClear::util::demangle(symbol); + + THEN("It should return the demangled symbol") { + REQUIRE(result == expected); + } + } + } + + GIVEN("A templated symbol") { + const char* symbol = typeid(TestTemplate).name(); + const std::string expected = "TestTemplate"; + + WHEN("Demangle is called") { + const std::string result = NUClear::util::demangle(symbol); + + THEN("It should return the demangled symbol") { + REQUIRE(result == expected); + } + } + } +} From acaf2f9f2c967c655434b73a53fa8445139dd17c Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 22:00:55 +1000 Subject: [PATCH 163/176] Add license string --- tests/util/network/resolve.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/util/network/resolve.cpp b/tests/util/network/resolve.cpp index 55371b9e0..632e9532c 100644 --- a/tests/util/network/resolve.cpp +++ b/tests/util/network/resolve.cpp @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2013 Trent Houliston , Jake Woods + * 2014-2023 Trent Houliston + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + #include "util/network/resolve.hpp" #include From 824a7d0efeb7d22d845e0f873e8b497453442a1d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 22:15:48 +1000 Subject: [PATCH 164/176] Remove unused code --- tests/dsl/UDP.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index 3fa7039aa..b09248391 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -143,14 +143,6 @@ std::vector send_targets(const std::string& type, in_port_t port) { return results; } -struct TestUDP { - TestUDP(std::string name, std::string address, in_port_t port) - : name(std::move(name)), address(std::move(address)), port(port) {} - std::string name; - std::string address; - in_port_t port; -}; - struct Finished { Finished(std::string name) : name(std::move(name)) {} std::string name; @@ -260,12 +252,6 @@ class TestReactor : public test_util::TestBase { } } - // Send a test message to the known port - on>().then([this](const TestUDP& target) { - events.push_back(" -> " + target.address + ":" + std::to_string(target.port)); - emit(std::make_unique(target.name), target.address, target.port); - }); - on>().then([this](const Finished&) { auto send_all = [this](const std::string& type, const in_port_t& port) { for (const auto& t : send_targets(type, port)) { From 33ab54c4a80f56ce291684bb1bf06ccb4aa7cb9d Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 22:25:07 +1000 Subject: [PATCH 165/176] Add exceptions when sending fails --- tests/dsl/UDP.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index b09248391..35a373c3e 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -256,11 +256,16 @@ class TestReactor : public test_util::TestBase { auto send_all = [this](const std::string& type, const in_port_t& port) { for (const auto& t : send_targets(type, port)) { events.push_back(" -> " + t.to.address + ":" + std::to_string(t.to.port)); + try { emit(std::make_unique(t.data), t.to.address, t.to.port, t.from.address, t.from.port); + } + catch (std::exception& e) { + events.push_back("Exception: " + std::string(e.what())); + } } }; From 245287df9a1da3a04f0426ea222d5d90cad6ecdd Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 22:51:12 +1000 Subject: [PATCH 166/176] Windows returns "??" sometimes for demangle --- src/util/demangle.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/demangle.cpp b/src/util/demangle.cpp index 8a8757568..6ae277a68 100644 --- a/src/util/demangle.cpp +++ b/src/util/demangle.cpp @@ -74,7 +74,7 @@ namespace util { demangled = std::regex_replace(demangled, std::regex(R"(struct\s+)"), ""); demangled = std::regex_replace(demangled, std::regex(R"(class\s+)"), ""); demangled = std::regex_replace(demangled, std::regex(R"(\s+)"), ""); - return demangled; + return demangled == "??" ? symbol : demangled; } else { return symbol; From 9fa1ada229a2d78e43b254d6e81c5dc0ee3434c6 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 22:53:01 +1000 Subject: [PATCH 167/176] Always return the if number even if it's not multicast --- src/util/network/if_number_from_address.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/util/network/if_number_from_address.cpp b/src/util/network/if_number_from_address.cpp index 540271e4f..8c7934d40 100644 --- a/src/util/network/if_number_from_address.cpp +++ b/src/util/network/if_number_from_address.cpp @@ -39,8 +39,8 @@ namespace util { // Find the correct interface to join on (the one that has our bind address) for (auto& iface : get_interfaces()) { - // iface must be, ipv6, multicast, and have the same address as our bind address - if (iface.ip.sock.sa_family == AF_INET6 && iface.flags.multicast + // iface must be, ipv6, and have the same address as our bind address + if (iface.ip.sock.sa_family == AF_INET6 && ::memcmp(iface.ip.ipv6.sin6_addr.s6_addr, ipv6.sin6_addr.s6_addr, sizeof(in6_addr)) == 0) { // Get the interface for this @@ -49,7 +49,10 @@ namespace util { } // If we get here then we couldn't find an interface - throw std::runtime_error("Could not find interface for address"); + sock_t s{}; + s.ipv6 = ipv6; + auto a = s.address(); + throw std::runtime_error("Could not find interface for address " + a.first + " (is it up?)"); } } // namespace network From 733b02db9c296cf5c36f2f195e4959d9835b5dde Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 22:53:29 +1000 Subject: [PATCH 168/176] Formatting --- tests/dsl/UDP.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index 35a373c3e..a3676438a 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -257,11 +257,11 @@ class TestReactor : public test_util::TestBase { for (const auto& t : send_targets(type, port)) { events.push_back(" -> " + t.to.address + ":" + std::to_string(t.to.port)); try { - emit(std::make_unique(t.data), - t.to.address, - t.to.port, - t.from.address, - t.from.port); + emit(std::make_unique(t.data), + t.to.address, + t.to.port, + t.from.address, + t.from.port); } catch (std::exception& e) { events.push_back("Exception: " + std::string(e.what())); From 526e660e1a5b42c074d4dbb0c8b5a31b0a8602e4 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 23:08:29 +1000 Subject: [PATCH 169/176] Change UDP test so it works on linux --- tests/dsl/UDP.cpp | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/dsl/UDP.cpp b/tests/dsl/UDP.cpp index a3676438a..948c78c06 100644 --- a/tests/dsl/UDP.cpp +++ b/tests/dsl/UDP.cpp @@ -38,6 +38,16 @@ enum TestPorts { const std::string IPV4_MULTICAST_ADDRESS = "230.12.3.22"; // NOLINT(cert-err58-cpp) const std::string IPV6_MULTICAST_ADDRESS = "ff02::230:12:3:22"; // NOLINT(cert-err58-cpp) +#ifdef __APPLE__ +// For the IPv6 test we need to bind to the IPv6 localhost address and send from it when using udp emit. +// This is because on OSX without a fully connected IPv6 there is no default route for IPv6 multicast packets +// (see `netstat -nr`) As a result if you don't specify an interface to use when sending and receiving IPv6 multicast +// packets the send/bind fails which makes the tests fail. +const std::string IPV6_BIND = "::1"; // NOLINT(cert-err58-cpp) +#else +const std::string IPV6_BIND = "::"; // NOLINT(cert-err58-cpp) +#endif + // Ephemeral ports that we will use in_port_t uni_v4_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) in_port_t uni_v6_port = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) @@ -117,7 +127,7 @@ std::vector send_targets(const std::string& type, in_port_t port) { } break; case MULTICAST_V6_KNOWN: case MULTICAST_V6_EPHEMERAL: { - results.push_back(SendTarget{type + ":Mv6", {IPV6_MULTICAST_ADDRESS, port}, {"::1", 0}}); + results.push_back(SendTarget{type + ":Mv6", {IPV6_MULTICAST_ADDRESS, port}, {IPV6_BIND, 0}}); } break; } } @@ -229,24 +239,16 @@ class TestReactor : public test_util::TestBase { multi_v4_port = std::get<1>(multi_v4); } break; - // For the IPv6 test we need to bind to the IPv6 localhost address and send from it when using udp - // emit This is because on OSX by default there is no default route for IPv6 multicast packets (see - // `netstat -nr`) As a result if you don't specify an interface to use when sending and receiving - // IPv6 multicast packets the send/bind fails which makes the tests not work. Linux does not care - // about this extra step so it doesn't break the tests - // IPv6 Multicast Known port case MULTICAST_V6_KNOWN: { - on(IPV6_MULTICAST_ADDRESS, MULTICAST_V6, "::1") + on(IPV6_MULTICAST_ADDRESS, MULTICAST_V6, IPV6_BIND) .then([this](const UDP::Packet& packet) { handle_data("Mv6K", packet); }); } break; // IPv6 Multicast Ephemeral port case MULTICAST_V6_EPHEMERAL: { - auto multi_v6 = - on(IPV6_MULTICAST_ADDRESS, 0, "::1").then([this](const UDP::Packet& packet) { - handle_data("Mv6E", packet); - }); + auto multi_v6 = on(IPV6_MULTICAST_ADDRESS, 0, IPV6_BIND) + .then([this](const UDP::Packet& packet) { handle_data("Mv6E", packet); }); multi_v6_port = std::get<1>(multi_v6); } break; } From e66f493e3574ab57f7282478f3ccdbceb1cb37e1 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 23:11:18 +1000 Subject: [PATCH 170/176] Just short circuit for empty strings --- src/util/demangle.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/util/demangle.cpp b/src/util/demangle.cpp index 6ae277a68..45eeb4691 100644 --- a/src/util/demangle.cpp +++ b/src/util/demangle.cpp @@ -59,6 +59,11 @@ namespace util { } std::string demangle(const char* symbol) { + // If the symbol is the empty string then just return it + if (symbol != nullptr && symbol[0] == '\0') { + return symbol; + } + std::lock_guard lock(symbol_mutex); // Initialise the symbols if we have to @@ -74,7 +79,7 @@ namespace util { demangled = std::regex_replace(demangled, std::regex(R"(struct\s+)"), ""); demangled = std::regex_replace(demangled, std::regex(R"(class\s+)"), ""); demangled = std::regex_replace(demangled, std::regex(R"(\s+)"), ""); - return demangled == "??" ? symbol : demangled; + return demangled; } else { return symbol; From 08d522a65fd0fe8bef5a501aea2968630c99b63a Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 23:24:53 +1000 Subject: [PATCH 171/176] Remove -s it's too hard to read --- .github/workflows/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 224663fca..d88c34a3b 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -60,7 +60,7 @@ jobs: # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | - build/tests/test_nuclear -s + build/tests/test_nuclear for f in build/tests/individual/*; do echo "Testing $f"; ./$f -s; done build-osx: @@ -86,7 +86,7 @@ jobs: # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | - build/tests/test_nuclear -s + build/tests/test_nuclear for f in build/tests/individual/*; do echo "Testing $f"; ./$f -s; done build-windows: @@ -114,7 +114,7 @@ jobs: # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | - build/tests/Release/test_nuclear.exe -s + build/tests/Release/test_nuclear.exe for f in build/tests/individual/Release/*; do echo "Testing $f"; ./$f -s; done shell: bash From 8a5200f3df5a44c027d0050f1ba7fdb96d84659f Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Fri, 15 Sep 2023 23:49:06 +1000 Subject: [PATCH 172/176] Windows hates resolving ipv6 sometimes --- tests/util/network/resolve.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/util/network/resolve.cpp b/tests/util/network/resolve.cpp index 632e9532c..e965f0faf 100644 --- a/tests/util/network/resolve.cpp +++ b/tests/util/network/resolve.cpp @@ -111,6 +111,8 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r REQUIRE(ntohl(result.ipv4.sin_addr.s_addr) != 0); } +// For some reason windows hates this test and I'm not sure what to check to see if a windows instance can do this +#ifndef _WIN32 SECTION("Hostname with valid IPv6 address") { const std::string address = "ipv6.google.com"; const uint16_t port = 80; @@ -128,6 +130,7 @@ TEST_CASE("resolve function returns expected socket address", "[util][network][r REQUIRE(nonzero); } +#endif SECTION("Invalid address") { const std::string address = "this.url.is.invalid"; From 8fa71c45765e8b073f1a8db740ae849e024b5449 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Sat, 16 Sep 2023 00:04:03 +1000 Subject: [PATCH 173/176] Who let this obvious bug through! --- tests/test_util/has_ipv6.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_util/has_ipv6.cpp b/tests/test_util/has_ipv6.cpp index a99ee6166..41d9754b2 100644 --- a/tests/test_util/has_ipv6.cpp +++ b/tests/test_util/has_ipv6.cpp @@ -23,9 +23,10 @@ namespace test_util { bool has_ipv6() { // See if any interface has an ipv6 address - return std::any_of(NUClear::util::network::get_interfaces().begin(), - NUClear::util::network::get_interfaces().end(), - [](const auto& iface) { return iface.ip.sock.sa_family == AF_INET6; }); + auto ifaces = NUClear::util::network::get_interfaces(); + return std::any_of(ifaces.begin(), ifaces.end(), [](const auto& iface) { + return iface.ip.sock.sa_family == AF_INET6; + }); } } // namespace test_util From fdf098c6d6c3ed3dcc398e7de48c1235cb986117 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Sat, 16 Sep 2023 00:07:47 +1000 Subject: [PATCH 174/176] Increase timeouts (CI is slow) and add log line for when echos time out --- tests/dsl/TCP.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/dsl/TCP.cpp b/tests/dsl/TCP.cpp index 33b2502be..36b63b071 100644 --- a/tests/dsl/TCP.cpp +++ b/tests/dsl/TCP.cpp @@ -132,11 +132,11 @@ class TestReactor : public test_util::TestBase { // Set a timeout so we don't hang forever if something goes wrong #ifdef _WIN32 - DWORD timeout = 100; + DWORD timeout = 500; #else timeval timeout{}; timeout.tv_sec = 0; - timeout.tv_usec = 100000; + timeout.tv_usec = 500000; #endif ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast(&timeout), sizeof(timeout)); @@ -152,7 +152,12 @@ class TestReactor : public test_util::TestBase { // Receive the echo std::array buff{}; const ssize_t recv = ::recv(fd, buff.data(), socklen_t(target.name.size()), 0); - events.push_back(target.name + " echoed: " + std::string(buff.data(), recv)); + if (recv <= 1) { + events.push_back(target.name + " failed to receive echo"); + } + else { + events.push_back(target.name + " echoed: " + std::string(buff.data(), recv)); + } }); on, Sync>().then([this](const Finished&) { From 93f2a1c6703fe995fdf9ecb415c479023c7476b7 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Sat, 16 Sep 2023 00:40:57 +1000 Subject: [PATCH 175/176] . From 73cfd62148fc7a69688e5f20651a9f60736b81c1 Mon Sep 17 00:00:00 2001 From: Trent Houliston Date: Sat, 16 Sep 2023 08:37:05 +1000 Subject: [PATCH 176/176] . --- .github/workflows/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d88c34a3b..af028dc64 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -61,7 +61,7 @@ jobs: # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | build/tests/test_nuclear - for f in build/tests/individual/*; do echo "Testing $f"; ./$f -s; done + for f in build/tests/individual/*; do echo "Testing $f"; ./$f; done build-osx: name: MacOS Clang @@ -87,7 +87,7 @@ jobs: # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | build/tests/test_nuclear - for f in build/tests/individual/*; do echo "Testing $f"; ./$f -s; done + for f in build/tests/individual/*; do echo "Testing $f"; ./$f; done build-windows: name: Windows MSVC @@ -115,7 +115,7 @@ jobs: # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: | build/tests/Release/test_nuclear.exe - for f in build/tests/individual/Release/*; do echo "Testing $f"; ./$f -s; done + for f in build/tests/individual/Release/*; do echo "Testing $f"; ./$f; done shell: bash check-clang-tidy-linux: