diff --git a/.github/workflows/gcc.yaml b/.github/workflows/gcc.yaml index 25a8b7ced..6d8b6c832 100644 --- a/.github/workflows/gcc.yaml +++ b/.github/workflows/gcc.yaml @@ -92,8 +92,14 @@ jobs: - name: Test timeout-minutes: 2 - working-directory: build/tests - run: ctest --output-on-failure -E "dsl/UDP" + working-directory: build + run: ninja run_all_tests -k 0 + + - name: Test Summary + if: ${{ !cancelled() }} + uses: test-summary/action@v2 + with: + paths: "build/reports/tests/*.junit.xml" - name: Upload Traces if: ${{ !cancelled() }} diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 0e7d47289..4483dd9f6 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -60,8 +60,14 @@ jobs: - name: Test timeout-minutes: 5 - working-directory: build/tests - run: ctest --output-on-failure + working-directory: build + run: ninja run_all_tests -k 0 + + - name: Test Summary + if: ${{ !cancelled() }} + uses: test-summary/action@v2 + with: + paths: "build/reports/tests/*.junit.xml" - name: Upload Traces if: ${{ !cancelled() }} diff --git a/.github/workflows/sonarcloud.yaml b/.github/workflows/sonarcloud.yaml index 69f0b0ec9..f147354d5 100644 --- a/.github/workflows/sonarcloud.yaml +++ b/.github/workflows/sonarcloud.yaml @@ -33,9 +33,9 @@ jobs: uses: SonarSource/sonarcloud-github-c-cpp@v3 - name: Install CMake - uses: lukka/get-cmake@latest + uses: lukka/get-cmake@v3.30.5 with: - cmakeVersion: 3.27.1 + cmakeVersion: 3.30.5 ninjaVersion: 1.11.1 - name: Setup CCache @@ -45,14 +45,16 @@ jobs: max-size: 100M - name: Configure CMake + env: + CXXFLAGS: -DNUCLEAR_TEST_TIME_UNIT_DEN=10 run: | cmake -E make_directory build cmake -S . -B build \ -GNinja \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DBUILD_TESTS=ON \ -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_TESTS=ON \ -DCI_BUILD=ON \ -DENABLE_COVERAGE=ON \ -DENABLE_CLANG_TIDY=OFF @@ -63,12 +65,28 @@ jobs: - name: Run tests to generate coverage statistics timeout-minutes: 10 - working-directory: build/tests - run: ctest --output-on-failure + working-directory: build + run: ninja run_all_tests -j1 -k 0 + + - name: Test Summary + if: ${{ !cancelled() }} + uses: test-summary/action@v2 + with: + paths: "build/reports/tests/*.junit.xml" - name: Collect coverage into one XML report if: ${{ !cancelled() }} - run: gcovr --gcov-ignore-parse-errors=negative_hits.warn_once_per_file --exclude-unreachable-branches --exclude-noncode-lines --sonarqube > coverage.xml + run: | + gcovr \ + --root . \ + --object-directory build \ + --force-color \ + --no-markers \ + --decisions \ + --calls \ + --exclude-noncode-lines \ + --gcov-ignore-parse-errors negative_hits.warn \ + --sonarqube "coverage.xml" - name: Upload coverage report if: ${{ !cancelled() }} @@ -86,6 +104,10 @@ jobs: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: | sonar-scanner \ + --define sonar.projectKey=Fastcode_NUClear \ + --define sonar.organization=fastcode \ + --define sonar.sources=src \ + --define sonar.tests=tests \ --define sonar.cfamily.compile-commands=build/compile_commands.json \ --define sonar.coverageReportPaths=coverage.xml diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index 380ef47cf..dd728120b 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -81,8 +81,14 @@ jobs: - name: Test timeout-minutes: 5 - working-directory: build/tests - run: ctest --output-on-failure + working-directory: build + run: ninja run_all_tests -k 0 + + - name: Test Summary + if: ${{ !cancelled() }} + uses: test-summary/action@v2 + with: + paths: "build/reports/tests/*.junit.xml" - name: Upload Traces if: ${{ !cancelled() }} diff --git a/CMakeLists.txt b/CMakeLists.txt index f61d16bc7..b30c71945 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,6 +89,7 @@ option(BUILD_TESTS "Builds all of the NUClear unit tests." ON) if(BUILD_TESTS) enable_testing() add_subdirectory(tests) + include(TestRunner) endif() # Add the documentation subdirectory diff --git a/cmake/TestRunner.cmake b/cmake/TestRunner.cmake new file mode 100644 index 000000000..0d3070821 --- /dev/null +++ b/cmake/TestRunner.cmake @@ -0,0 +1,60 @@ +# Collect all currently added targets in all subdirectories +# +# Parameters: +# - _result the list containing all found targets +# - _dir root directory to start looking from +function(get_all_catch_test_targets _result _dir) + get_property( + _subdirs + DIRECTORY "${_dir}" + PROPERTY SUBDIRECTORIES + ) + foreach(_subdir IN LISTS _subdirs) + get_all_catch_test_targets(${_result} "${_subdir}") + endforeach() + + unset(catch_targets) + get_directory_property(_sub_targets DIRECTORY "${_dir}" BUILDSYSTEM_TARGETS) + foreach(target ${_sub_targets}) + get_target_property(target_type ${target} TYPE) + if(target_type STREQUAL "EXECUTABLE") + get_target_property(target_link_libraries ${target} INTERFACE_LINK_LIBRARIES) + if(target_link_libraries MATCHES "Catch2::Catch2") + list(APPEND catch_targets ${target}) + endif() + endif() + endforeach() + + set(${_result} + ${${_result}} ${catch_targets} + PARENT_SCOPE + ) +endfunction() + +# Find all executable targets that link to Catch2::Catch2WithMain || Catch2::Catch2 +get_all_catch_test_targets(all_targets ${PROJECT_SOURCE_DIR}) + +# Create a custom command for each test target to run it +# Make sure that coverage data is written with paths relative to the source directory +unset(reports) +foreach(target ${all_targets}) + + set(sonarqube_report_file "${PROJECT_BINARY_DIR}/reports/tests/${target}.sonarqube.xml") + set(junit_report_file "${PROJECT_BINARY_DIR}/reports/tests/${target}.junit.xml") + list(APPEND reports ${sonarqube_report_file} ${junit_report_file}) + add_custom_command( + OUTPUT ${sonarqube_report_file} ${junit_report_file} + COMMAND $ --reporter console --reporter SonarQube::out=${sonarqube_report_file} --reporter + JUnit::out=${junit_report_file} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + USES_TERMINAL + COMMENT "Running test ${target}" + ) +endforeach() + +# Create a custom target that depends on all test targets +add_custom_target( + run_all_tests + DEPENDS ${reports} + COMMENT "Running all Catch2 tests" +) diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index 60dc88e40..000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,9 +0,0 @@ -sonar.projectName=NUClear -sonar.projectKey=Fastcode_NUClear -sonar.organization=fastcode -sonar.projectVersion=1.0 - -sonar.sources=src -sonar.tests=tests - -sonar.sourceEncoding=UTF-8 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 28cd6f3b9..65f180916 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -37,12 +37,14 @@ target_link_libraries(nuclear ${CMAKE_THREAD_LIBS_INIT}) set_target_properties(nuclear PROPERTIES POSITION_INDEPENDENT_CODE ON) target_compile_features(nuclear PUBLIC cxx_std_14) -# When enabling coverage, just set it on the nuclear target so it doesn't end up on catch2 -option(ENABLE_COVERAGE "Enable coverage support" OFF) +option(ENABLE_COVERAGE "Compile with coverage support enabled.") if(ENABLE_COVERAGE) - target_compile_options(nuclear PUBLIC "-fprofile-arcs" "-ftest-coverage" "-fprofile-abs-path") - target_link_options(nuclear PUBLIC "-fprofile-arcs" "-ftest-coverage" "-fprofile-abs-path") -endif() + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(WARNING "Coverage is enabled but not build in debug mode. Coverage results may be misleading.") + endif() + target_compile_options(nuclear PUBLIC -fprofile-arcs -ftest-coverage -fprofile-abs-path -fprofile-update=atomic) + target_link_options(nuclear PUBLIC -fprofile-arcs) +endif(ENABLE_COVERAGE) # Enable warnings, and all warnings are errors if(MSVC) diff --git a/src/util/platform.hpp b/src/util/platform.hpp index a151deaf3..95a1ba98a 100644 --- a/src/util/platform.hpp +++ b/src/util/platform.hpp @@ -96,17 +96,6 @@ #endif // _WIN32 -/******************************************* - * SHIM FOR THREAD LOCAL STORAGE * - *******************************************/ -#if defined(__GNUC__) - #define ATTRIBUTE_TLS __thread -#elif defined(_WIN32) - #define ATTRIBUTE_TLS __declspec(thread) -#else // !__GNUC__ && !_MSC_VER - #error "Define a thread local storage qualifier for your compiler/platform!" -#endif - /******************************************* * SHIM FOR NETWORKING * *******************************************/ diff --git a/tests/test_util/TestBase.hpp b/tests/test_util/TestBase.hpp index f2ae21517..4c5dcc318 100644 --- a/tests/test_util/TestBase.hpp +++ b/tests/test_util/TestBase.hpp @@ -28,6 +28,7 @@ #include #include "nuclear" +#include "test_util/TimeUnit.hpp" // IWYU pragma: export #include "test_util/diff_string.hpp" // IWYU pragma: export namespace test_util { @@ -57,7 +58,7 @@ class TestBase : public NUClear::Reactor { private: explicit TestBase(std::unique_ptr environment, const bool& shutdown_on_idle = true, - const std::chrono::milliseconds& timeout = std::chrono::milliseconds(1000)) + const std::chrono::milliseconds& timeout = TimeUnit(20)) : Reactor(std::move(environment)) { // Shutdown if the system is idle diff --git a/tests/test_util/has_multicast.cpp b/tests/test_util/has_multicast.cpp new file mode 100644 index 000000000..4b5b0fbca --- /dev/null +++ b/tests/test_util/has_multicast.cpp @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2025 NUClear Contributors + * + * This file is part of the NUClear codebase. + * See https://github.com/Fastcode/NUClear for further info. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "has_multicast.hpp" + +#include + +#include "util/network/get_interfaces.hpp" +#include "util/platform.hpp" + +namespace test_util { + +bool has_ipv4_multicast() { + // See if any interface has multicast ipv4 + auto ifaces = NUClear::util::network::get_interfaces(); + return std::any_of(ifaces.begin(), ifaces.end(), [](const auto& iface) { + return iface.ip.sock.sa_family == AF_INET && iface.flags.multicast; + }); +} + +bool has_ipv6_multicast() { + // See if any interface has multicast ipv6 + auto ifaces = NUClear::util::network::get_interfaces(); + return std::any_of(ifaces.begin(), ifaces.end(), [](const auto& iface) { + return iface.ip.sock.sa_family == AF_INET6 && iface.flags.multicast; + }); +} + +} // namespace test_util diff --git a/tests/test_util/has_multicast.hpp b/tests/test_util/has_multicast.hpp new file mode 100644 index 000000000..d94b2e179 --- /dev/null +++ b/tests/test_util/has_multicast.hpp @@ -0,0 +1,33 @@ +/* + * MIT License + * + * Copyright (c) 2025 NUClear Contributors + * + * This file is part of the NUClear codebase. + * See https://github.com/Fastcode/NUClear for further info. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef TEST_UTIL_HAS_IPV4_MULTICAST_HPP +#define TEST_UTIL_HAS_IPV4_MULTICAST_HPP + +namespace test_util { + +bool has_ipv4_multicast(); +bool has_ipv6_multicast(); + +} // namespace test_util + +#endif // TEST_UTIL_HAS_IPV4_MULTICAST_HPP diff --git a/tests/tests/dsl/Every.cpp b/tests/tests/dsl/Every.cpp index ca6b00df3..0540fe1c6 100644 --- a/tests/tests/dsl/Every.cpp +++ b/tests/tests/dsl/Every.cpp @@ -39,7 +39,7 @@ class TestReactor : public test_util::TestBase { public: TestReactor(std::unique_ptr environment) - : TestBase(std::move(environment), false, std::chrono::seconds(10)) { + : TestBase(std::move(environment), false, test_util::TimeUnit(200)) { // Trigger on 3 different types of every on>>().then([this]() { every_times.push_back(NUClear::clock::now()); }); diff --git a/tests/tests/dsl/Idle.cpp b/tests/tests/dsl/Idle.cpp index 5e4026f30..318c202d6 100644 --- a/tests/tests/dsl/Idle.cpp +++ b/tests/tests/dsl/Idle.cpp @@ -57,7 +57,7 @@ class TestReactor : public test_util::TestBase { } explicit TestReactor(std::unique_ptr environment) - : TestBase(std::move(environment), false, std::chrono::seconds(5)) { + : TestBase(std::move(environment), false, test_util::TimeUnit(100)) { start_time = NUClear::clock::now(); diff --git a/tests/tests/dsl/IdleSingleGlobal.cpp b/tests/tests/dsl/IdleSingleGlobal.cpp index 76c044c2f..3ac6f814b 100644 --- a/tests/tests/dsl/IdleSingleGlobal.cpp +++ b/tests/tests/dsl/IdleSingleGlobal.cpp @@ -59,7 +59,7 @@ class TestReactor : public test_util::TestBase { static constexpr int n_loops = 10000; explicit TestReactor(std::unique_ptr environment) - : TestBase(std::move(environment), false, std::chrono::seconds(2)) { + : TestBase(std::move(environment), false, test_util::TimeUnit(200)) { /* * Run idle on the default pool, and a task on the main pool. diff --git a/tests/tests/dsl/SyncOrder.cpp b/tests/tests/dsl/SyncOrder.cpp index a9caa2c7e..760f9c533 100644 --- a/tests/tests/dsl/SyncOrder.cpp +++ b/tests/tests/dsl/SyncOrder.cpp @@ -39,7 +39,8 @@ class TestReactor : public test_util::TestBase { Message(int val) : val(val) {}; }; - TestReactor(std::unique_ptr environment) : TestBase(std::move(environment)) { + TestReactor(std::unique_ptr environment) + : TestBase(std::move(environment), true, test_util::TimeUnit(150)) { on>, Sync>().then([this](const Message<'A'>& m) { // events.emplace_back('A', m.val); diff --git a/tests/tests/dsl/TCP.cpp b/tests/tests/dsl/TCP.cpp index db872e631..d354d6b9c 100644 --- a/tests/tests/dsl/TCP.cpp +++ b/tests/tests/dsl/TCP.cpp @@ -84,7 +84,7 @@ class TestReactor : public test_util::TestBase { } TestReactor(std::unique_ptr environment, const std::vector& active_tests_) - : TestBase(std::move(environment), false, std::chrono::seconds(2)), active_tests(active_tests_) { + : TestBase(std::move(environment), false, test_util::TimeUnit(50)), active_tests(active_tests_) { for (const auto& t : active_tests) { switch (t) { diff --git a/tests/tests/dsl/UDP.cpp b/tests/tests/dsl/UDP.cpp index 38ede4176..16ca247db 100644 --- a/tests/tests/dsl/UDP.cpp +++ b/tests/tests/dsl/UDP.cpp @@ -36,6 +36,7 @@ #include "test_util/TestBase.hpp" #include "test_util/common.hpp" #include "test_util/has_ipv6.hpp" +#include "test_util/has_multicast.hpp" #include "util/network/get_interfaces.hpp" #include "util/platform.hpp" @@ -377,9 +378,11 @@ TEST_CASE("Testing sending and receiving of UDP messages", "[api][network][udp]" } active_tests.push_back(BROADCAST_V4_KNOWN); active_tests.push_back(BROADCAST_V4_EPHEMERAL); - active_tests.push_back(MULTICAST_V4_KNOWN); - active_tests.push_back(MULTICAST_V4_EPHEMERAL); - if (test_util::has_ipv6()) { + if (test_util::has_ipv4_multicast()) { + active_tests.push_back(MULTICAST_V4_KNOWN); + active_tests.push_back(MULTICAST_V4_EPHEMERAL); + } + if (test_util::has_ipv6() && test_util::has_ipv6_multicast()) { active_tests.push_back(MULTICAST_V6_KNOWN); active_tests.push_back(MULTICAST_V6_EPHEMERAL); } diff --git a/tests/tests/dsl/emit/Delay.cpp b/tests/tests/dsl/emit/Delay.cpp index c221dc43d..01ef51f8a 100644 --- a/tests/tests/dsl/emit/Delay.cpp +++ b/tests/tests/dsl/emit/Delay.cpp @@ -54,7 +54,7 @@ class TestReactor : public test_util::TestBase { struct FinishTest {}; TestReactor(std::unique_ptr environment) - : TestBase(std::move(environment), false, std::chrono::seconds(2)) { + : TestBase(std::move(environment), false, test_util::TimeUnit(50)) { // Measure when messages were sent and received and print those values on>().then([this](const DelayedMessage& m) {