diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 132dc80d49..379e00dc1d 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -445,6 +445,15 @@ Run Tests This option will run the tests in a random order. It is commonly used to detect implicit dependencies in a test suite. +.. option:: --schedule-random-seed + + .. versionadded:: 4.1 + + Override the random order seed + + This option is used to allow recreating failures owing to + random order of execution by ``--schedule-random``. + .. option:: --submit-index Legacy option for old Dart2 dashboard server feature. diff --git a/Help/release/dev/ctest-schedule-random-seed.rst b/Help/release/dev/ctest-schedule-random-seed.rst new file mode 100644 index 0000000000..3a24292538 --- /dev/null +++ b/Help/release/dev/ctest-schedule-random-seed.rst @@ -0,0 +1,7 @@ +ctest-schedule-random-seed +-------------------------- + +* :manual:`ctest(1)` gained a + :option:`--schedule-random-seed ` + option to specify a numeric random seed to make + :option:`ctest --schedule-random` deterministic for reproduction. diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index 5c90c64b79..090d42d37b 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -43,6 +43,7 @@ protected: cm::optional> ParallelLevel; std::string Repeat; std::string ScheduleRandom; + std::string ScheduleRandomSeed; std::string StopTime; std::string TestLoad; std::string ResourceSpecFile; @@ -70,6 +71,7 @@ protected: .Bind("PARALLEL_LEVEL"_s, &TestArguments::ParallelLevel) .Bind("REPEAT"_s, &TestArguments::Repeat) .Bind("SCHEDULE_RANDOM"_s, &TestArguments::ScheduleRandom) + .Bind("SCHEDULE_RANDOM_SEED"_s, &TestArguments::ScheduleRandomSeed) .Bind("STOP_TIME"_s, &TestArguments::StopTime) .Bind("TEST_LOAD"_s, &TestArguments::TestLoad) .Bind("RESOURCE_SPEC_FILE"_s, &TestArguments::ResourceSpecFile) diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 69ab748ca3..80e6214fab 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -1324,7 +1324,14 @@ bool cmCTestTestHandler::ProcessDirectory(std::vector& passed, bool randomSchedule = this->CTest->GetScheduleType() == "Random"; if (randomSchedule) { - srand(static_cast(time(nullptr))); + cm::optional scheduleRandomSeed = + this->CTest->GetRandomSeed(); + if (!scheduleRandomSeed.has_value()) { + scheduleRandomSeed = static_cast(time(nullptr)); + } + srand(*scheduleRandomSeed); + *this->LogFile << "Test order random seed: " << *scheduleRandomSeed + << std::endl; } for (cmCTestTestProperties& p : this->TestList) { diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index 6ad2e60331..9388425db9 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -34,6 +34,7 @@ struct cmCTestTestOptions bool ScheduleRandom = false; bool StopOnFailure = false; bool UseUnion = false; + cm::optional ScheduleRandomSeed; int OutputSizePassed = 1 * 1024; int OutputSizeFailed = 300 * 1024; diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index eb2e93a0de..f5af092386 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -2527,6 +2527,20 @@ int cmCTest::Run(std::vector const& args) this->Impl->TestOptions.ScheduleRandom = true; return true; } }, + CommandArgument{ + "--schedule-random-seed", CommandArgument::Values::One, + [this](std::string const& sz) -> bool { + unsigned long seed_value; + if (cmStrToULong(sz, &seed_value)) { + this->Impl->TestOptions.ScheduleRandomSeed = + static_cast(seed_value); + } else { + cmCTestLog(this, WARNING, + "Invalid value for '--schedule-random-seed': " << sz + << "\n"); + } + return true; + } }, CommandArgument{ "--rerun-failed", CommandArgument::Values::Zero, [this](std::string const&) -> bool { this->Impl->TestOptions.RerunFailed = true; @@ -2795,6 +2809,11 @@ void cmCTest::SetStopTime(std::string const& time_str) } } +cm::optional cmCTest::GetRandomSeed() const +{ + return this->Impl->TestOptions.ScheduleRandomSeed; +} + std::string cmCTest::GetScheduleType() const { return this->Impl->ScheduleType; diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 53e9b0eff4..cc240ccde7 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -198,6 +198,8 @@ public: std::string GetScheduleType() const; void SetScheduleType(std::string const& type); + cm::optional GetRandomSeed() const; + /** The max output width */ int GetMaxTestNameWidth() const; void SetMaxTestNameWidth(int w); diff --git a/Source/ctest.cxx b/Source/ctest.cxx index b25f58b752..d0f7a18883 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -149,6 +149,7 @@ cmDocumentationEntry const cmDocumentationOptions[] = { { "--extra-submit [;]", "Submit extra files to the dashboard." }, { "--http-header
", "Append HTTP header when submitting" }, { "--schedule-random", "Use a random order for scheduling tests" }, + { "--schedule-random-seed", "Override seed for random order of tests" }, { "--submit-index", "Submit individual dashboard tests with specific index" }, { "--timeout ", "Set the default test timeout." }, diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index 31f3b72c05..d12ab92430 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -656,3 +656,17 @@ set_tests_properties(test1 PROPERTIES TIMEOUT_SIGNAL_GRACE_PERIOD 1000) run_cmake_command(TimeoutSignalBad ${CMAKE_CTEST_COMMAND}) endblock() endif() + +block() + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/ScheduleRandomSeed) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" " +foreach(i RANGE 1 5) + add_test(test\${i} \"${CMAKE_COMMAND}\" -E true) +endforeach() +") + run_cmake_command(ScheduleRandomSeed1 ${CMAKE_CTEST_COMMAND} --schedule-random --schedule-random-seed 42) + run_cmake_command(ScheduleRandomSeed2 ${CMAKE_CTEST_COMMAND} --schedule-random --schedule-random-seed 42) +endblock() diff --git a/Tests/RunCMake/CTestCommandLine/ScheduleRandomSeed1-check.cmake b/Tests/RunCMake/CTestCommandLine/ScheduleRandomSeed1-check.cmake new file mode 100644 index 0000000000..5c1ac43994 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/ScheduleRandomSeed1-check.cmake @@ -0,0 +1,2 @@ +string(REGEX MATCHALL "Start [1-5]" ScheduleRandomSeed1_ORDER "${actual_stdout}") +set_property(DIRECTORY PROPERTY ScheduleRandomSeed1_ORDER "${ScheduleRandomSeed1_ORDER}") diff --git a/Tests/RunCMake/CTestCommandLine/ScheduleRandomSeed2-check.cmake b/Tests/RunCMake/CTestCommandLine/ScheduleRandomSeed2-check.cmake new file mode 100644 index 0000000000..2115f45824 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/ScheduleRandomSeed2-check.cmake @@ -0,0 +1,10 @@ +string(REGEX MATCHALL "Start [1-5]" ScheduleRandomSeed2_ORDER "${actual_stdout}") +get_property(ScheduleRandomSeed1_ORDER DIRECTORY PROPERTY ScheduleRandomSeed1_ORDER) +if(NOT "${ScheduleRandomSeed1_ORDER}" STREQUAL "${ScheduleRandomSeed2_ORDER}") + string(CONCAT RunCMake_TEST_FAILED + "ScheduleRandomSeed1 order:\n" + " ${ScheduleRandomSeed1_ORDER}\n" + "does not match ScheduleRandomSeed2 order:\n" + " ${ScheduleRandomSeed2_ORDER}\n" + ) +endif()