Merge topic 'ctest-repeat'

32c165d263 CTest: Consolidate '--repeat-* n' options as `--repeat *:n`

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !4053

Help/manual/ctest.1.rst

@@ -261,23 +261,27 @@ Options
  fail, subsequent calls to CTest with the ``--rerun-failed`` option will run
  the set of tests that most recently failed (if any).
 
+``--repeat <mode>:<n>``
+  Run tests repeatedly based on the given ``<mode>`` up to ``<n>`` times.
+  The modes are:
+
+  ``until-fail``
+    Require each test to run ``<n>`` times without failing in order to pass.
+    This is useful in finding sporadic failures in test cases.
+
+  ``until-pass``
+    Allow each test to run up to ``<n>`` times in order to pass.
+    Repeats tests if they fail for any reason.
+    This is useful in tolerating sporadic failures in test cases.
+
+  ``after-timeout``
+    Allow each test to run up to ``<n>`` times in order to pass.
+    Repeats tests only if they timeout.
+    This is useful in tolerating sporadic timeouts in test cases
+    on busy machines.
+
 ``--repeat-until-fail <n>``
- Require each test to run ``<n>`` times without failing in order to pass.
+ Equivalent to ``--repeat until-fail:<n>``.
-
- This is useful in finding sporadic failures in test cases.
-
-``--repeat-until-pass <n>``
- Allow each test to run up to ``<n>`` times in order to pass.
- Repeats tests if they fail for any reason.
-
- This is useful in tolerating sporadic failures in test cases.
-
-``--repeat-after-timeout <n>``
- Allow each test to run up to ``<n>`` times in order to pass.
- Repeats tests only if they timeout.
-
- This is useful in tolerating sporadic timeouts in test cases
- on busy machines.
 
 ``--max-width <width>``
  Set the max width for a test name to output.

Help/release/dev/ctest-repeat-until-pass.rst

@@ -1,9 +0,0 @@
-ctest-repeat-until-pass
------------------------
-
-* The :manual:`ctest(1)` tool learned new ``--repeat-until-pass <n>``
-  and ``--repeat-after-timeout <n>`` options to help tolerate sporadic
-  test failures.
-
-* The :command:`ctest_test` command gained a ``REPEAT <mode>:<n>`` option
-  to specify conditions in which to repeat tests.

Help/release/dev/ctest-repeat.rst

@@ -0,0 +1,10 @@
+ctest-repeat
+------------
+
+* The :manual:`ctest(1)` tool gained a ``--repeat <mode>:<n>`` option
+  to specify conditions in which to repeat tests.  This generalizes
+  the existing ``--repeat-until-fail <n>`` option to add modes for
+  ``until-pass`` and ``after-timeout``.
+
+* The :command:`ctest_test` command gained a ``REPEAT <mode>:<n>`` option
+  to specify conditions in which to repeat tests.

Source/cmCTest.cxx

@@ -21,6 +21,7 @@
 #include "cmsys/FStream.hxx"
 #include "cmsys/Glob.hxx"
 #include "cmsys/Process.h"
+#include "cmsys/RegularExpression.hxx"
 #include "cmsys/SystemInformation.hxx"
 
 #include "cm_curl.h"
@@ -1846,7 +1847,7 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
       return false;
     }
     if (this->Impl->RepeatMode != cmCTest::Repeat::Never) {
-      errormsg = "At most one '--repeat-*' option may be used.";
+      errormsg = "At most one '--repeat' option may be used.";
       return false;
     }
     i++;
@@ -1862,48 +1863,37 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
     }
   }
 
-  if (this->CheckArgument(arg, "--repeat-until-pass")) {
+  if (this->CheckArgument(arg, "--repeat")) {
     if (i >= args.size() - 1) {
-      errormsg = "'--repeat-until-pass' requires an argument";
+      errormsg = "'--repeat' requires an argument";
       return false;
     }
     if (this->Impl->RepeatMode != cmCTest::Repeat::Never) {
-      errormsg = "At most one '--repeat-*' option may be used.";
+      errormsg = "At most one '--repeat' option may be used.";
       return false;
     }
     i++;
-    long repeat = 1;
+    cmsys::RegularExpression repeatRegex(
-    if (!cmStrToLong(args[i], &repeat)) {
+      "^(until-fail|until-pass|after-timeout):([0-9]+)$");
-      errormsg =
+    if (repeatRegex.find(args[i])) {
-        "'--repeat-until-pass' given non-integer value '" + args[i] + "'";
+      std::string const& count = repeatRegex.match(2);
+      unsigned long n = 1;
+      cmStrToULong(count, &n); // regex guarantees success
+      this->Impl->RepeatCount = static_cast<int>(n);
+      if (this->Impl->RepeatCount > 1) {
+        std::string const& mode = repeatRegex.match(1);
+        if (mode == "until-fail") {
+          this->Impl->RepeatMode = cmCTest::Repeat::UntilFail;
+        } else if (mode == "until-pass") {
+          this->Impl->RepeatMode = cmCTest::Repeat::UntilPass;
+        } else if (mode == "after-timeout") {
+          this->Impl->RepeatMode = cmCTest::Repeat::AfterTimeout;
+        }
+      }
+    } else {
+      errormsg = "'--repeat' given invalid value '" + args[i] + "'";
       return false;
     }
-    this->Impl->RepeatCount = static_cast<int>(repeat);
-    if (repeat > 1) {
-      this->Impl->RepeatMode = cmCTest::Repeat::UntilPass;
-    }
-  }
-
-  if (this->CheckArgument(arg, "--repeat-after-timeout")) {
-    if (i >= args.size() - 1) {
-      errormsg = "'--repeat-after-timeout' requires an argument";
-      return false;
-    }
-    if (this->Impl->RepeatMode != cmCTest::Repeat::Never) {
-      errormsg = "At most one '--repeat-*' option may be used.";
-      return false;
-    }
-    i++;
-    long repeat = 1;
-    if (!cmStrToLong(args[i], &repeat)) {
-      errormsg =
-        "'--repeat-after-timeout' given non-integer value '" + args[i] + "'";
-      return false;
-    }
-    this->Impl->RepeatCount = static_cast<int>(repeat);
-    if (repeat > 1) {
-      this->Impl->RepeatMode = cmCTest::Repeat::AfterTimeout;
-    }
   }
 
   if (this->CheckArgument(arg, "--test-load") && i < args.size() - 1) {

Source/ctest.cxx

@@ -98,11 +98,11 @@ static const char* cmDocumentationOptions[][2] = {
     "Run a specific number of tests by number." },
   { "-U, --union", "Take the Union of -I and -R" },
   { "--rerun-failed", "Run only the tests that failed previously" },
-  { "--repeat-until-fail <n>",
+  { "--repeat until-fail:<n>, --repeat-until-fail <n>",
     "Require each test to run <n> times without failing in order to pass" },
-  { "--repeat-until-pass <n>",
+  { "--repeat until-pass:<n>",
     "Allow each test to run up to <n> times in order to pass" },
-  { "--repeat-after-timeout <n>",
+  { "--repeat after-timeout:<n>",
     "Allow each test to run up to <n> times if it times out" },
   { "--max-width <width>", "Set the max width for a test name to output" },
   { "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." },

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -4,14 +4,26 @@ set(RunCMake_TEST_TIMEOUT 60)
 unset(ENV{CTEST_PARALLEL_LEVEL})
 unset(ENV{CTEST_OUTPUT_ON_FAILURE})
 
-run_cmake_command(repeat-until-pass-bad1
+run_cmake_command(repeat-opt-bad1
-  ${CMAKE_CTEST_COMMAND} --repeat-until-pass
+  ${CMAKE_CTEST_COMMAND} --repeat until-pass
   )
-run_cmake_command(repeat-until-pass-bad2
+run_cmake_command(repeat-opt-bad2
-  ${CMAKE_CTEST_COMMAND} --repeat-until-pass foo
+  ${CMAKE_CTEST_COMMAND} --repeat until-pass:foo
   )
-run_cmake_command(repeat-until-pass-good
+run_cmake_command(repeat-opt-bad3
-  ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2
+  ${CMAKE_CTEST_COMMAND} --repeat until-fail:2 --repeat-until-fail 2
+  )
+run_cmake_command(repeat-opt-bad4
+  ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat until-fail:2
+  )
+run_cmake_command(repeat-opt-until-pass
+  ${CMAKE_CTEST_COMMAND} --repeat until-pass:2
+  )
+run_cmake_command(repeat-opt-until-fail
+  ${CMAKE_CTEST_COMMAND} --repeat until-fail:2
+  )
+run_cmake_command(repeat-opt-after-timeout
+  ${CMAKE_CTEST_COMMAND} --repeat after-timeout:2
   )
 
 run_cmake_command(repeat-until-fail-bad1
@@ -24,33 +36,13 @@ run_cmake_command(repeat-until-fail-good
   ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
   )
 
-run_cmake_command(repeat-after-timeout-bad1
-  ${CMAKE_CTEST_COMMAND} --repeat-after-timeout
-  )
-run_cmake_command(repeat-after-timeout-bad2
-  ${CMAKE_CTEST_COMMAND} --repeat-after-timeout foo
-  )
-run_cmake_command(repeat-after-timeout-good
-  ${CMAKE_CTEST_COMMAND} --repeat-after-timeout 2
-  )
-
-run_cmake_command(repeat-until-pass-and-fail
-  ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2
-  )
-run_cmake_command(repeat-until-fail-and-pass
-  ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2
-  )
-run_cmake_command(repeat-until-fail-and-timeout
-  ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-after-timeout 2
-  )
-
 function(run_repeat_until_pass_tests)
   # Use a single build tree for a few tests without cleaning.
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-pass-build)
   run_cmake(repeat-until-pass-cmake)
   set(RunCMake_TEST_NO_CLEAN 1)
   run_cmake_command(repeat-until-pass-ctest
-    ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-pass 3
+    ${CMAKE_CTEST_COMMAND} -C Debug --repeat until-pass:3
     )
 endfunction()
 run_repeat_until_pass_tests()
@@ -61,7 +53,7 @@ function(run_repeat_after_timeout_tests)
   run_cmake(repeat-after-timeout-cmake)
   set(RunCMake_TEST_NO_CLEAN 1)
   run_cmake_command(repeat-after-timeout-ctest
-    ${CMAKE_CTEST_COMMAND} -C Debug --repeat-after-timeout 3
+    ${CMAKE_CTEST_COMMAND} -C Debug --repeat after-timeout:3
     )
 endfunction()
 run_repeat_after_timeout_tests()
@@ -72,10 +64,11 @@ function(run_repeat_until_fail_tests)
   run_cmake(repeat-until-fail-cmake)
   set(RunCMake_TEST_NO_CLEAN 1)
   run_cmake_command(repeat-until-fail-ctest
-    ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3
+    ${CMAKE_CTEST_COMMAND} -C Debug ${ARGN}
     )
 endfunction()
-run_repeat_until_fail_tests()
+run_repeat_until_fail_tests(--repeat-until-fail 3)
+run_repeat_until_fail_tests(--repeat until-fail:3)
 
 function(run_BadCTestTestfile)
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/BadCTestTestfile)

Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt

@@ -1 +0,0 @@
-^CMake Error: '--repeat-after-timeout' requires an argument$

Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt

@@ -1 +0,0 @@
-^CMake Error: '--repeat-after-timeout' given non-integer value 'foo'$

Tests/RunCMake/CTestCommandLine/repeat-opt-bad1-stderr.txt

@@ -0,0 +1 @@
+^CMake Error: '--repeat' given invalid value 'until-pass'$

Tests/RunCMake/CTestCommandLine/repeat-opt-bad2-stderr.txt

@@ -0,0 +1 @@
+^CMake Error: '--repeat' given invalid value 'until-pass:foo'$

Tests/RunCMake/CTestCommandLine/repeat-opt-bad3-stderr.txt

@@ -0,0 +1 @@
+^CMake Error: At most one '--repeat' option may be used\.$

Tests/RunCMake/CTestCommandLine/repeat-opt-bad4-stderr.txt

@@ -0,0 +1 @@
+^CMake Error: At most one '--repeat' option may be used\.$

Tests/RunCMake/CTestCommandLine/repeat-opt-until-pass-stderr.txt

@@ -0,0 +1 @@
+^No tests were found!!!$

Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt

@@ -1 +0,0 @@
-^CMake Error: At most one '--repeat-\*' option may be used\.$

Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt

@@ -1 +0,0 @@
-^CMake Error: At most one '--repeat-\*' option may be used\.$

Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt

@@ -1 +0,0 @@
-1

Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt

@@ -1 +0,0 @@
-^CMake Error: At most one '--repeat-\*' option may be used\.$

Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt

@@ -1 +0,0 @@
-1

Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt

@@ -1 +0,0 @@
-^CMake Error: '--repeat-until-pass' requires an argument$

Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt

@@ -1 +0,0 @@
-1

Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt

@@ -1 +0,0 @@
-^CMake Error: '--repeat-until-pass' given non-integer value 'foo'$