CTest: Allow setting exit code in ctest scripts

Source/CTest/cmCTestEmptyBinaryDirectoryCommand.cxx

@@ -7,6 +7,7 @@
 #include "cmExecutionStatus.h"
 #include "cmMakefile.h"
 #include "cmMessageType.h"
+#include "cmMessenger.h"
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
 
@@ -94,10 +95,12 @@ bool cmCTestEmptyBinaryDirectoryCommand(std::vector<std::string> const& args,
 
   std::string err;
   if (!EmptyBinaryDirectory(args[0], err)) {
-    status.GetMakefile().IssueMessage(
+    cmMakefile& mf = status.GetMakefile();
+    mf.GetMessenger()->DisplayMessage(
       MessageType::FATAL_ERROR,
       cmStrCat("Did not remove the binary directory:\n ", args[0],
-               "\nbecause:\n ", err));
+               "\nbecause:\n ", err),
+      mf.GetBacktrace());
     return true;
   }
 

Source/CTest/cmCTestScriptHandler.cxx

@@ -63,10 +63,7 @@ int cmCTestScriptHandler::ProcessHandler()
     res |= this->RunConfigurationScript(this->ConfigurationScripts[i],
                                         this->ScriptProcessScope[i]);
   }
-  if (res) {
-    return -1;
-  }
-  return 0;
+  return res;
 }
 
 void cmCTestScriptHandler::UpdateElapsedTime()
@@ -180,6 +177,8 @@ void cmCTestScriptHandler::CreateCMake()
   this->CMake->SetHomeOutputDirectory("");
   this->CMake->GetCurrentSnapshot().SetDefaultDefinitions();
   this->CMake->AddCMakePaths();
+  this->CMake->SetWorkingMode(cmake::SCRIPT_MODE,
+                              cmake::CommandFailureAction::EXIT_CODE);
   this->GlobalGenerator =
     cm::make_unique<cmGlobalGenerator>(this->CMake.get());
 
@@ -289,7 +288,7 @@ int cmCTestScriptHandler::ReadInScript(std::string const& total_script_arg)
       cmSystemTools::GetErrorOccurredFlag()) {
     cmCTestLog(this->CTest, ERROR_MESSAGE,
                "Error in read:" << systemFile << "\n");
-    return 2;
+    return -1;
   }
 
   // Add definitions of variables passed in on the command line:
@@ -299,16 +298,20 @@ int cmCTestScriptHandler::ReadInScript(std::string const& total_script_arg)
     this->Makefile->AddDefinition(d.first, d.second);
   }
 
+  int res = 0;
+
   // finally read in the script
   if (!this->Makefile->ReadListFile(script) ||
       cmSystemTools::GetErrorOccurredFlag()) {
     // Reset the error flag so that it can run more than
     // one script with an error when you use ctest_run_script.
     cmSystemTools::ResetErrorOccurredFlag();
-    return 2;
+    res = -1;
   }
 
-  return 0;
+  return this->CMake->HasScriptModeExitCode()
+    ? this->CMake->GetScriptModeExitCode()
+    : res;
 }
 
 // run a specific script

Source/cmMakefile.cxx

@@ -525,7 +525,8 @@ bool cmMakefile::ExecuteCommand(cmListFileFunction const& lff,
           this->IssueMessage(MessageType::FATAL_ERROR, error);
         }
         result = false;
-        if (this->GetCMakeInstance()->GetWorkingMode() != cmake::NORMAL_MODE) {
+        if (this->GetCMakeInstance()->GetCommandFailureAction() ==
+            cmake::CommandFailureAction::FATAL_ERROR) {
           cmSystemTools::SetFatalErrorOccurred();
         }
       }

Source/cmake.cxx

@@ -659,7 +659,8 @@ bool cmake::SetCacheArgs(std::vector<std::string> const& args)
     GetProjectCommandsInScriptMode(state->GetState());
     // Documented behavior of CMAKE{,_CURRENT}_{SOURCE,BINARY}_DIR is to be
     // set to $PWD for -P mode.
-    state->SetWorkingMode(SCRIPT_MODE);
+    state->SetWorkingMode(SCRIPT_MODE,
+                          cmake::CommandFailureAction::FATAL_ERROR);
     state->SetHomeDirectory(cmSystemTools::GetLogicalWorkingDirectory());
     state->SetHomeOutputDirectory(cmSystemTools::GetLogicalWorkingDirectory());
     state->ReadListFile(args, path);
@@ -1561,7 +1562,8 @@ void cmake::SetArgs(std::vector<std::string> const& args)
         presetsGraph.PrintAllPresets();
       }
 
-      this->SetWorkingMode(WorkingMode::HELP_MODE);
+      this->SetWorkingMode(WorkingMode::HELP_MODE,
+                           cmake::CommandFailureAction::FATAL_ERROR);
       return;
     }
 

Source/cmake.h

@@ -129,6 +129,16 @@ public:
     FIND_PACKAGE_MODE
   };
 
+  enum class CommandFailureAction
+  {
+    // When a command fails to execute, treat it as a fatal error.
+    FATAL_ERROR,
+
+    // When a command fails to execute, continue execution, but set the exit
+    // code accordingly.
+    EXIT_CODE,
+  };
+
   using TraceFormat = cmTraceEnums::TraceOutputFormat;
 
   struct GeneratorInfo
@@ -441,8 +451,18 @@ public:
   //! Do all the checks before running configure
   int DoPreConfigureChecks();
 
-  void SetWorkingMode(WorkingMode mode) { this->CurrentWorkingMode = mode; }
-  WorkingMode GetWorkingMode() { return this->CurrentWorkingMode; }
+  void SetWorkingMode(WorkingMode mode, CommandFailureAction policy)
+  {
+    this->CurrentWorkingMode = mode;
+    this->CurrentCommandFailureAction = policy;
+  }
+
+  WorkingMode GetWorkingMode() const { return this->CurrentWorkingMode; }
+
+  CommandFailureAction GetCommandFailureAction() const
+  {
+    return this->CurrentCommandFailureAction;
+  }
 
   //! Debug the try compile stuff by not deleting the files
   bool GetDebugTryCompile() const { return this->DebugTryCompile; }
@@ -780,6 +800,8 @@ private:
   std::string CMakeWorkingDirectory;
   ProgressCallbackType ProgressCallback;
   WorkingMode CurrentWorkingMode = NORMAL_MODE;
+  CommandFailureAction CurrentCommandFailureAction =
+    CommandFailureAction::FATAL_ERROR;
   bool DebugOutput = false;
   bool DebugFindOutput = false;
   bool Trace = false;

Source/cmakemain.cxx

@@ -396,6 +396,9 @@ int do_cmake(int ac, char const* const* av)
       mode = cmState::FindPackage;
       break;
   }
+  auto const failurePolicy = workingMode == cmake::NORMAL_MODE
+    ? cmake::CommandFailureAction::EXIT_CODE
+    : cmake::CommandFailureAction::FATAL_ERROR;
   cmake cm(role, mode);
   cm.SetHomeDirectory("");
   cm.SetHomeOutputDirectory("");
@@ -406,7 +409,7 @@ int do_cmake(int ac, char const* const* av)
   cm.SetProgressCallback([&cm](std::string const& msg, float prog) {
     cmakemainProgressCallback(msg, prog, &cm);
   });
-  cm.SetWorkingMode(workingMode);
+  cm.SetWorkingMode(workingMode, failurePolicy);
 
   int res = cm.Run(parsedArgs, view_only);
   if (list_cached || list_all_cached) {
@@ -988,7 +991,8 @@ int do_install(int ac, char const* const* av)
         cm.SetHomeDirectory("");
         cm.SetHomeOutputDirectory("");
         cm.SetDebugOutputOn(verbose);
-        cm.SetWorkingMode(cmake::SCRIPT_MODE);
+        cm.SetWorkingMode(cmake::SCRIPT_MODE,
+                          cmake::CommandFailureAction::FATAL_ERROR);
         ret_ = int(bool(cm.Run(cmd)));
       }
     }

Tests/RunCMake/CMakeLists.txt

@@ -672,6 +672,7 @@ add_RunCMake_test(ctest_update)
 add_RunCMake_test(ctest_upload)
 add_RunCMake_test(ctest_environment)
 add_RunCMake_test(ctest_empty_binary_directory)
+add_RunCMake_test(ctest_exit)
 add_RunCMake_test(ctest_fixtures)
 if(CMAKE_GENERATOR MATCHES "Make|Ninja")
   add_RunCMake_test(ctest_instrumentation)

Tests/RunCMake/ctest_exit/3-result.txt

@@ -0,0 +1 @@
+3

Tests/RunCMake/ctest_exit/3-stderr.txt

@@ -0,0 +1,3 @@
+^CMake Error at [^
+]*/Tests/RunCMake/ctest_exit/exit1.cmake:2 \(message\):
+  send error$

Tests/RunCMake/ctest_exit/7-result.txt

@@ -0,0 +1 @@
+7

Tests/RunCMake/ctest_exit/7-stderr.txt

@@ -0,0 +1,3 @@
+^CMake Error at [^
+]*/Tests/RunCMake/ctest_exit/exit1.cmake:2 \(message\):
+  send error$

Tests/RunCMake/ctest_exit/RunCMakeTest.cmake

@@ -0,0 +1,45 @@
+include(RunCMake)
+
+message(STATUS "Multiple -S options:")
+
+run_cmake_command(3 ${CMAKE_CTEST_COMMAND} -V
+  -S ${RunCMake_SOURCE_DIR}/exit1.cmake
+  -S ${RunCMake_SOURCE_DIR}/exit2.cmake
+  )
+
+run_cmake_command(7 ${CMAKE_CTEST_COMMAND} -V
+  -S ${RunCMake_SOURCE_DIR}/exit4.cmake
+  -S ${RunCMake_SOURCE_DIR}/exit1.cmake
+  -S ${RunCMake_SOURCE_DIR}/exit2.cmake
+  )
+
+message(STATUS "Multiple -SP options:")
+
+run_cmake_command(3 ${CMAKE_CTEST_COMMAND} -V
+  -SP ${RunCMake_SOURCE_DIR}/exit1.cmake
+  -SP ${RunCMake_SOURCE_DIR}/exit2.cmake
+  )
+
+run_cmake_command(7 ${CMAKE_CTEST_COMMAND} -V
+  -SP ${RunCMake_SOURCE_DIR}/exit4.cmake
+  -SP ${RunCMake_SOURCE_DIR}/exit1.cmake
+  -SP ${RunCMake_SOURCE_DIR}/exit2.cmake
+  )
+
+message(STATUS "Mixed -S and -SP options:")
+
+run_cmake_command(7 ${CMAKE_CTEST_COMMAND} -V
+  -S ${RunCMake_SOURCE_DIR}/exit4.cmake
+  -SP ${RunCMake_SOURCE_DIR}/exit1.cmake
+  -S ${RunCMake_SOURCE_DIR}/exit2.cmake
+  )
+
+message(STATUS "ctest_run_script:")
+
+configure_file(
+  ${RunCMake_SOURCE_DIR}/test.cmake.in
+  ${RunCMake_BINARY_DIR}/test.cmake @ONLY)
+
+run_cmake_command(Script ${CMAKE_CTEST_COMMAND} -V
+  -S ${RunCMake_BINARY_DIR}/test.cmake
+  )

Tests/RunCMake/ctest_exit/Script-result.txt

@@ -0,0 +1 @@
+(-1|255)

Tests/RunCMake/ctest_exit/Script-stderr.txt

@@ -0,0 +1,8 @@
+^CMake Error at [^
+]*/Tests/RunCMake/ctest_exit/exit1.cmake:2 \(message\):
+  send error
+
+
+CMake Error at [^
+]*/Tests/RunCMake/ctest_exit/exit1.cmake:2 \(message\):
+  send error$

Tests/RunCMake/ctest_exit/exit1.cmake

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.10)
+message(SEND_ERROR "send error")
+cmake_language(EXIT 1)

Tests/RunCMake/ctest_exit/exit2.cmake

@@ -0,0 +1,2 @@
+cmake_minimum_required(VERSION 3.10)
+cmake_language(EXIT 2)

Tests/RunCMake/ctest_exit/exit4.cmake

@@ -0,0 +1,2 @@
+cmake_minimum_required(VERSION 3.10)
+cmake_language(EXIT 4)

Tests/RunCMake/ctest_exit/test.cmake.in

@@ -0,0 +1,23 @@
+cmake_minimum_required(VERSION 3.10)
+
+ctest_run_script(
+  "@RunCMake_SOURCE_DIR@/exit1.cmake"
+  "@RunCMake_SOURCE_DIR@/exit4.cmake"
+  RETURN_VALUE ret
+  )
+
+if(NOT ret EQUAL 4)
+  message(FATAL_ERROR "Expected ret == 4, got ${ret}")
+endif()
+
+unset(ret)
+
+ctest_run_script(NEW_PROCESS
+  "@RunCMake_SOURCE_DIR@/exit1.cmake"
+  "@RunCMake_SOURCE_DIR@/exit4.cmake"
+  RETURN_VALUE ret
+  )
+
+if(NOT ret EQUAL 4)
+  message(FATAL_ERROR "Expected ret == 4, got ${ret}")
+endif()