/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying file LICENSE.rst or https://cmake.org/licensing for details. */ #pragma once #include "cmConfigure.h" // IWYU pragma: keep #include #include #include #include #include #include #include #include #include #include #include "cmDuration.h" #include "cmProcessOutput.h" class cmake; class cmGeneratedFileStream; class cmInstrumentation; class cmMakefile; class cmValue; class cmXMLWriter; struct cmCTestTestOptions; /** \class cmCTest * \brief Represents a ctest invocation. * * This class represents a ctest invocation. It is the top level class when * running ctest. * */ class cmCTest { public: using Encoding = cmProcessOutput::Encoding; /** Enumerate parts of the testing and submission process. */ enum Part { PartStart, PartUpdate, PartConfigure, PartBuild, PartTest, PartCoverage, PartMemCheck, PartSubmit, PartNotes, PartExtraFiles, PartUpload, PartDone, PartCount // Update names in constructor when adding a part }; /** Get a testing part id from its string name. Returns PartCount if the string does not name a valid part. */ Part GetPartFromName(std::string const& name); /** Process Command line arguments */ int Run(std::vector const& args); /** Initialize a dashboard run in the given build tree. */ void Initialize(std::string const& binary_dir); bool CreateNewTag(bool quiet); bool ReadExistingTag(bool quiet); /** * Process the dashboard client steps. */ int ProcessSteps(); /** * A utility function that returns the nightly time */ struct tm* GetNightlyTime(std::string const& str, bool tomorrowtag); /** * Is the tomorrow tag set? */ bool GetTomorrowTag() const; /** * Try to run tests of the project */ int TestDirectory(bool memcheck); /** what is the configuration type, e.g. Debug, Release etc. */ std::string const& GetConfigType(); cmDuration GetTimeOut() const; void SetTimeOut(cmDuration t); cmDuration GetGlobalTimeout() const; /** how many test to run at the same time */ cm::optional GetParallelLevel() const; void SetParallelLevel(cm::optional level); unsigned long GetTestLoad() const; void SetTestLoad(unsigned long); /** * Check if CTest file exists */ bool CTestFileExists(std::string const& filename); bool AddIfExists(Part part, std::string const& file); /** * Set the cmake test */ bool SetTest(std::string const&, bool report = true); /** * Set the cmake test mode (experimental, nightly, continuous). */ void SetTestModel(int mode); int GetTestModel() const; std::string GetTestModelString() const; std::string GetTestGroupString() const; static int GetTestModelFromString(std::string const& str); static std::string CleanString(std::string const& str, std::string::size_type spos = 0); std::string GetCTestConfiguration(std::string const& name); void SetCTestConfiguration(char const* name, std::string const& value, bool suppress = false); void EmptyCTestConfiguration(); std::string GetSubmitURL(); /** * constructor and destructor */ cmCTest(); ~cmCTest(); cmCTest(cmCTest const&) = delete; cmCTest& operator=(cmCTest const&) = delete; /** Set the notes files to be created. */ void SetNotesFiles(std::string const& notes); void PopulateCustomVector(cmMakefile* mf, std::string const& definition, std::vector& vec); void PopulateCustomInteger(cmMakefile* mf, std::string const& def, int& val); /** Get the current time as string */ std::string CurrentTime(); /** tar/gzip and then base 64 encode a file */ std::string Base64GzipEncodeFile(std::string const& file); /** base64 encode a file */ std::string Base64EncodeFile(std::string const& file); void SetTimeLimit(cmValue val); cmDuration GetElapsedTime() const; /** * Return the time remaining that the script is allowed to run in * seconds if the user has set the variable CTEST_TIME_LIMIT. If that has * not been set it returns a very large duration. */ cmDuration GetRemainingTimeAllowed() const; static cmDuration MaxDuration(); /** * Open file in the output directory and set the stream */ bool OpenOutputFile(std::string const& path, std::string const& name, cmGeneratedFileStream& stream, bool compress = false); /** Should we only show what we would do? */ bool GetShowOnly(); bool GetOutputAsJson(); int GetOutputAsJsonVersion(); bool ShouldUseHTTP10() const; bool ShouldPrintLabels() const; bool ShouldCompressTestOutput(); bool CompressString(std::string& str); bool GetStopOnFailure() const; void SetStopOnFailure(bool stop); std::chrono::system_clock::time_point GetStopTime() const; void SetStopTime(std::string const& time); /** Used for parallel ctest job scheduling */ std::string GetScheduleType() const; void SetScheduleType(std::string const& type); cm::optional GetRandomSeed() const; /** The max output width */ int GetMaxTestNameWidth() const; void SetMaxTestNameWidth(int w); /** * Run a single executable command and put the stdout and stderr * in output. * * If verbose is false, no user-viewable output from the program * being run will be generated. * * If timeout is specified, the command will be terminated after * timeout expires. Timeout is specified in seconds. * * Argument retVal should be a pointer to the location where the * exit code will be stored. If the retVal is not specified and * the program exits with a code other than 0, then the this * function will return false. */ bool RunCommand(std::vector const& args, std::string* stdOut, std::string* stdErr, int* retVal = nullptr, char const* dir = nullptr, cmDuration timeout = cmDuration::zero(), Encoding encoding = cmProcessOutput::Auto); /** * Clean/make safe for xml the given value such that it may be used as * one of the key fields by CDash when computing the buildid. */ static std::string SafeBuildIdField(std::string const& value); /** Start CTest XML output file */ void StartXML(cmXMLWriter& xml, cmake* cm, bool append); /** End CTest XML output file */ void EndXML(cmXMLWriter& xml); /** * Run command specialized for make and configure. Returns process status * and retVal is return value or exception. */ bool RunMakeCommand(std::string const& command, std::string& output, int* retVal, char const* dir, cmDuration timeout, std::ostream& ofs, Encoding encoding = cmProcessOutput::Auto); /** Return the current tag */ std::string GetCurrentTag(); /** Get the path to the build tree */ std::string GetBinaryDir(); /** * Get the short path to the file. * * This means if the file is in binary or * source directory, it will become /.../relative/path/to/file */ std::string GetShortPathToFile(std::string const& fname); enum { UNKNOWN = -1, EXPERIMENTAL = 0, NIGHTLY = 1, CONTINUOUS = 2, }; /** provide some more detailed info on the return code for ctest */ enum { UPDATE_ERRORS = 0x01, CONFIGURE_ERRORS = 0x02, BUILD_ERRORS = 0x04, TEST_ERRORS = 0x08, MEMORY_ERRORS = 0x10, COVERAGE_ERRORS = 0x20, SUBMIT_ERRORS = 0x40 }; /** Are we producing XML */ bool GetProduceXML(); void SetProduceXML(bool v); /** * Set the CTest variable from CMake variable */ bool SetCTestConfigurationFromCMakeVariable(cmMakefile* mf, char const* dconfig, std::string const& cmake_var, bool suppress = false); /** * Set CMake variables from CTest Options */ void SetCMakeVariables(cmMakefile& mf); /** Decode a URL to the original string. */ static std::string DecodeURL(std::string const&); /** * Add overwrite to ctest configuration. * * The format is key=value */ void AddCTestConfigurationOverwrite(std::string const& encstr); /** Create XML file that contains all the notes specified */ int GenerateNotesFile(cmake* cm, std::vector const& files); /** Create XML file to indicate that build is complete */ int GenerateDoneFile(); /** Submit extra files to the server */ bool SubmitExtraFiles(std::string const& files); bool SubmitExtraFiles(std::vector const& files); /** Set the output log file name */ void SetOutputLogFileName(std::string const& name); /** Set the output JUnit file name */ void SetOutputJUnitFileName(std::string const& name); /** Set the visual studio or Xcode config type */ void SetConfigType(std::string const& ct); /** Various log types */ enum LogType { DEBUG, OUTPUT, HANDLER_OUTPUT, HANDLER_PROGRESS_OUTPUT, HANDLER_TEST_PROGRESS_OUTPUT, HANDLER_VERBOSE_OUTPUT, WARNING, ERROR_MESSAGE, }; /** Add log to the output */ void Log(LogType logType, std::string msg, bool suppress = false); /** Color values */ enum class Color { CLEAR_COLOR = 0, RED = 31, GREEN = 32, YELLOW = 33, BLUE = 34 }; /** Get color code characters for a specific color */ std::string GetColorCode(Color color) const; /** The Build ID is assigned by CDash */ void SetBuildID(std::string const& id); std::string GetBuildID() const; /** Add file to be submitted */ void AddSubmitFile(Part part, std::string const& name); std::vector const& GetSubmitFiles(Part part) const; void ClearSubmitFiles(Part part); /** * Read the custom configuration files and apply them to the current ctest */ void ReadCustomConfigurationFileTree(std::string const& dir, cmMakefile* mf); std::vector& GetInitialCommandLineArguments(); /** Set the group to submit to */ void SetSpecificGroup(char const* group); char const* GetSpecificGroup(); void SetFailover(bool failover); bool GetFailover() const; bool GetTestProgressOutput() const; bool GetVerbose() const; bool GetExtraVerbose() const; bool StartResultingXML(Part part, char const* name, int submitIndex, cmGeneratedFileStream& xofs); bool StartLogFile(char const* name, int submitIndex, cmGeneratedFileStream& xofs); void ConvertInstrumentationSnippetsToXML(cmXMLWriter& xml, std::string const& subdir); bool ConvertInstrumentationJSONFileToXML(std::string const& fpath, cmXMLWriter& xml); void AddSiteProperties(cmXMLWriter& xml, cmake* cm); bool GetInteractiveDebugMode() const; bool GetLabelSummary() const; bool GetSubprojectSummary() const; std::string GetCostDataFile(); bool GetOutputTestOutputOnTestFailure() const; std::map const& GetDefinitions() const; /** Return the number of times a test should be run */ int GetRepeatCount() const; enum class Repeat { Never, UntilFail, UntilPass, AfterTimeout, }; Repeat GetRepeatMode() const; enum class NoTests { Legacy, Error, Ignore }; NoTests GetNoTestsMode() const; void GenerateSubprojectsOutput(cmXMLWriter& xml); std::vector GetLabelsForSubprojects(); /** Reread the configuration file */ bool UpdateCTestConfiguration(); cmCTestTestOptions const& GetTestOptions() const; std::vector GetCommandLineHttpHeaders() const; cmInstrumentation& GetInstrumentation(); bool GetUseVerboseInstrumentation() const; private: int GenerateNotesFile(cmake* cm, std::string const& files); void BlockTestErrorDiagnostics(); /** parse the option after -D and convert it into the appropriate steps */ bool AddTestsForDashboardType(std::string const& targ); /** read as "emit an error message for an unknown -D value" */ void ErrorMessageUnknownDashDValue(std::string const& val); /** add a variable definition from a command line -D value */ bool AddVariableDefinition(std::string const& arg); /** set command line arguments read from a test preset */ bool SetArgsFromPreset(std::string const& presetName, bool listPresets); #if !defined(_WIN32) /** returns true iff the console supports progress output */ static bool ConsoleIsNotDumb(); #endif /** returns true iff the console supports progress output */ static bool ProgressOutputSupportedByConsole(); /** returns true iff the console supports colored output */ static bool ColoredOutputSupportedByConsole(); /** Create note from files. */ int GenerateCTestNotesOutput(cmXMLWriter& xml, cmake* cm, std::vector const& files); /** Check if the argument is the one specified */ static bool CheckArgument(std::string const& arg, cm::string_view varg1, char const* varg2 = nullptr); int RunCMakeAndTest(); int RunScripts(std::vector> const& scripts); int ExecuteTests(std::vector const& args); struct Private; std::unique_ptr Impl; }; #define cmCTestLog(ctSelf, logType, msg) \ do { \ std::ostringstream cmCTestLog_msg; \ cmCTestLog_msg << msg; \ (ctSelf)->Log(cmCTest::logType, cmCTestLog_msg.str()); \ } while (false) #define cmCTestOptionalLog(ctSelf, logType, msg, suppress) \ do { \ std::ostringstream cmCTestLog_msg; \ cmCTestLog_msg << msg; \ (ctSelf)->Log(cmCTest::logType, cmCTestLog_msg.str(), suppress); \ } while (false)