echo/CMake
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ctest: Add support for writing test results in JUnit XML format

Browse Source 
Addresses #18654
This commit is contained in:
Zack Galbreath 2021-04-23 09:38:43 +10:00
parent eeb771e4d6
commit 25bf514447
18 changed files with 271 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Help/command/ctest_test.rst
View File

@ -25,6 +25,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`.
[RETURN_VALUE <result-var>]
[CAPTURE_CMAKE_ERROR <result-var>]
[REPEAT <mode>:<n>]
[OUTPUT_JUNIT <file>]
[QUIET]
)
@ -150,6 +151,15 @@ The options are:
Store in the ``<result-var>`` variable -1 if there are any errors running
the command and prevent ctest from returning non-zero if an error occurs.
``OUTPUT_JUNIT``
.. versionadded:: 3.21
Write test results to ``<file>`` in JUnit XML format. If ``<file>`` is a
relative path it will be placed in the build directory. If ``<file>>``
already exists it will be overwritten. Note that the resulting JUnit XML
file is **not** uploaded to CDash because it would be redundant with
CTest's ``Test.xml`` file.
``QUIET``
.. versionadded:: 3.3

6
Help/manual/ctest.1.rst
View File

@ -134,6 +134,12 @@ Options
This option tells CTest to write all its output to a ``<file>`` log file.
``--output-junit <file>``
Write test results in JUnit format.
This option tells CTest to write test results to a ``<file>`` JUnit XML file.
If ``<file>`` already exists it will be overwritten.
``-N,--show-only[=<format>]``
Disable actual execution of tests.

5
Help/release/dev/ctest-output-junit.rst Normal file
View File

@ -0,0 +1,5 @@
ctest-output-junit
------------------
* :manual:`ctest(1)` gained a ``--output-junit`` option to write test results
to a JUnit XML file.

2
Source/CTest/cmCTestMemCheckCommand.cxx
View File

@ -14,7 +14,7 @@ void cmCTestMemCheckCommand::BindArguments()
this->Bind("DEFECT_COUNT"_s, this->DefectCount);
}
cmCTestGenericHandler* cmCTestMemCheckCommand::InitializeActualHandler()
cmCTestTestHandler* cmCTestMemCheckCommand::InitializeActualHandler()
{
cmCTestMemCheckHandler* handler = this->CTest->GetMemCheckHandler();
handler->Initialize();

3
Source/CTest/cmCTestMemCheckCommand.h
View File

@ -13,6 +13,7 @@
#include "cmCommand.h"
class cmCTestGenericHandler;
class cmCTestTestHandler;
/** \class cmCTestMemCheck
* \brief Run a ctest script
@ -36,7 +37,7 @@ public:
protected:
void BindArguments() override;
cmCTestGenericHandler* InitializeActualHandler() override;
cmCTestTestHandler* InitializeActualHandler() override;
void ProcessAdditionalValues(cmCTestGenericHandler* handler) override;

10
Source/CTest/cmCTestTestCommand.cxx
View File

@ -9,7 +9,6 @@
#include <cmext/string_view>
#include "cmCTest.h"
#include "cmCTestGenericHandler.h"
#include "cmCTestTestHandler.h"
#include "cmDuration.h"
#include "cmMakefile.h"
@ -36,6 +35,7 @@ void cmCTestTestCommand::BindArguments()
this->Bind("TEST_LOAD"_s, this->TestLoad);
this->Bind("RESOURCE_SPEC_FILE"_s, this->ResourceSpecFile);
this->Bind("STOP_ON_FAILURE"_s, this->StopOnFailure);
this->Bind("OUTPUT_JUNIT"_s, this->OutputJUnit);
}
cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
@ -60,7 +60,7 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
this->ResourceSpecFile = *resourceSpecFile;
}
cmCTestGenericHandler* handler = this->InitializeActualHandler();
cmCTestTestHandler* handler = this->InitializeActualHandler();
if (!this->Start.empty() || !this->End.empty() || !this->Stride.empty()) {
handler->SetOption(
"TestsToRunInformation",
@ -140,11 +140,15 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
*labelsForSubprojects, this->Quiet);
}
if (!this->OutputJUnit.empty()) {
handler->SetJUnitXMLFileName(this->OutputJUnit);
}
handler->SetQuiet(this->Quiet);
return handler;
}
cmCTestGenericHandler* cmCTestTestCommand::InitializeActualHandler()
cmCTestTestHandler* cmCTestTestCommand::InitializeActualHandler()
{
cmCTestTestHandler* handler = this->CTest->GetTestHandler();
handler->Initialize();

4
Source/CTest/cmCTestTestCommand.h
View File

@ -13,6 +13,7 @@
#include "cmCommand.h"
class cmCTestGenericHandler;
class cmCTestTestHandler;
/** \class cmCTestTest
* \brief Run a ctest script
@ -40,7 +41,7 @@ public:
protected:
void BindArguments() override;
virtual cmCTestGenericHandler* InitializeActualHandler();
virtual cmCTestTestHandler* InitializeActualHandler();
cmCTestGenericHandler* InitializeHandler() override;
std::string Start;
@ -59,5 +60,6 @@ protected:
std::string StopTime;
std::string TestLoad;
std::string ResourceSpecFile;
std::string OutputJUnit;
bool StopOnFailure = false;
};

130
Source/CTest/cmCTestTestHandler.cxx
View File

@ -42,6 +42,7 @@
#include "cmStateSnapshot.h"
#include "cmStringAlgorithms.h"
#include "cmSystemTools.h"
#include "cmTimestamp.h"
#include "cmWorkingDirectory.h"
#include "cmXMLWriter.h"
#include "cmake.h"
@ -299,6 +300,9 @@ cmCTestTestHandler::cmCTestTestHandler()
this->LogFile = nullptr;
// Support for JUnit XML output.
this->JUnitXMLFileName = "";
// regex to detect <DartMeasurement>...</DartMeasurement>
this->DartStuff.compile("(<DartMeasurement.*/DartMeasurement[a-zA-Z]*>)");
// regex to detect each individual <DartMeasurement>...</DartMeasurement>
@ -456,6 +460,10 @@ int cmCTestTestHandler::ProcessHandler()
return 1;
}
if (!this->WriteJUnitXML()) {
return 1;
}
if (!this->PostProcessHandler()) {
this->LogFile = nullptr;
return -1;
@ -2457,3 +2465,125 @@ bool cmCTestTestHandler::cmCTestTestResourceRequirement::operator!=(
{
return !(*this == other);
}
void cmCTestTestHandler::SetJUnitXMLFileName(const std::string& filename)
{
this->JUnitXMLFileName = filename;
}
bool cmCTestTestHandler::WriteJUnitXML()
{
if (this->JUnitXMLFileName.empty()) {
return true;
}
// Open new XML file for writing.
cmGeneratedFileStream xmlfile;
xmlfile.SetTempExt("tmp");
xmlfile.Open(this->JUnitXMLFileName);
if (!xmlfile) {
cmCTestLog(this->CTest, ERROR_MESSAGE,
"Problem opening file: " << this->JUnitXMLFileName
<< std::endl);
return false;
}
cmXMLWriter xml(xmlfile);
// Iterate over the test results to get the number of tests that
// passed, failed, etc.
auto num_tests = 0;
auto num_passed = 0;
auto num_failed = 0;
auto num_notrun = 0;
auto num_disabled = 0;
SetOfTests resultsSet(this->TestResults.begin(), this->TestResults.end());
for (cmCTestTestResult const& result : resultsSet) {
num_tests++;
if (result.Status == cmCTestTestHandler::COMPLETED) {
num_passed++;
} else if (result.Status == cmCTestTestHandler::NOT_RUN) {
if (result.CompletionStatus == "Disabled") {
num_disabled++;
} else {
num_notrun++;
}
} else {
num_failed++;
}
}
// Write <testsuite> element.
xml.StartDocument();
xml.StartElement("testsuite");
xml.Attribute("name",
cmCTest::SafeBuildIdField(
this->CTest->GetCTestConfiguration("BuildName")));
xml.BreakAttributes();
xml.Attribute("tests", num_tests);
xml.Attribute("failures", num_failed);
// CTest disabled => JUnit disabled
xml.Attribute("disabled", num_disabled);
// Otherwise, CTest notrun => JUnit skipped.
// The distinction between JUnit disabled vs. skipped is that
// skipped tests can have a message associated with them
// (why the test was skipped).
xml.Attribute("skipped", num_notrun);
xml.Attribute("hostname", this->CTest->GetCTestConfiguration("Site"));
xml.Attribute(
"time",
std::chrono::duration_cast<std::chrono::seconds>(this->ElapsedTestingTime)
.count());
const std::time_t start_test_time_t =
std::chrono::system_clock::to_time_t(this->StartTestTime);
cmTimestamp cmts;
xml.Attribute("timestamp",
cmts.CreateTimestampFromTimeT(start_test_time_t,
"%Y-%m-%dT%H:%M:%S", false));
// Write <testcase> elements.
for (cmCTestTestResult const& result : resultsSet) {
xml.StartElement("testcase");
xml.Attribute("name", result.Name);
xml.Attribute("classname", result.Name);
xml.Attribute("time", result.ExecutionTime.count());
std::string status;
if (result.Status == cmCTestTestHandler::COMPLETED) {
status = "run";
} else if (result.Status == cmCTestTestHandler::NOT_RUN) {
if (result.CompletionStatus == "Disabled") {
status = "disabled";
} else {
status = "notrun";
}
} else {
status = "fail";
}
xml.Attribute("status", status);
if (status == "notrun") {
xml.StartElement("skipped");
xml.Attribute("message", result.CompletionStatus);
xml.EndElement(); // </skipped>
} else if (status == "fail") {
xml.StartElement("failure");
xml.Attribute("message", result.Reason);
xml.EndElement(); // </failure>
}
// Note: compressed test output is unconditionally disabled when
// --output-junit is specified.
xml.Element("system-out", result.Output);
xml.EndElement(); // </testcase>
}
xml.EndElement(); // </testsuite>
xml.EndDocument();
return true;
}

10
Source/CTest/cmCTestTestHandler.h
View File

@ -209,6 +209,9 @@ public:
using ListOfTests = std::vector<cmCTestTestProperties>;
// Support for writing test results in JUnit XML format.
void SetJUnitXMLFileName(const std::string& id);
protected:
using SetOfTests =
std::set<cmCTestTestHandler::cmCTestTestResult, cmCTestTestResultLess>;
@ -274,6 +277,11 @@ private:
*/
virtual void GenerateDartOutput(cmXMLWriter& xml);
/**
* Write test results in JUnit XML format
*/
bool WriteJUnitXML();
void PrintLabelOrSubprojectSummary(bool isSubProject);
/**
@ -354,4 +362,6 @@ private:
cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never;
int RepeatCount = 1;
bool RerunFailed;
std::string JUnitXMLFileName;
};

11
Source/cmCTest.cxx
View File

@ -2069,6 +2069,17 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
}
i++;
this->Impl->TestDir = std::string(args[i]);
} else if (this->CheckArgument(arg, "--output-junit"_s)) {
if (i >= args.size() - 1) {
errormsg = "'--output-junit' requires an argument";
return false;
}
i++;
this->Impl->TestHandler.SetJUnitXMLFileName(std::string(args[i]));
// Turn test output compression off.
// This makes it easier to include test output in the resulting
// JUnit XML report.
this->Impl->CompressTestOutput = false;
}
cm::string_view noTestsPrefix = "--no-tests=";

1
Source/ctest.cxx
View File

@ -50,6 +50,7 @@ static const char* cmDocumentationOptions[][2] = {
"given number of jobs." },
{ "-Q,--quiet", "Make ctest quiet." },
{ "-O <file>, --output-log <file>", "Output to log file" },
{ "--output-junit <file>", "Output test results to JUnit XML file." },
{ "-N,--show-only[=format]",
"Disable actual execution of tests. The optional 'format' defines the "
"format of the test information and can be 'human' for the current text "

19
Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
View File

@ -397,3 +397,22 @@ function(run_testDir)
run_cmake_command(testDir ${CMAKE_CTEST_COMMAND} --test-dir "${RunCMake_TEST_BINARY_DIR}/sub")
endfunction()
run_testDir()
# Test --output-junit
function(run_output_junit)
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/output-junit)
set(RunCMake_TEST_NO_CLEAN 1)
file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "
add_test(test1 \"${CMAKE_COMMAND}\" -E false)
add_test(test2 \"${CMAKE_COMMAND}\" -E echo \"hello world\")
add_test(test3 \"${CMAKE_COMMAND}\" -E true)
set_tests_properties(test3 PROPERTIES DISABLED \"ON\")
add_test(test4 \"${CMAKE_COMMAND}/doesnt_exist\")
add_test(test5 \"${CMAKE_COMMAND}\" -E echo \"please skip\")
set_tests_properties(test5 PROPERTIES SKIP_REGULAR_EXPRESSION \"please skip\")
")
run_cmake_command(output-junit ${CMAKE_CTEST_COMMAND} --output-junit "${RunCMake_TEST_BINARY_DIR}/junit.xml")
endfunction()
run_output_junit()

36
Tests/RunCMake/CTestCommandLine/output-junit-check.cmake Normal file
View File

@ -0,0 +1,36 @@
file(GLOB junit_xml_file "${RunCMake_TEST_BINARY_DIR}/junit.xml")
if(junit_xml_file)
file(READ "${junit_xml_file}" junit_xml LIMIT 4096)
if(NOT "${junit_xml}" MATCHES "tests=\"5\"")
set(RunCMake_TEST_FAILED "tests=\"5\" not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "failures=\"1\"")
set(RunCMake_TEST_FAILED "failures=\"1\" not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "disabled=\"1\"")
set(RunCMake_TEST_FAILED "disabled=\"1\" not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "skipped=\"2\"")
set(RunCMake_TEST_FAILED "skipped=\"2\" not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "<system-out>hello world")
set(RunCMake_TEST_FAILED "<system-out>hello world not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "<system-out>Disabled")
set(RunCMake_TEST_FAILED "<system-out>Disabled not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "<skipped message=\"Unable to find executable\"/>")
set(RunCMake_TEST_FAILED "<skipped message=\"Unable to find executable\"/> not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "<system-out>Unable to find executable:")
set(RunCMake_TEST_FAILED "<system-out>Unable to find executable: not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "<skipped message=\"SKIP_REGULAR_EXPRESSION_MATCHED\"/>")
set(RunCMake_TEST_FAILED "<skipped message=\"SKIP_REGULAR_EXPRESSION_MATCHED\"/> not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "<system-out>please skip")
set(RunCMake_TEST_FAILED "<system-out>please skip not found when expected")
endif()
else()
set(RunCMake_TEST_FAILED "junit.xml not found")
endif()

1
Tests/RunCMake/CTestCommandLine/output-junit-result.txt Normal file
View File

@ -0,0 +1 @@
8

1
Tests/RunCMake/CTestCommandLine/output-junit-stderr.txt Normal file
View File

@ -0,0 +1 @@
Unable to find executable: .*doesnt_exist

24
Tests/RunCMake/ctest_test/OutputJUnit-check.cmake Normal file
View File

@ -0,0 +1,24 @@
file(GLOB junit_xml_file "${RunCMake_TEST_BINARY_DIR}/junit.xml")
if(junit_xml_file)
file(READ "${junit_xml_file}" junit_xml LIMIT 4096)
if(NOT "${junit_xml}" MATCHES "tests=\"1\"")
set(RunCMake_TEST_FAILED "tests=\"1\" not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "failures=\"0\"")
set(RunCMake_TEST_FAILED "failures=\"0\" not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "disabled=\"0\"")
set(RunCMake_TEST_FAILED "disabled=\"0\" not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "skipped=\"0\"")
set(RunCMake_TEST_FAILED "skipped=\"0\" not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "<testcase name=\"RunCMakeVersion\" classname=\"RunCMakeVersion\"")
set(RunCMake_TEST_FAILED "RunCMakeVersion not found when expected")
endif()
if(NOT "${junit_xml}" MATCHES "<system-out>cmake version")
set(RunCMake_TEST_FAILED "<system-out>cmake version not found when expected")
endif()
else()
set(RunCMake_TEST_FAILED "junit.xml not found")
endif()

3
Tests/RunCMake/ctest_test/RunCMakeTest.cmake
View File

@ -146,3 +146,6 @@ set_property(TEST RunCMakeVersion PROPERTY ENVIRONMENT "ENV1=env1;ENV2=env2")
run_ctest(TestEnvironment)
endfunction()
run_environment()
# test for OUTPUT_JUNIT
run_ctest_test(OutputJUnit OUTPUT_JUNIT junit.xml REPEAT UNTIL_FAIL:2)

1
Utilities/IWYU/mapping.imp
View File

@ -21,6 +21,7 @@
{ include: [ "<wctype.h>", public, "<cwctype>", public ] },
# HACK: check whether this can be removed with next iwyu release.
{ include: [ "<bits/cxxabi_forced.h>", private, "<ctime>", public ] },
{ include: [ "<bits/shared_ptr.h>", private, "<memory>", public ] },
{ include: [ "<bits/std_function.h>", private, "<functional>", public ] },
{ include: [ "<bits/refwrap.h>", private, "<functional>", public ] },