Install: Add parallel installation option

Fixes: #26000
This commit is contained in:
Martin Duffy 2024-06-19 13:22:30 -04:00 committed by Brad King
parent 73f368b3c1
commit 159ba027b9
20 changed files with 367 additions and 41 deletions

View File

@ -38,6 +38,13 @@ are executed in order during installation.
The environment variable :envvar:`CMAKE_INSTALL_MODE` can override the
default copying behavior of ``install()``.
.. versionchanged:: 3.31
Projects can enable :prop_gbl:`INSTALL_PARALLEL` to enable a parallel
installation. When using the parallel install, subdirectories added by calls
to the :command:`add_subdirectory` command are installed independently
and the order that install rules added in different subdirectories will run is
not guaranteed.
.. _`common options`:
There are multiple signatures for this command. Some of them define

View File

@ -0,0 +1,11 @@
CMAKE_INSTALL_PARALLEL_LEVEL
----------------------------
.. versionadded:: 3.31
.. include:: ENV_VAR.txt
Specifies the default maximum number of concurrent processes to use when
installing using ``cmake --install``.
This has no impact unless :prop_gbl:`INSTALL_PARALLEL` is enabled.

View File

@ -52,6 +52,7 @@ Environment Variables that Control the Build
/envvar/CMAKE_GENERATOR_PLATFORM
/envvar/CMAKE_GENERATOR_TOOLSET
/envvar/CMAKE_INSTALL_MODE
/envvar/CMAKE_INSTALL_PARALLEL_LEVEL
/envvar/CMAKE_INSTALL_PREFIX
/envvar/CMAKE_LANG_COMPILER_LAUNCHER
/envvar/CMAKE_LANG_IMPLICIT_LINK_DIRECTORIES_EXCLUDE

View File

@ -745,6 +745,15 @@ The options are:
This option can be omitted if :envvar:`VERBOSE` environment variable is set.
.. option:: -j <jobs>, --parallel <jobs>
.. versionadded:: 3.31
Install in parallel using the given number of jobs. Only available if
:prop_gbl:`INSTALL_PARALLEL` is enabled. The
:envvar:`CMAKE_INSTALL_PARALLEL_LEVEL` environment variable specifies a
default parallel level when this option is not provided.
Run :option:`cmake --install` with no options for quick help.
Open a Project

View File

@ -3,18 +3,22 @@ INSTALL_PARALLEL
.. versionadded:: 3.30
Enables parallel installation option for the Ninja generator.
Enables parallel installation option for a project. The install code for each
subdirectory added with ``add_subdirectory`` can run independently.
When this property is ``ON``, ``install/local`` targets have the
console pool disabled, allowing them to run concurrently.
When using the Ninja generator, setting this property to ``ON``, causes
``install/local`` targets have the console pool disabled, allowing them to run
concurrently.
This property also provides the target ``install/parallel``, which has an
explicit dependency on the ``install/local`` target for each subdirectory,
recursing down the project.
explicit dependency on the ``install/local`` target for each subdirectory.
Setting this property has no affect on the behavior of ``cmake --install``.
The install must be invoked by building the ``install/parallel`` target
directly.
.. versionadded:: 3.31
When this property is ``ON``, ``cmake --install`` can be given the ``-j <jobs>``
or ``--parallel <jobs>`` option to specify a maximum number of jobs.
The :envvar:`CMAKE_INSTALL_PARALLEL_LEVEL` environment variable specifies a
default parallel level if this option is not provided.
Calls to :command:`install(CODE)` or :command:`install(SCRIPT)` might depend
on actions performed by an earlier :command:`install` command in a different

View File

@ -336,6 +336,8 @@ add_library(
cmInstallTargetGenerator.cxx
cmInstallDirectoryGenerator.h
cmInstallDirectoryGenerator.cxx
cmInstallScriptHandler.h
cmInstallScriptHandler.cxx
cmJSONHelpers.cxx
cmJSONHelpers.h
cmJSONState.cxx

View File

@ -135,6 +135,13 @@ cmGlobalGenerator::cmGlobalGenerator(cmake* cm)
cm->GetState()->SetWatcomWMake(false);
cm->GetState()->SetWindowsShell(false);
cm->GetState()->SetWindowsVSIDE(false);
#if !defined(CMAKE_BOOTSTRAP)
Json::StreamWriterBuilder wbuilder;
wbuilder["indentation"] = "\t";
this->JsonWriter =
std::unique_ptr<Json::StreamWriter>(wbuilder.newStreamWriter());
#endif
}
cmGlobalGenerator::~cmGlobalGenerator()
@ -1758,6 +1765,30 @@ void cmGlobalGenerator::Generate()
}
}
#if !defined(CMAKE_BOOTSTRAP)
void cmGlobalGenerator::WriteJsonContent(const std::string& path,
const Json::Value& value) const
{
cmsys::ofstream ftmp(path.c_str());
this->JsonWriter->write(value, &ftmp);
ftmp << '\n';
ftmp.close();
}
void cmGlobalGenerator::WriteInstallJson() const
{
if (this->GetCMakeInstance()->GetState()->GetGlobalPropertyAsBool(
"INSTALL_PARALLEL")) {
Json::Value index(Json::objectValue);
index["InstallScripts"] = Json::arrayValue;
for (const auto& file : this->InstallScripts) {
index["InstallScripts"].append(file);
}
this->WriteJsonContent("CMakeFiles/InstallScripts.json", index);
}
}
#endif
bool cmGlobalGenerator::ComputeTargetDepends()
{
cmComputeTargetDepends ctd(this);
@ -3732,3 +3763,8 @@ cmGlobalGenerator::StripCommandStyle cmGlobalGenerator::GetStripCommandStyle(
return StripCommandStyle::Default;
#endif
}
void cmGlobalGenerator::AddInstallScript(std::string const& file)
{
this->InstallScripts.push_back(file);
}

View File

@ -91,6 +91,9 @@ struct GeneratedMakeCommand
bool RequiresOutputForward = false;
};
}
namespace Json {
class StreamWriter;
}
/** \class cmGlobalGenerator
* \brief Responsible for overseeing the generation process for the entire tree
@ -655,6 +658,8 @@ public:
bool CheckCMP0171() const;
void AddInstallScript(std::string const& file);
protected:
// for a project collect all its targets by following depend
// information, and also collect all the targets
@ -674,6 +679,12 @@ protected:
virtual bool ComputeTargetDepends();
#if !defined(CMAKE_BOOTSTRAP)
void WriteJsonContent(const std::string& fname,
const Json::Value& value) const;
void WriteInstallJson() const;
#endif
virtual bool CheckALLOW_DUPLICATE_CUSTOM_TARGETS() const;
bool ApplyCXXStdTargets();
@ -790,6 +801,10 @@ private:
std::map<std::string, int> LanguageToLinkerPreference;
std::map<std::string, std::string> LanguageToOriginalSharedLibFlags;
#if !defined(CMAKE_BOOTSTRAP)
std::unique_ptr<Json::StreamWriter> JsonWriter;
#endif
#ifdef __APPLE__
std::map<std::string, StripCommandStyle> StripCommandStyleMap;
#endif
@ -882,6 +897,8 @@ private:
std::map<std::string, cmInstallRuntimeDependencySet*>
RuntimeDependencySetsByName;
std::vector<std::string> InstallScripts;
#if !defined(CMAKE_BOOTSTRAP)
// Pool of file locks
cmFileLockPool FileLockPool;

View File

@ -0,0 +1,119 @@
/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
file Copyright.txt or https://cmake.org/licensing for details. */
#include "cmInstallScriptHandler.h"
#include <algorithm>
#include <cstddef>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include <cm/memory>
#include <cm3p/json/reader.h>
#include <cm3p/json/value.h>
#include <cm3p/uv.h>
#include "cmJSONState.h"
#include "cmProcessOutput.h"
#include "cmStringAlgorithms.h"
#include "cmSystemTools.h"
#include "cmUVHandlePtr.h"
#include "cmUVProcessChain.h"
#include "cmUVStream.h"
using InstallScript = cmInstallScriptHandler::InstallScript;
cmInstallScriptHandler::cmInstallScriptHandler(const std::string& binary_dir,
std::vector<std::string>& args)
{
const std::string& file =
cmStrCat(binary_dir, "/CMakeFiles/InstallScripts.json");
if (cmSystemTools::FileExists(file)) {
int compare;
cmSystemTools::FileTimeCompare(
cmStrCat(binary_dir, "/CMakeFiles/cmake.check_cache"), file, &compare);
if (compare < 1) {
args.insert(args.end() - 1, "-DCMAKE_INSTALL_LOCAL_ONLY=1");
Json::CharReaderBuilder rbuilder;
auto JsonReader =
std::unique_ptr<Json::CharReader>(rbuilder.newCharReader());
std::vector<char> content;
Json::Value value;
cmJSONState state(file, &value);
for (auto const& script : value["InstallScripts"]) {
this->commands.push_back(args);
this->commands.back().emplace_back(script.asCString());
}
}
}
}
bool cmInstallScriptHandler::isParallel()
{
return !this->commands.empty();
}
int cmInstallScriptHandler::install(unsigned int j)
{
cm::uv_loop_ptr loop;
loop.init();
std::vector<InstallScript> scripts;
for (auto const& cmd : this->commands) {
scripts.push_back(InstallScript(cmd));
}
std::size_t working = 0;
std::size_t installed = 0;
std::size_t i = 0;
while (installed < scripts.size()) {
for (auto queue = std::min(j - working, scripts.size() - i); queue > 0;
--queue) {
scripts[i].start(loop, [&scripts, &working, &installed, i]() {
scripts[i].printResult(++installed, scripts.size());
--working;
});
++i;
}
uv_run(loop, UV_RUN_DEFAULT);
}
return 0;
}
InstallScript::InstallScript(const std::vector<std::string>& cmd)
{
this->name = cmSystemTools::RelativePath(
cmSystemTools::GetCurrentWorkingDirectory(), cmd.back());
this->command = cmd;
}
void InstallScript::start(cm::uv_loop_ptr& loop,
std::function<void()> callback)
{
cmUVProcessChainBuilder builder;
builder.AddCommand(this->command)
.SetExternalLoop(*loop)
.SetMergedBuiltinStreams();
this->chain = cm::make_unique<cmUVProcessChain>(builder.Start());
this->pipe.init(this->chain->GetLoop(), 0);
uv_pipe_open(this->pipe, this->chain->OutputStream());
this->streamHandler = cmUVStreamRead(
this->pipe,
[this](std::vector<char> data) {
std::string strdata;
cmProcessOutput(cmProcessOutput::Auto)
.DecodeText(data.data(), data.size(), strdata);
this->output.push_back(strdata);
},
std::move(callback));
}
void InstallScript::printResult(std::size_t n, std::size_t total)
{
cmSystemTools::Stdout(cmStrCat("[", n, "/", total, "] ", this->name, "\n"));
for (auto const& line : this->output) {
cmSystemTools::Stdout(line);
}
}

View File

@ -0,0 +1,40 @@
/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
file Copyright.txt or https://cmake.org/licensing for details. */
#pragma once
#include <cstddef>
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include "cmUVHandlePtr.h"
#include "cmUVProcessChain.h"
#include "cmUVStream.h"
class cmInstallScriptHandler
{
public:
cmInstallScriptHandler() = default;
cmInstallScriptHandler(const std::string&, std::vector<std::string>&);
bool isParallel();
int install(unsigned int j);
class InstallScript
{
public:
InstallScript(const std::vector<std::string>&);
void start(cm::uv_loop_ptr&, std::function<void()>);
void printResult(std::size_t n, std::size_t total);
private:
std::vector<std::string> command;
std::vector<std::string> output;
std::string name;
std::unique_ptr<cmUVProcessChain> chain;
std::unique_ptr<cmUVStreamReadHandle> streamHandler;
cm::uv_pipe_ptr pipe;
};
private:
std::vector<std::vector<std::string>> commands;
};

View File

@ -500,6 +500,7 @@ void cmLocalGenerator::GenerateInstallRules()
toplevel_install = 1;
}
file += "/cmake_install.cmake";
this->GetGlobalGenerator()->AddInstallScript(file);
cmGeneratedFileStream fout(file);
fout.SetCopyIfDifferent(true);

View File

@ -2936,6 +2936,7 @@ int cmake::Generate()
this->SaveCache(this->GetHomeOutputDirectory());
#if !defined(CMAKE_BOOTSTRAP)
this->GetGlobalGenerator()->WriteInstallJson();
this->FileAPI->WriteReplies();
#endif

View File

@ -17,6 +17,7 @@
#include <vector>
#include <cm/memory>
#include <cm/optional>
#include <cmext/algorithm>
#include <cm3p/uv.h>
@ -26,6 +27,7 @@
#include "cmConsoleBuf.h"
#include "cmDocumentationEntry.h"
#include "cmGlobalGenerator.h"
#include "cmInstallScriptHandler.h"
#include "cmList.h"
#include "cmMakefile.h"
#include "cmMessageMetadata.h"
@ -429,6 +431,17 @@ int extract_job_number(std::string const& command,
}
return jobs;
}
std::function<bool(std::string const&)> extract_job_number_lambda_builder(
std::string& dir, int& jobs, const std::string& flag)
{
return [&dir, &jobs, flag](std::string const& value) -> bool {
jobs = extract_job_number(flag, value);
if (jobs < 0) {
dir.clear();
}
return true;
};
};
#endif
int do_build(int ac, char const* const* av)
@ -451,20 +464,10 @@ int do_build(int ac, char const* const* av)
std::string presetName;
bool listPresets = false;
auto jLambda = [&](std::string const& value) -> bool {
jobs = extract_job_number("-j", value);
if (jobs < 0) {
dir.clear();
}
return true;
};
auto parallelLambda = [&](std::string const& value) -> bool {
jobs = extract_job_number("--parallel", value);
if (jobs < 0) {
dir.clear();
}
return true;
};
auto jLambda = extract_job_number_lambda_builder(dir, jobs, "-j");
auto parallelLambda =
extract_job_number_lambda_builder(dir, jobs, "--parallel");
auto targetLambda = [&](std::string const& value) -> bool {
if (!value.empty()) {
cmList values{ value };
@ -787,9 +790,14 @@ int do_install(int ac, char const* const* av)
std::string defaultDirectoryPermissions;
std::string prefix;
std::string dir;
int jobs = 0;
bool strip = false;
bool verbose = cmSystemTools::HasEnv("VERBOSE");
auto jLambda = extract_job_number_lambda_builder(dir, jobs, "-j");
auto parallelLambda =
extract_job_number_lambda_builder(dir, jobs, "--parallel");
auto verboseLambda = [&](std::string const&) -> bool {
verbose = true;
return true;
@ -806,6 +814,9 @@ int do_install(int ac, char const* const* av)
CommandArgument{
"--default-directory-permissions", CommandArgument::Values::One,
CommandArgument::setToValue(defaultDirectoryPermissions) },
CommandArgument{ "-j", CommandArgument::Values::One, jLambda },
CommandArgument{ "--parallel", CommandArgument::Values::One,
parallelLambda },
CommandArgument{ "--prefix", CommandArgument::Values::One,
CommandArgument::setToValue(prefix) },
CommandArgument{ "--strip", CommandArgument::Values::Zero,
@ -822,7 +833,6 @@ int do_install(int ac, char const* const* av)
inputArgs.reserve(ac - 3);
cm::append(inputArgs, av + 3, av + ac);
for (decltype(inputArgs.size()) i = 0; i < inputArgs.size(); ++i) {
std::string const& arg = inputArgs[i];
bool matched = false;
bool parsed = false;
@ -853,6 +863,10 @@ int do_install(int ac, char const* const* av)
" --component <comp> = Component-based install. Only install <comp>.\n"
" --default-directory-permissions <permission> \n"
" Default install permission. Use default permission <permission>.\n"
" -j <jobs> --parallel <jobs>\n"
" Build in parallel using the given number of jobs. \n"
" The CMAKE_INSTALL_PARALLEL_LEVEL environment variable\n"
" specifies a default parallel level when this option is not given.\n"
" --prefix <prefix> = The installation prefix CMAKE_INSTALL_PREFIX.\n"
" --strip = Performing install/strip.\n"
" -v --verbose = Enable verbose output.\n"
@ -906,9 +920,29 @@ int do_install(int ac, char const* const* av)
}
args.emplace_back("-P");
args.emplace_back(dir + "/cmake_install.cmake");
return cm.Run(args) ? 1 : 0;
auto handler = cmInstallScriptHandler(dir, args);
int ret = 0;
if (!handler.isParallel()) {
args.emplace_back(cmStrCat(dir, "/cmake_install.cmake"));
ret = int(bool(cm.Run(args)));
} else {
if (!jobs) {
jobs = 1;
auto envvar = cmSystemTools::GetEnvVar("CMAKE_INSTALL_PARALLEL_LEVEL");
if (envvar.has_value()) {
jobs = extract_job_number("", envvar.value());
if (jobs < 1) {
std::cerr << "Value of CMAKE_INSTALL_PARALLEL_LEVEL environment"
" variable must be a positive integer.\n";
return 1;
}
}
}
ret = handler.install(jobs);
}
return int(ret > 0);
#endif
}

View File

@ -258,8 +258,8 @@ if(CMAKE_GENERATOR MATCHES "Ninja")
add_RunCMake_test(NinjaPrivateDeps
-DCMAKE_C_OUTPUT_EXTENSION=${CMAKE_C_OUTPUT_EXTENSION}
-DRunCMake_GENERATOR_IS_MULTI_CONFIG=${_isMultiConfig})
add_RunCMake_test(InstallParallel)
endif()
add_RunCMake_test(InstallParallel)
add_RunCMake_test(CTest)
if(NOT CMake_TEST_EXTERNAL_CMAKE)

View File

@ -1,17 +1,36 @@
include(RunCMake)
function(install_test test parallel install_target check_script)
function(install_test test parallel install_arg)
cmake_parse_arguments(ARGS "NINJA;TOUCH_CACHE" "VERIFY_SCRIPT" "" ${ARGN})
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${test}-install)
set(RunCMake_TEST_OPTIONS -DINSTALL_PARALLEL=${parallel})
set(RunCMake_TEST_OUTPUT_MERGE 1)
if (NOT RunCMake_GENERATOR_IS_MULTI_CONFIG)
list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_BUILD_TYPE=Debug)
endif()
run_cmake(install)
set(RunCMake_TEST_NO_CLEAN 1)
run_cmake_command(${test}-install ${CMAKE_COMMAND} --build . --config Debug -t ${install_target})
if (ARGS_TOUCH_CACHE)
run_cmake_command(${test}-touch
${CMAKE_COMMAND} -E touch ${RunCMake_TEST_BINARY_DIR}/CMakeFiles/cmake.check_cache)
endif()
if (ARGS_NINJA)
run_cmake_command(${test}-install ${CMAKE_COMMAND} --build . --config Debug -t ${install_arg})
else()
run_cmake_command(${test}-install ${CMAKE_COMMAND} --install . -j ${install_arg})
endif()
set(RunCMake_TEST_COMMAND_WORKING_DIRECTORY ${RunCMake_SOURCE_DIR})
run_cmake_command(verify-parallel ${CMAKE_COMMAND} -P ${check_script} ${RunCMake_TEST_BINARY_DIR}/.ninja_log)
if (ARGS_VERIFY_SCRIPT)
run_cmake_command(${test}-verify-parallel
${CMAKE_COMMAND} -P ${ARGS_VERIFY_SCRIPT} ${RunCMake_TEST_BINARY_DIR}/.ninja_log)
endif()
endfunction()
install_test(parallel 1 install/parallel check-parallel.cmake)
install_test(no-parallel 0 install check-single.cmake)
install_test(parallel 1 4)
install_test(no-parallel 0 4)
install_test(out-of-date-json 1 4 TOUCH_CACHE)
if(RunCMake_GENERATOR MATCHES "Ninja")
install_test(ninja-parallel 1 install/parallel VERIFY_SCRIPT check-parallel.cmake NINJA)
install_test(ninja-no-parallel 0 install VERIFY_SCRIPT check-single.cmake NINJA)
endif()

View File

@ -0,0 +1,15 @@
\[1\/5\] Installing only the local directory...
\-\- Install configuration:.*
installing:.*
\[2\/5\] Installing only the local directory...
\-\- Install configuration:.*
installing:.*
\[3\/5\] Installing only the local directory...
\-\- Install configuration:.*
installing:.*
\[4\/5\] Installing only the local directory...
\-\- Install configuration:.*
installing:.*
\[5\/5\] Installing only the local directory...
\-\- Install configuration:.*
installing:.*

View File

@ -0,0 +1,5 @@
installing:.*
installing:.*
installing:.*
installing:.*
installing:.*

View File

@ -0,0 +1,5 @@
installing:.*
installing:.*
installing:.*
installing:.*
installing:.*

View File

@ -1,15 +1,15 @@
\[1\/5\] Installing only the local directory...
\-\- Install configuration: \"Debug\"
\[1\/5\] .*
\-\- Install configuration:.*
installing:.*
\[2\/5\] Installing only the local directory...
\-\- Install configuration: \"Debug\"
\[2\/5\] .*
\-\- Install configuration:.*
installing:.*
\[3\/5\] Installing only the local directory...
\-\- Install configuration: \"Debug\"
\[3\/5\] .*
\-\- Install configuration:.*
installing:.*
\[4\/5\] Installing only the local directory...
\-\- Install configuration: \"Debug\"
\[4\/5\] .*
\-\- Install configuration:.*
installing:.*
\[5\/5\] Installing only the local directory...
\-\- Install configuration: \"Debug\"
\[5\/5\] .*
\-\- Install configuration:.*
installing:.*