Support job pools in custom commands and targets

Provide a way for custom commands and targets to set the pool variable
of the ninja build statement. Setting `JOB_POOL` is not compatible with
`USES_TERMINAL`, which implies the `console` pool.

The option is silently ignored with other generators.

Closes: #18483

Help/command/add_custom_command.rst

@@ -23,6 +23,7 @@ The first signature is for adding a custom command to produce an output:
                      [WORKING_DIRECTORY dir]
                      [COMMENT comment]
                      [DEPFILE depfile]
+                     [JOB_POOL job_pool]
                      [VERBATIM] [APPEND] [USES_TERMINAL]
                      [COMMAND_EXPAND_LISTS])
 
@@ -144,6 +145,13 @@ The options are:
   Note that the ``IMPLICIT_DEPENDS`` option is currently supported
   only for Makefile generators and will be ignored by other generators.
 
+``JOB_POOL``
+  Specify a :prop_gbl:`pool <JOB_POOLS>` for the :generator:`Ninja`
+  generator. Incompatible with ``USES_TERMINAL``, which implies
+  the ``console`` pool.
+  Using a pool that is not defined by :prop_gbl:`JOB_POOLS` causes
+  an error by ninja at build time.
+
 ``MAIN_DEPENDENCY``
   Specify the primary input source file to the command.  This is
   treated just like any value given to the ``DEPENDS`` option

Help/command/add_custom_target.rst

@@ -11,6 +11,7 @@ Add a target with no output so it will always be built.
                     [BYPRODUCTS [files...]]
                     [WORKING_DIRECTORY dir]
                     [COMMENT comment]
+                    [JOB_POOL job_pool]
                     [VERBATIM] [USES_TERMINAL]
                     [COMMAND_EXPAND_LISTS]
                     [SOURCES src1 [src2...]])
@@ -97,6 +98,13 @@ The options are:
   ``${CC} "-I$<JOIN:$<TARGET_PROPERTY:foo,INCLUDE_DIRECTORIES>,;-I>" foo.cc``
   to be properly expanded.
 
+``JOB_POOL``
+  Specify a :prop_gbl:`pool <JOB_POOLS>` for the :generator:`Ninja`
+  generator. Incompatible with ``USES_TERMINAL``, which implies
+  the ``console`` pool.
+  Using a pool that is not defined by :prop_gbl:`JOB_POOLS` causes
+  an error by ninja at build time.
+
 ``SOURCES``
   Specify additional source files to be included in the custom target.
   Specified source files will be added to IDE project files for

Help/prop_gbl/JOB_POOLS.rst

@@ -18,6 +18,11 @@ Defined pools could be used globally by setting
 :variable:`CMAKE_JOB_POOL_COMPILE` and :variable:`CMAKE_JOB_POOL_LINK`
 or per target by setting the target properties
 :prop_tgt:`JOB_POOL_COMPILE` and :prop_tgt:`JOB_POOL_LINK`.
+:command:`Custom commands <add_custom_command>` and
+:command:`custom targets <add_custom_target>` can specify pools using the
+option ``JOB_POOL``.
+Using a pool that is not defined by :prop_gbl:`JOB_POOLS` causes
+an error by ninja at build time.
 
 If not set, this property uses the value of the :variable:`CMAKE_JOB_POOLS`
 variable.

Help/release/dev/ninja-add_custom_command-pool.rst

@@ -0,0 +1,6 @@
+ninja-add_custom_command-pool
+--------------------------------
+
+* The commands :command:`add_custom_command` and :command:`add_custom_target`
+  gained a new ``JOB_POOL`` option that works with the :generator:`Ninja`
+  generator to set the pool variable on the build statement.

Source/cmAddCustomCommandCommand.cxx

@@ -31,7 +31,7 @@ bool cmAddCustomCommandCommand::InitialPass(
     return false;
   }
 
-  std::string source, target, main_dependency, working, depfile;
+  std::string source, target, main_dependency, working, depfile, job_pool;
   std::string comment_buffer;
   const char* comment = nullptr;
   std::vector<std::string> depends, outputs, output, byproducts;
@@ -65,6 +65,7 @@ bool cmAddCustomCommandCommand::InitialPass(
     doing_comment,
     doing_working_directory,
     doing_depfile,
+    doing_job_pool,
     doing_nothing
   };
 
@@ -81,6 +82,7 @@ bool cmAddCustomCommandCommand::InitialPass(
   MAKE_STATIC_KEYWORD(DEPENDS);
   MAKE_STATIC_KEYWORD(DEPFILE);
   MAKE_STATIC_KEYWORD(IMPLICIT_DEPENDS);
+  MAKE_STATIC_KEYWORD(JOB_POOL);
   MAKE_STATIC_KEYWORD(MAIN_DEPENDENCY);
   MAKE_STATIC_KEYWORD(OUTPUT);
   MAKE_STATIC_KEYWORD(OUTPUTS);
@@ -104,6 +106,7 @@ bool cmAddCustomCommandCommand::InitialPass(
     keywords.insert(keyDEPENDS);
     keywords.insert(keyDEPFILE);
     keywords.insert(keyIMPLICIT_DEPENDS);
+    keywords.insert(keyJOB_POOL);
     keywords.insert(keyMAIN_DEPENDENCY);
     keywords.insert(keyOUTPUT);
     keywords.insert(keyOUTPUTS);
@@ -170,6 +173,8 @@ bool cmAddCustomCommandCommand::InitialPass(
                          this->Makefile->GetGlobalGenerator()->GetName());
           return false;
         }
+      } else if (copy == keyJOB_POOL) {
+        doing = doing_job_pool;
       }
     } else {
       std::string filename;
@@ -211,6 +216,9 @@ bool cmAddCustomCommandCommand::InitialPass(
         case doing_depfile:
           depfile = copy;
           break;
+        case doing_job_pool:
+          job_pool = copy;
+          break;
         case doing_working_directory:
           working = copy;
           break;
@@ -318,6 +326,11 @@ bool cmAddCustomCommandCommand::InitialPass(
     return false;
   }
 
+  if (uses_terminal && !job_pool.empty()) {
+    this->SetError("JOB_POOL is shadowed by USES_TERMINAL.");
+    return false;
+  }
+
   // Choose which mode of the command to use.
   bool escapeOldStyle = !verbatim;
   if (source.empty() && output.empty()) {
@@ -325,14 +338,14 @@ bool cmAddCustomCommandCommand::InitialPass(
     std::vector<std::string> no_depends;
     this->Makefile->AddCustomCommandToTarget(
       target, byproducts, no_depends, commandLines, cctype, comment,
-      working.c_str(), escapeOldStyle, uses_terminal, depfile,
+      working.c_str(), escapeOldStyle, uses_terminal, depfile, job_pool,
       command_expand_lists);
   } else if (target.empty()) {
     // Target is empty, use the output.
     this->Makefile->AddCustomCommandToOutput(
       output, byproducts, depends, main_dependency, commandLines, comment,
       working.c_str(), false, escapeOldStyle, uses_terminal,
-      command_expand_lists, depfile);
+      command_expand_lists, depfile, job_pool);
 
     // Add implicit dependency scanning requests if any were given.
     if (!implicit_depends.empty()) {

Source/cmAddCustomTargetCommand.cxx

@@ -52,6 +52,7 @@ bool cmAddCustomTargetCommand::InitialPass(
   std::string comment_buffer;
   const char* comment = nullptr;
   std::vector<std::string> sources;
+  std::string job_pool;
 
   // Keep track of parser state.
   enum tdoing
@@ -62,6 +63,7 @@ bool cmAddCustomTargetCommand::InitialPass(
     doing_working_directory,
     doing_comment,
     doing_source,
+    doing_job_pool,
     doing_nothing
   };
   tdoing doing = doing_command;
@@ -97,6 +99,8 @@ bool cmAddCustomTargetCommand::InitialPass(
       command_expand_lists = true;
     } else if (copy == "COMMENT") {
       doing = doing_comment;
+    } else if (copy == "JOB_POOL") {
+      doing = doing_job_pool;
     } else if (copy == "COMMAND") {
       doing = doing_command;
 
@@ -137,6 +141,9 @@ bool cmAddCustomTargetCommand::InitialPass(
         case doing_source:
           sources.push_back(copy);
           break;
+        case doing_job_pool:
+          job_pool = copy;
+          break;
         default:
           this->SetError("Wrong syntax. Unknown type of argument.");
           return false;
@@ -200,12 +207,17 @@ bool cmAddCustomTargetCommand::InitialPass(
     return true;
   }
 
+  if (uses_terminal && !job_pool.empty()) {
+    this->SetError("JOB_POOL is shadowed by USES_TERMINAL.");
+    return false;
+  }
+
   // Add the utility target to the makefile.
   bool escapeOldStyle = !verbatim;
   cmTarget* target = this->Makefile->AddUtilityCommand(
     targetName, cmMakefile::TargetOrigin::Project, excludeFromAll,
     working_directory.c_str(), byproducts, depends, commandLines,
-    escapeOldStyle, comment, uses_terminal, command_expand_lists);
+    escapeOldStyle, comment, uses_terminal, command_expand_lists, job_pool);
 
   // Add additional user-specified source files to the target.
   target->AddSources(sources);

Source/cmCustomCommand.cxx

@@ -134,3 +134,13 @@ void cmCustomCommand::SetDepfile(const std::string& depfile)
 {
   this->Depfile = depfile;
 }
+
+const std::string& cmCustomCommand::GetJobPool() const
+{
+  return this->JobPool;
+}
+
+void cmCustomCommand::SetJobPool(const std::string& job_pool)
+{
+  this->JobPool = job_pool;
+}

Source/cmCustomCommand.h

@@ -89,6 +89,10 @@ public:
   const std::string& GetDepfile() const;
   void SetDepfile(const std::string& depfile);
 
+  /** Set/Get the job_pool (used by the Ninja generator) */
+  const std::string& GetJobPool() const;
+  void SetJobPool(const std::string& job_pool);
+
 private:
   std::vector<std::string> Outputs;
   std::vector<std::string> Byproducts;
@@ -99,6 +103,7 @@ private:
   std::string Comment;
   std::string WorkingDirectory;
   std::string Depfile;
+  std::string JobPool;
   bool HaveComment = false;
   bool EscapeAllowMakeVars = false;
   bool EscapeOldStyle = true;

Source/cmGlobalNinjaGenerator.cxx

@@ -252,8 +252,9 @@ void cmGlobalNinjaGenerator::AddCustomCommandRule()
 
 void cmGlobalNinjaGenerator::WriteCustomCommandBuild(
   const std::string& command, const std::string& description,
-  const std::string& comment, const std::string& depfile, bool uses_terminal,
-  bool restat, const cmNinjaDeps& outputs, const cmNinjaDeps& deps,
+  const std::string& comment, const std::string& depfile,
+  const std::string& job_pool, bool uses_terminal, bool restat,
+  const cmNinjaDeps& outputs, const cmNinjaDeps& deps,
   const cmNinjaDeps& orderOnly)
 {
   std::string cmd = command; // NOLINT(*)
@@ -273,6 +274,8 @@ void cmGlobalNinjaGenerator::WriteCustomCommandBuild(
   }
   if (uses_terminal && SupportsConsolePool()) {
     vars["pool"] = "console";
+  } else if (!job_pool.empty()) {
+    vars["pool"] = job_pool;
   }
   if (!depfile.empty()) {
     vars["depfile"] = depfile;
@@ -951,7 +954,8 @@ void cmGlobalNinjaGenerator::WriteAssumedSourceDependencies()
               std::back_inserter(orderOnlyDeps));
     WriteCustomCommandBuild(/*command=*/"", /*description=*/"",
                             "Assume dependencies for generated source file.",
-                            /*depfile*/ "", /*uses_terminal*/ false,
+                            /*depfile*/ "", /*job_pool*/ "",
+                            /*uses_terminal*/ false,
                             /*restat*/ true, cmNinjaDeps(1, asd.first),
                             cmNinjaDeps(), orderOnlyDeps);
   }

Source/cmGlobalNinjaGenerator.h

@@ -129,7 +129,8 @@ public:
   void WriteCustomCommandBuild(const std::string& command,
                                const std::string& description,
                                const std::string& comment,
-                               const std::string& depfile, bool uses_terminal,
+                               const std::string& depfile,
+                               const std::string& pool, bool uses_terminal,
                                bool restat, const cmNinjaDeps& outputs,
                                const cmNinjaDeps& deps = cmNinjaDeps(),
                                const cmNinjaDeps& orderOnly = cmNinjaDeps());

Source/cmGlobalXCodeGenerator.cxx

@@ -568,7 +568,7 @@ void cmGlobalXCodeGenerator::AddExtraTargets(
         gen->GetMakefile()->AddCustomCommandToTarget(
           target->GetName(), no_byproducts, no_depends, commandLines,
           cmTarget::POST_BUILD, "Depend check for xcode", dir.c_str(), true,
-          false, "", false, cmMakefile::AcceptObjectLibraryCommands);
+          false, "", "", false, cmMakefile::AcceptObjectLibraryCommands);
       }
 
       if (!this->IsExcluded(target)) {

Source/cmLocalNinjaGenerator.cxx

@@ -509,7 +509,7 @@ void cmLocalNinjaGenerator::WriteCustomCommandBuildStatement(
     this->GetGlobalNinjaGenerator()->WriteCustomCommandBuild(
       this->BuildCommandLine(cmdLines, customStep),
       this->ConstructComment(ccg), "Custom command for " + ninjaOutputs[0],
-      cc->GetDepfile(), cc->GetUsesTerminal(),
+      cc->GetDepfile(), cc->GetJobPool(), cc->GetUsesTerminal(),
       /*restat*/ !symbolic || !byproducts.empty(), ninjaOutputs, ninjaDeps,
       orderOnlyDeps);
   }

Source/cmMakefile.cxx

@@ -814,8 +814,8 @@ void cmMakefile::AddCustomCommandToTarget(
   const std::vector<std::string>& depends,
   const cmCustomCommandLines& commandLines, cmTarget::CustomCommandType type,
   const char* comment, const char* workingDir, bool escapeOldStyle,
-  bool uses_terminal, const std::string& depfile, bool command_expand_lists,
-  ObjectLibraryCommands objLibraryCommands)
+  bool uses_terminal, const std::string& depfile, const std::string& job_pool,
+  bool command_expand_lists, ObjectLibraryCommands objLibraryCommands)
 {
   // Find the target to which to add the custom command.
   cmTargets::iterator ti = this->Targets.find(target);
@@ -890,6 +890,7 @@ void cmMakefile::AddCustomCommandToTarget(
   cc.SetUsesTerminal(uses_terminal);
   cc.SetCommandExpandLists(command_expand_lists);
   cc.SetDepfile(depfile);
+  cc.SetJobPool(job_pool);
   switch (type) {
     case cmTarget::PRE_BUILD:
       t.AddPreBuildCommand(cc);
@@ -909,7 +910,8 @@ cmSourceFile* cmMakefile::AddCustomCommandToOutput(
   const std::vector<std::string>& depends, const std::string& main_dependency,
   const cmCustomCommandLines& commandLines, const char* comment,
   const char* workingDir, bool replace, bool escapeOldStyle,
-  bool uses_terminal, bool command_expand_lists, const std::string& depfile)
+  bool uses_terminal, bool command_expand_lists, const std::string& depfile,
+  const std::string& job_pool)
 {
   // Make sure there is at least one output.
   if (outputs.empty()) {
@@ -1003,6 +1005,7 @@ cmSourceFile* cmMakefile::AddCustomCommandToOutput(
     cc->SetUsesTerminal(uses_terminal);
     cc->SetCommandExpandLists(command_expand_lists);
     cc->SetDepfile(depfile);
+    cc->SetJobPool(job_pool);
     file->SetCustomCommand(cc);
     this->UpdateOutputToSourceMap(outputs, file);
   }
@@ -1040,7 +1043,7 @@ cmSourceFile* cmMakefile::AddCustomCommandToOutput(
   const std::string& main_dependency, const cmCustomCommandLines& commandLines,
   const char* comment, const char* workingDir, bool replace,
   bool escapeOldStyle, bool uses_terminal, bool command_expand_lists,
-  const std::string& depfile)
+  const std::string& depfile, const std::string& job_pool)
 {
   std::vector<std::string> outputs;
   outputs.push_back(output);
@@ -1048,7 +1051,7 @@ cmSourceFile* cmMakefile::AddCustomCommandToOutput(
   return this->AddCustomCommandToOutput(
     outputs, no_byproducts, depends, main_dependency, commandLines, comment,
     workingDir, replace, escapeOldStyle, uses_terminal, command_expand_lists,
-    depfile);
+    depfile, job_pool);
 }
 
 void cmMakefile::AddCustomCommandOldStyle(
@@ -1142,13 +1145,14 @@ cmTarget* cmMakefile::AddUtilityCommand(
   const std::string& utilityName, TargetOrigin origin, bool excludeFromAll,
   const char* workingDirectory, const std::vector<std::string>& depends,
   const cmCustomCommandLines& commandLines, bool escapeOldStyle,
-  const char* comment, bool uses_terminal, bool command_expand_lists)
+  const char* comment, bool uses_terminal, bool command_expand_lists,
+  const std::string& job_pool)
 {
   std::vector<std::string> no_byproducts;
-  return this->AddUtilityCommand(utilityName, origin, excludeFromAll,
-                                 workingDirectory, no_byproducts, depends,
-                                 commandLines, escapeOldStyle, comment,
-                                 uses_terminal, command_expand_lists);
+  return this->AddUtilityCommand(
+    utilityName, origin, excludeFromAll, workingDirectory, no_byproducts,
+    depends, commandLines, escapeOldStyle, comment, uses_terminal,
+    command_expand_lists, job_pool);
 }
 
 cmTarget* cmMakefile::AddUtilityCommand(
@@ -1156,7 +1160,8 @@ cmTarget* cmMakefile::AddUtilityCommand(
   const char* workingDirectory, const std::vector<std::string>& byproducts,
   const std::vector<std::string>& depends,
   const cmCustomCommandLines& commandLines, bool escapeOldStyle,
-  const char* comment, bool uses_terminal, bool command_expand_lists)
+  const char* comment, bool uses_terminal, bool command_expand_lists,
+  const std::string& job_pool)
 {
   // Create a target instance for this utility.
   cmTarget* target = this->AddNewTarget(cmStateEnums::UTILITY, utilityName);
@@ -1182,7 +1187,7 @@ cmTarget* cmMakefile::AddUtilityCommand(
     this->AddCustomCommandToOutput(
       forced, byproducts, depends, no_main_dependency, commandLines, comment,
       workingDirectory, no_replace, escapeOldStyle, uses_terminal,
-      command_expand_lists);
+      command_expand_lists, /*depfile=*/"", job_pool);
     cmSourceFile* sf = target->AddSourceCMP0049(force);
 
     // The output is not actually created so mark it symbolic.

Source/cmMakefile.h

@@ -150,7 +150,7 @@ public:
     const cmCustomCommandLines& commandLines, cmTarget::CustomCommandType type,
     const char* comment, const char* workingDir, bool escapeOldStyle = true,
     bool uses_terminal = false, const std::string& depfile = "",
-    bool command_expand_lists = false,
+    const std::string& job_pool = "", bool command_expand_lists = false,
     ObjectLibraryCommands objLibraryCommands = RejectObjectLibraryCommands);
   cmSourceFile* AddCustomCommandToOutput(
     const std::vector<std::string>& outputs,
@@ -160,14 +160,14 @@ public:
     const cmCustomCommandLines& commandLines, const char* comment,
     const char* workingDir, bool replace = false, bool escapeOldStyle = true,
     bool uses_terminal = false, bool command_expand_lists = false,
-    const std::string& depfile = "");
+    const std::string& depfile = "", const std::string& job_pool = "");
   cmSourceFile* AddCustomCommandToOutput(
     const std::string& output, const std::vector<std::string>& depends,
     const std::string& main_dependency,
     const cmCustomCommandLines& commandLines, const char* comment,
     const char* workingDir, bool replace = false, bool escapeOldStyle = true,
     bool uses_terminal = false, bool command_expand_lists = false,
-    const std::string& depfile = "");
+    const std::string& depfile = "", const std::string& job_pool = "");
   void AddCustomCommandOldStyle(const std::string& target,
                                 const std::vector<std::string>& outputs,
                                 const std::vector<std::string>& depends,
@@ -223,14 +223,14 @@ public:
     const char* workingDirectory, const std::vector<std::string>& depends,
     const cmCustomCommandLines& commandLines, bool escapeOldStyle = true,
     const char* comment = nullptr, bool uses_terminal = false,
-    bool command_expand_lists = false);
+    bool command_expand_lists = false, const std::string& job_pool = "");
   cmTarget* AddUtilityCommand(
     const std::string& utilityName, TargetOrigin origin, bool excludeFromAll,
     const char* workingDirectory, const std::vector<std::string>& byproducts,
     const std::vector<std::string>& depends,
     const cmCustomCommandLines& commandLines, bool escapeOldStyle = true,
     const char* comment = nullptr, bool uses_terminal = false,
-    bool command_expand_lists = false);
+    bool command_expand_lists = false, const std::string& job_pool = "");
 
   /**
    * Add a subdirectory to the build.

Source/cmNinjaUtilityTargetGenerator.cxx

@@ -132,7 +132,7 @@ void cmNinjaUtilityTargetGenerator::Generate()
 
     this->GetGlobalGenerator()->WriteCustomCommandBuild(
       command, desc, "Utility command for " + this->GetTargetName(),
-      /*depfile*/ "", uses_terminal,
+      /*depfile*/ "", /*job_pool*/ "", uses_terminal,
       /*restat*/ true, util_outputs, deps);
 
     this->GetGlobalGenerator()->WritePhonyBuild(

Tests/RunCMake/Ninja/CustomCommandJobPool-check.cmake

@@ -0,0 +1,8 @@
+set(log "${RunCMake_BINARY_DIR}/CustomCommandJobPool-build/build.ninja")
+file(READ "${log}" build_file)
+if(NOT "${build_file}" MATCHES "pool = custom_command_pool")
+  set(RunCMake_TEST_FAILED "Log file:\n ${log}\ndoes not have expected line: pool = custom_command_pool")
+endif()
+if(NOT "${build_file}" MATCHES "pool = custom_target_pool")
+  set(RunCMake_TEST_FAILED "Log file:\n ${log}\ndoes not have expected line: pool = custom_target_pool")
+endif()

Tests/RunCMake/Ninja/CustomCommandJobPool.cmake

@@ -0,0 +1,17 @@
+add_custom_command(
+  OUTPUT hello.copy.c
+  COMMAND "${CMAKE_COMMAND}" -E copy
+          "${CMAKE_CURRENT_SOURCE_DIR}/hello.c"
+          hello.copy.c
+  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+  JOB_POOL "custom_command_pool"
+  )
+add_custom_target(copy ALL DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/hello.copy.c")
+
+add_custom_target(
+  hello.echo
+  COMMAND echo
+  JOB_POOL "custom_target_pool"
+)
+
+include(CheckNoPrefixSubDir.cmake)

Tests/RunCMake/Ninja/JobPoolUsesTerminal-result.txt

@@ -0,0 +1 @@
+1

Tests/RunCMake/Ninja/JobPoolUsesTerminal-stderr.txt

@@ -0,0 +1,9 @@
+^CMake Error at JobPoolUsesTerminal.cmake:1 \(add_custom_command\):
+  add_custom_command JOB_POOL is shadowed by USES_TERMINAL.
+Call Stack \(most recent call first\):
+  CMakeLists.txt:3 \(include\)
++
+CMake Error at JobPoolUsesTerminal.cmake:2 \(add_custom_target\):
+  add_custom_target JOB_POOL is shadowed by USES_TERMINAL.
+Call Stack \(most recent call first\):
+  CMakeLists.txt:3 \(include\)$

Tests/RunCMake/Ninja/JobPoolUsesTerminal.cmake

@@ -0,0 +1,2 @@
+add_custom_command(OUTPUT x COMMAND y USES_TERMINAL JOB_POOL z)
+add_custom_target(CustomTarget COMMAND y USES_TERMINAL JOB_POOL z)

Tests/RunCMake/Ninja/RunCMakeTest.cmake

@@ -57,6 +57,8 @@ run_CMP0058(NEW-no)
 run_CMP0058(NEW-by)
 
 run_cmake(CustomCommandDepfile)
+run_cmake(CustomCommandJobPool)
+run_cmake(JobPoolUsesTerminal)
 
 run_cmake(RspFileC)
 run_cmake(RspFileCXX)