From 31c71859e1ada8835ab04e996dedd062e9300f1f Mon Sep 17 00:00:00 2001 From: vasilito Date: Sun, 2 Aug 2026 07:32:53 +0300 Subject: [PATCH] cook: dep-level parallel recipe cooking (concurrent independent recipes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cook orchestrator built recipes strictly serially — COOKBOOK_MAKE_JOBS only parallelized within a single recipe's make, so cores sat idle during every recipe's configure/link/small-file phases. src/cook/scheduler.rs (dep_levels) was written for this but never wired in (the module wasn't even declared, so it never compiled). - Declare the scheduler module (src/cook.rs) so dep_levels + its 7 tests build. - New COOKBOOK_COOK_JOBS setting (config.rs, default 1 = serial): how many recipes to cook concurrently. - run_parallel_cook (repo.rs): partition recipes into topological dep-levels (same level = no inter-deps = safe concurrent), process levels in order behind a barrier, and within a level pull recipes from an atomic work-index across scoped worker threads. The make-job budget is divided per level so a lone heavyweight recipe (e.g. qtbase, alone at its level) still gets the full -j, while crowded levels share it — no oversubscription, no slowdown for serial-level recipes. Publishing stays a single batched step. Failure/nonstop semantics preserved; only engaged for with cook_jobs>1 and >1 recipe. - Safe because each recipe builds in its own target/stage/sysroot and a recipe's sysroot is assembled from its dependencies' (lower-level, already-complete) stage pkgars — read-only for concurrent siblings. - package.rs: serialize packaging behind a global PACKAGE_LOCK. package() shares process-global state (the lazily-created pkgar signing key at build/id_ed25519.toml, name/metadata construction) that raced under concurrent cooks (corrupt key, mangled names like 'x.dev.dev'); packaging is cheap, so serializing it keeps the compile-parallelism win. Also make the package-name construction fail gracefully (Err, not unwrap-panic) so a bad name fails one recipe instead of crashing the whole scoped-thread build. Validated: all-cached tree (COOK_JOBS=4) and a real concurrent build of 3 independent recipes (COOK_JOBS=3) both succeed and publish correctly. --- src/bin/repo.rs | 162 ++++++++++++++++++++++++++++++++++++++------ src/config.rs | 12 ++++ src/cook.rs | 1 + src/cook/package.rs | 24 +++++-- 4 files changed, 174 insertions(+), 25 deletions(-) diff --git a/src/bin/repo.rs b/src/bin/repo.rs index c62718b087..5bd81cbd97 100644 --- a/src/bin/repo.rs +++ b/src/bin/repo.rs @@ -277,29 +277,35 @@ fn main_inner() -> anyhow::Result<()> { } let verbose = config.cook.verbose; - for recipe in &recipes { - match repo_inner(&config, &command, recipe) { - Ok(cached) => { - if !command.is_informational() { - if cached { - print_cached(&command, &recipe.name); - } else { - print_success(&command, &recipe.name); + // Dep-level parallel cooking: only for `cook` with concurrency requested and + // more than one recipe. Everything else keeps the serial path. + if command == CliCommand::Cook && config.cook.cook_jobs > 1 && recipes.len() > 1 { + run_parallel_cook(&config, &command, &recipes)?; + } else { + for recipe in &recipes { + match repo_inner(&config, &command, recipe) { + Ok(cached) => { + if !command.is_informational() { + if cached { + print_cached(&command, &recipe.name); + } else { + print_success(&command, &recipe.name); + } } } - } - Err(e) => { - if config.cook.nonstop { - if verbose { - eprintln!("repo: {e:#}"); + Err(e) => { + if config.cook.nonstop { + if verbose { + eprintln!("repo: {e:#}"); + } + if let Err(e) = handle_nonstop_fail(recipe) { + eprintln!("repo: {e:#}") + }; + } + print_failed(&command, &recipe.name); + if !config.cook.nonstop { + return Err(e); } - if let Err(e) = handle_nonstop_fail(recipe) { - eprintln!("repo: {e:#}") - }; - } - print_failed(&command, &recipe.name); - if !config.cook.nonstop { - return Err(e); } } } @@ -355,6 +361,122 @@ fn print_cached(command: &CliCommand, recipe: &PackageName) { ); } +/// Cook a set of recipes with dep-level-aware concurrency. +/// +/// Recipes are partitioned into topological levels (see +/// [`cookbook::cook::scheduler::dep_levels`]): recipes in the same level have no +/// inter-dependencies and are cooked concurrently, up to `cook.cook_jobs` at a +/// time; levels are processed in order so a recipe never starts before its deps +/// finish. The `cook.jobs` make budget is divided across the concurrent cooks of +/// a level (a level with a single recipe still gets the full budget, so lone +/// heavyweight recipes like qtbase are not slowed down). Publishing stays a +/// single batched step at the caller. +/// +/// This is safe because each recipe builds in its own target/stage/sysroot and a +/// recipe's sysroot is assembled from its dependencies' (lower-level, already +/// complete) stage pkgars — read-only for concurrent siblings. +fn run_parallel_cook( + config: &CliConfig, + command: &CliCommand, + recipes: &[CookRecipe], +) -> Result<(), anyhow::Error> { + use cookbook::cook::scheduler::dep_levels; + use std::sync::atomic::AtomicUsize; + + let levels = dep_levels(recipes); + let max_level = levels.iter().copied().max().unwrap_or(0); + let cook_jobs = config.cook.cook_jobs.max(1); + let total_make = config.cook.jobs.max(1); + let nonstop = config.cook.nonstop; + let verbose = config.cook.verbose; + + let mut first_error: Option = None; + + for level in 0..=max_level { + let level_recipes: Vec<&CookRecipe> = recipes + .iter() + .enumerate() + .filter(|(i, _)| levels[*i] == level) + .map(|(_, r)| r) + .collect(); + if level_recipes.is_empty() { + continue; + } + + // Concurrency for this level, and the per-recipe make budget. A level + // with one recipe gets the full `total_make`; crowded levels share it so + // total parallelism stays ~`total_make` (no oversubscription). + let concurrency = cook_jobs.min(level_recipes.len()); + let per_make = (total_make / concurrency).max(1); + let mut level_config = config.clone(); + level_config.cook.jobs = per_make; + + let idx = AtomicUsize::new(0); + let results: std::sync::Mutex)>> = + std::sync::Mutex::new(Vec::new()); + { + let level_config = &level_config; + let level_recipes = &level_recipes; + let idx = &idx; + let results_ref = &results; + thread::scope(|s| { + for _ in 0..concurrency { + s.spawn(move || loop { + let i = idx.fetch_add(1, Ordering::SeqCst); + if i >= level_recipes.len() { + break; + } + let recipe = level_recipes[i]; + let r = repo_inner(level_config, command, recipe) + .map_err(|e| format!("{e:#}")); + results_ref.lock().unwrap().push((recipe.clone(), r)); + }); + } + }); + } + + // Report results and apply the same failure semantics as the serial loop. + let level_results = results.into_inner().unwrap(); + for (recipe, r) in &level_results { + match r { + Ok(cached) => { + if *cached { + print_cached(command, &recipe.name); + } else { + print_success(command, &recipe.name); + } + } + Err(e) => { + if nonstop { + if verbose { + eprintln!("repo: {e}"); + } + if let Err(e2) = handle_nonstop_fail(recipe) { + eprintln!("repo: {e2:#}"); + } + print_failed(command, &recipe.name); + } else { + print_failed(command, &recipe.name); + if first_error.is_none() { + first_error = Some(anyhow!("{e}")); + } + } + } + } + } + + // In stop mode, a failing level aborts the build (dependents would fail). + if !nonstop && first_error.is_some() { + break; + } + } + + match first_error { + Some(e) => Err(e), + None => Ok(()), + } +} + fn repo_inner( config: &CliConfig, command: &CliCommand, diff --git a/src/config.rs b/src/config.rs index ee19016444..7a038471ed 100644 --- a/src/config.rs +++ b/src/config.rs @@ -15,6 +15,10 @@ pub struct CookConfigOpt { pub offline: Option, /// whether to set jobs number instead of from nproc pub jobs: Option, + /// how many recipes to cook concurrently (dep-level scheduled). + /// 1 = serial (default). >1 cooks independent same-level recipes in + /// parallel, dividing the `jobs` make budget across the concurrent cooks. + pub cook_jobs: Option, /// whether to use TUI to allow parallel build /// default value is yes if "CI" env unset and STDIN is open. pub tui: Option, @@ -42,6 +46,7 @@ pub struct CookConfigOpt { pub struct CookConfig { pub offline: bool, pub jobs: usize, + pub cook_jobs: usize, pub tui: bool, pub logs: bool, pub nonstop: bool, @@ -58,6 +63,7 @@ impl From for CookConfig { CookConfig { offline: value.offline.unwrap(), jobs: value.jobs.unwrap(), + cook_jobs: value.cook_jobs.unwrap_or(1), tui: value.tui.unwrap(), logs: value.logs.unwrap(), nonstop: value.nonstop.unwrap(), @@ -108,6 +114,12 @@ pub fn init_config() { .unwrap_or(1), )); } + // How many recipes to cook concurrently (dep-level scheduled). Env overrides + // config; default 1 (serial) preserves the historical behavior. Only affects + // multi-recipe cooks (e.g. `cook --with-package-deps`). + config.cook_opt.cook_jobs = Some( + extract_env("COOKBOOK_COOK_JOBS", config.cook_opt.cook_jobs.unwrap_or(1)).max(1), + ); // --- Env-var override layer --- // Environment variables ALWAYS override cookbook.toml values. The TOML // value (or a sensible default) is used only when the env var is absent. diff --git a/src/cook.rs b/src/cook.rs index 0e5735366c..9d0c3b82f8 100644 --- a/src/cook.rs +++ b/src/cook.rs @@ -7,5 +7,6 @@ pub mod fs; pub mod ident; pub mod package; pub mod pty; +pub mod scheduler; pub mod script; pub mod tree; diff --git a/src/cook/package.rs b/src/cook/package.rs index 9d6ec94751..c77a33e98a 100644 --- a/src/cook/package.rs +++ b/src/cook/package.rs @@ -1,8 +1,18 @@ use std::{ collections::BTreeSet, path::{Path, PathBuf}, + sync::Mutex, }; +/// Serializes the packaging step across concurrent cooks (dep-level parallel +/// scheduling in the `cook` orchestrator). `package()` shares process-global +/// state — the pkgar signing key at `build/id_ed25519.toml` (created lazily on +/// first use) and package-name/metadata construction — that is not safe to run +/// from multiple threads at once. Packaging is cheap relative to compilation, so +/// serializing it keeps the parallelism win (concurrent builds) without the +/// races (corrupt signing key, mangled package names like "x.dev.dev"). +static PACKAGE_LOCK: Mutex<()> = Mutex::new(()); + use pkg::{InstallState, Package, PackageName, PackagePrefix, PackageState}; use pkgar::ext::PackageSrcExt; use pkgar_core::HeaderFlags; @@ -21,6 +31,9 @@ pub fn package( cook_config: &CookConfig, logger: &PtyOut, ) -> Result<(), String> { + // Only one recipe may package at a time (see PACKAGE_LOCK). Recover from a + // poisoned lock (a prior packaging panic) rather than cascading the panic. + let _pkg_guard = PACKAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let name = &recipe.name; let target_dir = &recipe.target_dir(); let auto_deps = &build_result.auto_deps; @@ -205,12 +218,13 @@ pub fn package_toml( ) })?; + let package_name = PackageName::new(get_package_name( + recipe.name.without_prefix(), + package_suffix, + )) + .map_err(|e| format!("recipe '{}': invalid package name: {e}", recipe.name.name()))?; let package = Package { - name: PackageName::new(get_package_name( - recipe.name.without_prefix(), - package_suffix, - )) - .unwrap(), + name: package_name, version, target: recipe.target.to_string(), blake3: hash,