cook: dep-level parallel recipe cooking (concurrent independent recipes)

The cook orchestrator built recipes strictly serially — COOKBOOK_MAKE_JOBS only
parallelized within a single recipe's make, so cores sat idle during every
recipe's configure/link/small-file phases. src/cook/scheduler.rs (dep_levels)
was written for this but never wired in (the module wasn't even declared, so it
never compiled).

- Declare the scheduler module (src/cook.rs) so dep_levels + its 7 tests build.
- New COOKBOOK_COOK_JOBS setting (config.rs, default 1 = serial): how many
  recipes to cook concurrently.
- run_parallel_cook (repo.rs): partition recipes into topological dep-levels
  (same level = no inter-deps = safe concurrent), process levels in order behind
  a barrier, and within a level pull recipes from an atomic work-index across
   scoped worker threads. The make-job budget is divided per level so
  a lone heavyweight recipe (e.g. qtbase, alone at its level) still gets the full
  -j, while crowded levels share it — no oversubscription, no slowdown for
  serial-level recipes. Publishing stays a single batched step. Failure/nonstop
  semantics preserved; only engaged for  with cook_jobs>1 and >1 recipe.
- Safe because each recipe builds in its own target/stage/sysroot and a recipe's
  sysroot is assembled from its dependencies' (lower-level, already-complete)
  stage pkgars — read-only for concurrent siblings.
- package.rs: serialize packaging behind a global PACKAGE_LOCK. package() shares
  process-global state (the lazily-created pkgar signing key at
  build/id_ed25519.toml, name/metadata construction) that raced under concurrent
  cooks (corrupt key, mangled names like 'x.dev.dev'); packaging is cheap, so
  serializing it keeps the compile-parallelism win. Also make the package-name
  construction fail gracefully (Err, not unwrap-panic) so a bad name fails one
  recipe instead of crashing the whole scoped-thread build.

Validated: all-cached tree (COOK_JOBS=4) and a real concurrent build of 3
independent recipes (COOK_JOBS=3) both succeed and publish correctly.

src/bin/repo.rs

@@ -277,29 +277,35 @@ fn main_inner() -> anyhow::Result<()> {
     }
 
     let verbose = config.cook.verbose;
-    for recipe in &recipes {
-        match repo_inner(&config, &command, recipe) {
-            Ok(cached) => {
-                if !command.is_informational() {
-                    if cached {
-                        print_cached(&command, &recipe.name);
-                    } else {
-                        print_success(&command, &recipe.name);
+    // Dep-level parallel cooking: only for `cook` with concurrency requested and
+    // more than one recipe. Everything else keeps the serial path.
+    if command == CliCommand::Cook && config.cook.cook_jobs > 1 && recipes.len() > 1 {
+        run_parallel_cook(&config, &command, &recipes)?;
+    } else {
+        for recipe in &recipes {
+            match repo_inner(&config, &command, recipe) {
+                Ok(cached) => {
+                    if !command.is_informational() {
+                        if cached {
+                            print_cached(&command, &recipe.name);
+                        } else {
+                            print_success(&command, &recipe.name);
+                        }
                     }
                 }
-            }
-            Err(e) => {
-                if config.cook.nonstop {
-                    if verbose {
-                        eprintln!("repo: {e:#}");
+                Err(e) => {
+                    if config.cook.nonstop {
+                        if verbose {
+                            eprintln!("repo: {e:#}");
+                        }
+                        if let Err(e) = handle_nonstop_fail(recipe) {
+                            eprintln!("repo: {e:#}")
+                        };
+                    }
+                    print_failed(&command, &recipe.name);
+                    if !config.cook.nonstop {
+                        return Err(e);
                     }
-                    if let Err(e) = handle_nonstop_fail(recipe) {
-                        eprintln!("repo: {e:#}")
-                    };
-                }
-                print_failed(&command, &recipe.name);
-                if !config.cook.nonstop {
-                    return Err(e);
                 }
             }
         }
@@ -355,6 +361,122 @@ fn print_cached(command: &CliCommand, recipe: &PackageName) {
     );
 }
 
+/// Cook a set of recipes with dep-level-aware concurrency.
+///
+/// Recipes are partitioned into topological levels (see
+/// [`cookbook::cook::scheduler::dep_levels`]): recipes in the same level have no
+/// inter-dependencies and are cooked concurrently, up to `cook.cook_jobs` at a
+/// time; levels are processed in order so a recipe never starts before its deps
+/// finish. The `cook.jobs` make budget is divided across the concurrent cooks of
+/// a level (a level with a single recipe still gets the full budget, so lone
+/// heavyweight recipes like qtbase are not slowed down). Publishing stays a
+/// single batched step at the caller.
+///
+/// This is safe because each recipe builds in its own target/stage/sysroot and a
+/// recipe's sysroot is assembled from its dependencies' (lower-level, already
+/// complete) stage pkgars — read-only for concurrent siblings.
+fn run_parallel_cook(
+    config: &CliConfig,
+    command: &CliCommand,
+    recipes: &[CookRecipe],
+) -> Result<(), anyhow::Error> {
+    use cookbook::cook::scheduler::dep_levels;
+    use std::sync::atomic::AtomicUsize;
+
+    let levels = dep_levels(recipes);
+    let max_level = levels.iter().copied().max().unwrap_or(0);
+    let cook_jobs = config.cook.cook_jobs.max(1);
+    let total_make = config.cook.jobs.max(1);
+    let nonstop = config.cook.nonstop;
+    let verbose = config.cook.verbose;
+
+    let mut first_error: Option<anyhow::Error> = None;
+
+    for level in 0..=max_level {
+        let level_recipes: Vec<&CookRecipe> = recipes
+            .iter()
+            .enumerate()
+            .filter(|(i, _)| levels[*i] == level)
+            .map(|(_, r)| r)
+            .collect();
+        if level_recipes.is_empty() {
+            continue;
+        }
+
+        // Concurrency for this level, and the per-recipe make budget. A level
+        // with one recipe gets the full `total_make`; crowded levels share it so
+        // total parallelism stays ~`total_make` (no oversubscription).
+        let concurrency = cook_jobs.min(level_recipes.len());
+        let per_make = (total_make / concurrency).max(1);
+        let mut level_config = config.clone();
+        level_config.cook.jobs = per_make;
+
+        let idx = AtomicUsize::new(0);
+        let results: std::sync::Mutex<Vec<(CookRecipe, Result<bool, String>)>> =
+            std::sync::Mutex::new(Vec::new());
+        {
+            let level_config = &level_config;
+            let level_recipes = &level_recipes;
+            let idx = &idx;
+            let results_ref = &results;
+            thread::scope(|s| {
+                for _ in 0..concurrency {
+                    s.spawn(move || loop {
+                        let i = idx.fetch_add(1, Ordering::SeqCst);
+                        if i >= level_recipes.len() {
+                            break;
+                        }
+                        let recipe = level_recipes[i];
+                        let r = repo_inner(level_config, command, recipe)
+                            .map_err(|e| format!("{e:#}"));
+                        results_ref.lock().unwrap().push((recipe.clone(), r));
+                    });
+                }
+            });
+        }
+
+        // Report results and apply the same failure semantics as the serial loop.
+        let level_results = results.into_inner().unwrap();
+        for (recipe, r) in &level_results {
+            match r {
+                Ok(cached) => {
+                    if *cached {
+                        print_cached(command, &recipe.name);
+                    } else {
+                        print_success(command, &recipe.name);
+                    }
+                }
+                Err(e) => {
+                    if nonstop {
+                        if verbose {
+                            eprintln!("repo: {e}");
+                        }
+                        if let Err(e2) = handle_nonstop_fail(recipe) {
+                            eprintln!("repo: {e2:#}");
+                        }
+                        print_failed(command, &recipe.name);
+                    } else {
+                        print_failed(command, &recipe.name);
+                        if first_error.is_none() {
+                            first_error = Some(anyhow!("{e}"));
+                        }
+                    }
+                }
+            }
+        }
+
+        // In stop mode, a failing level aborts the build (dependents would fail).
+        if !nonstop && first_error.is_some() {
+            break;
+        }
+    }
+
+    match first_error {
+        Some(e) => Err(e),
+        None => Ok(()),
+    }
+}
+
 fn repo_inner(
     config: &CliConfig,
     command: &CliCommand,

src/config.rs

@@ -15,6 +15,10 @@ pub struct CookConfigOpt {
     pub offline: Option<bool>,
     /// whether to set jobs number instead of from nproc
     pub jobs: Option<usize>,
+    /// how many recipes to cook concurrently (dep-level scheduled).
+    /// 1 = serial (default). >1 cooks independent same-level recipes in
+    /// parallel, dividing the `jobs` make budget across the concurrent cooks.
+    pub cook_jobs: Option<usize>,
     /// whether to use TUI to allow parallel build
     /// default value is yes if "CI" env unset and STDIN is open.
     pub tui: Option<bool>,
@@ -42,6 +46,7 @@ pub struct CookConfigOpt {
 pub struct CookConfig {
     pub offline: bool,
     pub jobs: usize,
+    pub cook_jobs: usize,
     pub tui: bool,
     pub logs: bool,
     pub nonstop: bool,
@@ -58,6 +63,7 @@ impl From<CookConfigOpt> for CookConfig {
         CookConfig {
             offline: value.offline.unwrap(),
             jobs: value.jobs.unwrap(),
+            cook_jobs: value.cook_jobs.unwrap_or(1),
             tui: value.tui.unwrap(),
             logs: value.logs.unwrap(),
             nonstop: value.nonstop.unwrap(),
@@ -108,6 +114,12 @@ pub fn init_config() {
                 .unwrap_or(1),
         ));
     }
+    // How many recipes to cook concurrently (dep-level scheduled). Env overrides
+    // config; default 1 (serial) preserves the historical behavior. Only affects
+    // multi-recipe cooks (e.g. `cook --with-package-deps`).
+    config.cook_opt.cook_jobs = Some(
+        extract_env("COOKBOOK_COOK_JOBS", config.cook_opt.cook_jobs.unwrap_or(1)).max(1),
+    );
     // --- Env-var override layer ---
     // Environment variables ALWAYS override cookbook.toml values.  The TOML
     // value (or a sensible default) is used only when the env var is absent.

src/cook.rs

@@ -7,5 +7,6 @@ pub mod fs;
 pub mod ident;
 pub mod package;
 pub mod pty;
+pub mod scheduler;
 pub mod script;
 pub mod tree;

src/cook/package.rs

@@ -1,8 +1,18 @@
 use std::{
     collections::BTreeSet,
     path::{Path, PathBuf},
+    sync::Mutex,
 };
 
+/// Serializes the packaging step across concurrent cooks (dep-level parallel
+/// scheduling in the `cook` orchestrator). `package()` shares process-global
+/// state — the pkgar signing key at `build/id_ed25519.toml` (created lazily on
+/// first use) and package-name/metadata construction — that is not safe to run
+/// from multiple threads at once. Packaging is cheap relative to compilation, so
+/// serializing it keeps the parallelism win (concurrent builds) without the
+/// races (corrupt signing key, mangled package names like "x.dev.dev").
+static PACKAGE_LOCK: Mutex<()> = Mutex::new(());
+
 use pkg::{InstallState, Package, PackageName, PackagePrefix, PackageState};
 use pkgar::ext::PackageSrcExt;
 use pkgar_core::HeaderFlags;
@@ -21,6 +31,9 @@ pub fn package(
     cook_config: &CookConfig,
     logger: &PtyOut,
 ) -> Result<(), String> {
+    // Only one recipe may package at a time (see PACKAGE_LOCK). Recover from a
+    // poisoned lock (a prior packaging panic) rather than cascading the panic.
+    let _pkg_guard = PACKAGE_LOCK.lock().unwrap_or_else(|e| e.into_inner());
     let name = &recipe.name;
     let target_dir = &recipe.target_dir();
     let auto_deps = &build_result.auto_deps;
@@ -205,12 +218,13 @@ pub fn package_toml(
         )
     })?;
 
+    let package_name = PackageName::new(get_package_name(
+        recipe.name.without_prefix(),
+        package_suffix,
+    ))
+    .map_err(|e| format!("recipe '{}': invalid package name: {e}", recipe.name.name()))?;
     let package = Package {
-        name: PackageName::new(get_package_name(
-            recipe.name.without_prefix(),
-            package_suffix,
-        ))
-        .unwrap(),
+        name: package_name,
         version,
         target: recipe.target.to_string(),
         blake3: hash,