Files
RedBear-OS/src/config.rs
T
vasilito 31c71859e1
redbear-ci / check (push) Has been cancelled
cook: dep-level parallel recipe cooking (concurrent independent recipes)
The cook orchestrator built recipes strictly serially — COOKBOOK_MAKE_JOBS only
parallelized within a single recipe's make, so cores sat idle during every
recipe's configure/link/small-file phases. src/cook/scheduler.rs (dep_levels)
was written for this but never wired in (the module wasn't even declared, so it
never compiled).

- Declare the scheduler module (src/cook.rs) so dep_levels + its 7 tests build.
- New COOKBOOK_COOK_JOBS setting (config.rs, default 1 = serial): how many
  recipes to cook concurrently.
- run_parallel_cook (repo.rs): partition recipes into topological dep-levels
  (same level = no inter-deps = safe concurrent), process levels in order behind
  a barrier, and within a level pull recipes from an atomic work-index across
   scoped worker threads. The make-job budget is divided per level so
  a lone heavyweight recipe (e.g. qtbase, alone at its level) still gets the full
  -j, while crowded levels share it — no oversubscription, no slowdown for
  serial-level recipes. Publishing stays a single batched step. Failure/nonstop
  semantics preserved; only engaged for  with cook_jobs>1 and >1 recipe.
- Safe because each recipe builds in its own target/stage/sysroot and a recipe's
  sysroot is assembled from its dependencies' (lower-level, already-complete)
  stage pkgars — read-only for concurrent siblings.
- package.rs: serialize packaging behind a global PACKAGE_LOCK. package() shares
  process-global state (the lazily-created pkgar signing key at
  build/id_ed25519.toml, name/metadata construction) that raced under concurrent
  cooks (corrupt key, mangled names like 'x.dev.dev'); packaging is cheap, so
  serializing it keeps the compile-parallelism win. Also make the package-name
  construction fail gracefully (Err, not unwrap-panic) so a bad name fails one
  recipe instead of crashing the whole scoped-thread build.

Validated: all-cached tree (COOK_JOBS=4) and a real concurrent build of 3
independent recipes (COOK_JOBS=3) both succeed and publish correctly.
2026-08-02 07:32:53 +03:00

290 lines
9.7 KiB
Rust

use std::{
collections::HashMap,
env, fs,
io::{IsTerminal, stdin},
str::FromStr,
sync::OnceLock,
};
use serde::{Deserialize, Serialize};
#[derive(Debug, Default, Clone, Deserialize, PartialEq, Serialize)]
#[serde(default)]
pub struct CookConfigOpt {
/// whether to run offline
pub offline: Option<bool>,
/// whether to set jobs number instead of from nproc
pub jobs: Option<usize>,
/// how many recipes to cook concurrently (dep-level scheduled).
/// 1 = serial (default). >1 cooks independent same-level recipes in
/// parallel, dividing the `jobs` make budget across the concurrent cooks.
pub cook_jobs: Option<usize>,
/// whether to use TUI to allow parallel build
/// default value is yes if "CI" env unset and STDIN is open.
pub tui: Option<bool>,
/// whether to write logs to build/logs dir, default true on TUI
pub logs: Option<bool>,
/// whether to ignore build errors
pub nonstop: Option<bool>,
/// whether to archive packages with compressed format
pub compressed: Option<bool>,
/// whether to print verbose logs to certain commands
/// build failure still be printed anyway
pub verbose: Option<bool>,
/// whether to always clean the build directory before building
pub clean_build: Option<bool>,
/// whether to always clean the target directory after building
/// (deletes everything except pkgar files)
pub clean_target: Option<bool>,
/// whether to always write stage.files metadata
pub write_filetree: Option<bool>,
/// whether to force rebuild ignoring content-hash cache
pub force_rebuild: Option<bool>,
}
#[derive(Debug, Default, Clone, Deserialize, PartialEq, Serialize)]
pub struct CookConfig {
pub offline: bool,
pub jobs: usize,
pub cook_jobs: usize,
pub tui: bool,
pub logs: bool,
pub nonstop: bool,
pub compressed: bool,
pub verbose: bool,
pub clean_build: bool,
pub clean_target: bool,
pub write_filetree: bool,
pub force_rebuild: bool,
}
impl From<CookConfigOpt> for CookConfig {
fn from(value: CookConfigOpt) -> Self {
CookConfig {
offline: value.offline.unwrap(),
jobs: value.jobs.unwrap(),
cook_jobs: value.cook_jobs.unwrap_or(1),
tui: value.tui.unwrap(),
logs: value.logs.unwrap(),
nonstop: value.nonstop.unwrap(),
compressed: value.compressed.unwrap(),
verbose: value.verbose.unwrap(),
clean_build: value.clean_build.unwrap(),
clean_target: value.clean_target.unwrap(),
write_filetree: value.write_filetree.unwrap(),
force_rebuild: value.force_rebuild.unwrap_or(false),
}
}
}
#[derive(Debug, Default, Deserialize, PartialEq, Serialize)]
#[serde(default)]
pub struct CookbookConfig {
#[serde(rename = "cook")]
cook_opt: CookConfigOpt,
#[serde(skip)]
pub cook: CookConfig,
pub mirrors: HashMap<String, String>,
}
static CONFIG: OnceLock<CookbookConfig> = OnceLock::new();
pub fn init_config() {
let mut config: CookbookConfig = if fs::exists("cookbook.toml").unwrap_or(false) {
let toml_content = fs::read_to_string("cookbook.toml")
.map_err(|e| format!("Unable to read config: {:?}", e))
.unwrap();
toml::from_str(&toml_content)
.map_err(|e| format!("Unable to parse config: {:?}", e))
.unwrap()
} else {
CookbookConfig::default()
};
if config.cook_opt.tui.is_none() {
let ci_set = env::var("CI").is_ok_and(|s| !s.is_empty());
let tty = stdin().is_terminal();
config.cook_opt.tui = Some(!ci_set && tty);
}
if config.cook_opt.jobs.is_none() {
config.cook_opt.jobs = Some(extract_env(
"COOKBOOK_MAKE_JOBS",
std::thread::available_parallelism()
.map(usize::from)
.unwrap_or(1),
));
}
// How many recipes to cook concurrently (dep-level scheduled). Env overrides
// config; default 1 (serial) preserves the historical behavior. Only affects
// multi-recipe cooks (e.g. `cook --with-package-deps`).
config.cook_opt.cook_jobs = Some(
extract_env("COOKBOOK_COOK_JOBS", config.cook_opt.cook_jobs.unwrap_or(1)).max(1),
);
// --- Env-var override layer ---
// Environment variables ALWAYS override cookbook.toml values. The TOML
// value (or a sensible default) is used only when the env var is absent.
// This follows the standard "env > config-file > built-in-default" hierarchy.
config.cook_opt.logs = Some(extract_env(
"COOKBOOK_LOGS",
config.cook_opt.logs.unwrap_or(config.cook_opt.tui.unwrap()),
));
config.cook_opt.offline = Some(extract_env(
"COOKBOOK_OFFLINE",
config.cook_opt.offline.unwrap_or(true),
));
config.cook_opt.compressed = Some(extract_env(
"COOKBOOK_COMPRESSED",
config.cook_opt.compressed.unwrap_or(false),
));
config.cook_opt.verbose = Some(extract_env(
"COOKBOOK_VERBOSE",
config.cook_opt.verbose.unwrap_or(true),
));
config.cook_opt.nonstop = Some(extract_env(
"COOKBOOK_NONSTOP",
config.cook_opt.nonstop.unwrap_or(false),
));
config.cook_opt.clean_build = Some(extract_env(
"COOKBOOK_CLEAN_BUILD",
config.cook_opt.clean_build.unwrap_or(false),
));
config.cook_opt.clean_target = Some(extract_env(
"COOKBOOK_CLEAN_TARGET",
config.cook_opt.clean_target.unwrap_or(false),
));
config.cook_opt.write_filetree = Some(extract_env(
"COOKBOOK_WRITE_FILETREE",
config.cook_opt.write_filetree.unwrap_or(
config.cook_opt.clean_target.unwrap_or(false) || extract_env("COOKBOOK_WEB", false),
),
));
if config.mirrors.is_empty() {
// The GNU FTP mirror below is automatically inserted for convenience
// You can choose other mirrors by setting it on cookbook.toml
config.mirrors.insert(
"ftp.gnu.org/gnu".to_string(),
"mirrors.ocf.berkeley.edu/gnu".to_string(),
);
}
config.cook = CookConfig::from(config.cook_opt.clone());
CONFIG.set(config).expect("config is initialized twice");
}
fn extract_env<T: FromStr>(key: &str, default: T) -> T {
if let Ok(e) = env::var(key) {
str::parse(&e).unwrap_or(default)
} else {
default
}
}
pub fn get_config() -> &'static CookbookConfig {
CONFIG.get().expect("Configuration is not initialized")
}
pub fn translate_mirror(original_url: &str) -> String {
let config = CONFIG.get().expect("Configuration is not initialized");
let stripped_url = original_url
.strip_prefix("https://")
.or_else(|| original_url.strip_prefix("http://"))
.unwrap_or(original_url);
let mut best_match_prefix: Option<&String> = None;
for prefix in config.mirrors.keys() {
if stripped_url.starts_with(prefix) {
match best_match_prefix {
Some(current_best) if prefix.len() > current_best.len() => {
best_match_prefix = Some(prefix);
}
None => {
best_match_prefix = Some(prefix);
}
_ => {}
}
}
}
if let Some(prefix) = best_match_prefix {
let mirror_base = config.mirrors.get(prefix).unwrap();
let suffix = &stripped_url[prefix.len()..];
let ptotocol = &original_url[..(original_url.len() - stripped_url.len())];
return format!("{}{}{}", ptotocol, mirror_base, suffix);
}
original_url.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
fn setup_test_config() {
let app_config = toml::from_str(
"[mirrors]\n\
\"ftp.gnu.org/gnu\" = \"example.com/gnu\"\n\
\"github.com/foo/bar\" = \"github.com/baz/bar\"\n\
\"github.com/a\" = \"github.com/b\"\n",
)
.expect("Unable to parse test config");
// This will be called for each test. If the config is already set,
// it will do nothing, which is fine as all tests use the same config.
let _ = CONFIG.set(app_config);
}
#[test]
fn test_parse_cook() {
let app_config: CookbookConfig = toml::from_str(
"[cook]\n\
offline = true\n",
)
.expect("Unable to parse test config");
assert_eq!(app_config.cook_opt.offline, Some(true));
assert_eq!(app_config.cook_opt.jobs, None);
}
#[test]
fn test_exact_match() {
setup_test_config();
assert_eq!(translate_mirror("ftp.gnu.org/gnu"), "example.com/gnu");
assert_eq!(translate_mirror("github.com/foo/bar"), "github.com/baz/bar");
}
#[test]
fn test_prefix_match() {
setup_test_config();
assert_eq!(
translate_mirror("https://github.com/a/c"),
"https://github.com/b/c"
);
assert_eq!(
translate_mirror("https://ftp.gnu.org/gnu/bash/bash-5.2.15.tar.gz"),
"https://example.com/gnu/bash/bash-5.2.15.tar.gz"
);
}
#[test]
fn test_longest_prefix_match() {
setup_test_config();
// "github.com/foo/bar" is a longer and more specific prefix than "github.com/a",
// so it should be chosen for the translation.
assert_eq!(
translate_mirror("https://github.com/foo/bar/baz"),
"https://github.com/baz/bar/baz"
);
}
#[test]
fn test_no_match() {
setup_test_config();
assert_eq!(translate_mirror("www.rust-lang.org"), "www.rust-lang.org");
assert_eq!(
translate_mirror("http://github.com/unrelated/repo"),
"http://github.com/unrelated/repo"
);
}
}