Files
RedBear-OS/local/scripts/audit-kf6-deps.py
T
vasilito ffbe098ef8 config: add tlc to redbear-mini and redbear-full; create recipe symlink
TLC (Twilight Commander) was missing from both ISO configs. Added
tlc = {} to [packages] in redbear-mini.toml and redbear-full.toml.
Created missing symlink: recipes/tui/tlc -> ../../local/recipes/tui/tlc.
2026-06-19 11:47:25 +03:00

558 lines
21 KiB
Python
Executable File

#!/usr/bin/env python3
"""Audit every KF6/Qt recipe's [build].dependencies against what its source
actually requires.
For each recipe under local/recipes/kde/ and recipes/, this script:
1. Resolves the upstream source (git or tar) at the pinned rev
2. Extracts/reads the source's CMakeLists.txt + all .cmake files
3. Greps for `find_package(KF6::* COMPONENTS ...)` and `find_package(Qt6* ...)` calls
4. Reads the recipe's [build].dependencies array
5. Reports any KF6::/Qt* component referenced in the source but missing
from the recipe's dependencies, AND any recipe dependency that is
unused (i.e. not referenced by the source)
6. Optionally: emits a fixed [build].dependencies array as a patch
Per AGENTS.md "BUILD DURABILITY" policy, the recipe.toml is the durable
artifact. This audit ensures the recipe matches what the source actually
needs, preventing "Package 'X' not found" failures at cook time.
Usage:
./local/scripts/audit-kf6-deps.py --verbose # audit all 46 recipes
./local/scripts/audit-kf6-deps.py --component kf6-kio
./local/scripts/audit-kf6-deps.py --fix --dry-run # show proposed fix
./local/scripts/audit-kf6-deps.py --fix # write the fix in place
"""
import argparse
import re
import shutil
import subprocess
import sys
import tempfile
import time
import tomllib
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
LOCAL_RECIPES = PROJECT_ROOT / "local/recipes"
MAINLINE_RECIPES = PROJECT_ROOT / "recipes"
# KF6 components appear in three forms in upstream KDE source. The
# dominant form, used by ~99% of KF6 code, is the named form: one
# find_package call per line, with the component name spelled out in
# full. The other two forms are occasional variants we still need
# to catch.
KF6_DIRECT_RE = re.compile(
r"find_package\s*\(\s*(KF6[A-Za-z]*(?:::[A-Za-z0-9]+)+)"
)
KF6_COMPONENTS_BLOCK_RE = re.compile(
r"find_package\s*\(\s*KF6\b[^\)]*?COMPONENTS[^\)]*\)"
)
KF6_NAMED_RE = re.compile(
r"find_package\s*\(\s*(KF6[A-Z][A-Za-z0-9]+)\b"
)
# Form 4: find_package(KF6 <Name> REQUIRED) — rare in modern KDE code but
# exists in some older Plasma components and a handful of KF6 addons.
KF6_PLAIN_NAME_RE = re.compile(
r"find_package\s*\(\s*KF6\s+([A-Z][A-Za-z0-9]+)\b"
)
KF6_COMPONENT_TOKEN_RE = re.compile(
r"\b([A-Z][A-Za-z0-9]+)\b"
)
# Qt6 components: find_package(Qt6Foo REQUIRED) — usually individual modules
QT6_COMPONENT_RE = re.compile(
r"find_package\s*\(\s*Qt6([A-Z][A-Za-z0-9]*)"
)
# Also catch the more explicit form
# find_package(Qt6 5.15.0 COMPONENTS Core Network DBus)
QT6_GENERIC_RE = re.compile(
r"find_package\s*\(\s*Qt6\b"
)
QT6_COMPONENTS_BLOCK_RE = re.compile(
r"find_package\s*\(\s*Qt6\b[^\)]*?COMPONENTS[^\n]+"
)
def run(cmd, **kwargs):
proc = subprocess.run(cmd, capture_output=True, text=True,
check=False, **kwargs)
return proc.returncode, proc.stdout, proc.stderr
def _strip_cmake_noise(text: str) -> str:
"""Strip CMake comments and string literals from source before regex.
CMake comments are introduced by `#` and run to end of line; string
literals are double-quoted with backslash escapes. Without this
pass, code like `set(MY_NOTE "needs find_package(KF6::Foo) later")`
or `# find_package(KF6::FakeUsedOnlyInComment)` would be falsely
classified as a real dependency reference.
"""
text = re.sub(r"(?m)#.*$", "", text)
text = re.sub(r'"(?:\\.|[^"\\])*"', '""', text)
return text
def fetch_source(recipe_toml: Path):
"""Fetch the upstream source at the pinned rev into a tempdir.
Returns (Path, error_msg)."""
with open(recipe_toml, "rb") as f:
data = tomllib.load(f)
source = data.get("source") or {}
url = source.get("git")
rev = source.get("rev")
tar = source.get("tar")
tmp = Path(tempfile.mkdtemp(prefix="audit-kf6-"))
if tar:
# tar-based: download to tmp/tarball, extract into tmp/src
tarball = tmp / "src.tar.xz"
rc, _, err = run(["curl", "-sSL", "-o", str(tarball), tar])
if rc != 0:
return None, f"download failed: {err}"
extract_dir = tmp / "src"
extract_dir.mkdir()
rc, _, err = run(["tar", "-xJf", str(tarball), "-C", str(extract_dir)])
if rc != 0:
return None, f"extract failed: {err}"
# The tarball may have a top-level dir; find it
candidates = list(extract_dir.iterdir())
if len(candidates) == 1 and candidates[0].is_dir():
return candidates[0], None
return extract_dir, None
elif url and rev:
# git-based: clone at the pinned rev
rc, _, err = run(["git", "clone", "--quiet", "--no-checkout",
url, str(tmp / "src")])
if rc != 0:
return None, f"clone failed: {err}"
rc, _, err = run(["git", "-C", str(tmp / "src"), "checkout", "--quiet", rev])
if rc != 0:
return None, f"checkout failed: {err}"
return tmp / "src", None
else:
return None, "no git or tar source"
def scan_source(source_dir: Path):
"""Walk the source tree and extract every KF6:: and Qt6 component used.
Returns (set_of_kf6_components, set_of_qt6_components).
"""
kf6 = set()
qt6 = set()
for cmake_file in list(source_dir.rglob("CMakeLists.txt")) + \
list(source_dir.rglob("*.cmake")):
try:
text = cmake_file.read_text(errors="replace")
except OSError:
continue
text = _strip_cmake_noise(text)
# Form 1: find_package(KF6::Foo REQUIRED) — rare, mostly Plasma
for m in KF6_DIRECT_RE.finditer(text):
kf6.add(m.group(1))
# Form 2: find_package(KF6 COMPONENTS Foo Bar Baz) — all in one
for m in KF6_COMPONENTS_BLOCK_RE.finditer(text):
line = m.group(0)
for tok in KF6_COMPONENT_TOKEN_RE.findall(line):
if tok in ("REQUIRED", "QUIET", "COMPONENTS", "CONFIG",
"VERSION", "EXACT", "QUIETLY", "MODULE", "KF6"):
continue
kf6.add(f"KF6::{tok}")
# Form 3 (the dominant KDE form): find_package(KF6Xxx REQUIRED).
# The full name is captured without the "KF6" prefix, then
# normalized to the KF6::Foo form so normalize_dep_name handles
# it like every other KF6 component. We filter out self-references
# like "KF6KF6" (which the regex would otherwise mis-capture).
for m in KF6_NAMED_RE.finditer(text):
rest = m.group(1)[len("KF6"):]
if rest.startswith("KF6") or not rest:
continue
kf6.add(f"KF6::{rest}")
# Form 4: find_package(KF6 <Name> REQUIRED) — rare, but emitted
# by some older Plasma components. The capture is the bare name.
for m in KF6_PLAIN_NAME_RE.finditer(text):
kf6.add(f"KF6::{m.group(1)}")
# Qt6 individual modules: find_package(Qt6Foo REQUIRED)
for m in QT6_COMPONENT_RE.finditer(text):
qt6.add(f"Qt6{m.group(1)}")
# Qt6 block form: find_package(Qt6 5.15.0 COMPONENTS Core Network DBus)
for m in QT6_COMPONENTS_BLOCK_RE.finditer(text):
line = m.group(0)
for tok in KF6_COMPONENT_TOKEN_RE.findall(line):
if tok in ("REQUIRED", "QUIET", "COMPONENTS", "CONFIG",
"VERSION", "EXACT", "QUIETLY", "MODULE",
"Qt6"):
continue
qt6.add(f"Qt6{tok}")
# Plain find_package(Qt6 ...) without components — minimal
if QT6_GENERIC_RE.search(text):
qt6.add("Qt6Core")
return kf6, qt6
def read_recipe_deps(recipe_toml: Path):
"""Return (set_of_dep_names, raw_deps_text)."""
with open(recipe_toml, "rb") as f:
data = tomllib.load(f)
build = data.get("build") or {}
raw = build.get("dependencies") or []
return {d.strip() for d in raw}, raw
KF6_RECIPE_OVERRIDES = {
"Archive": "karchive",
"Attica": "attica",
"Auth": "kauth",
"Bookmarks": "kbookmarks",
"Codecs": "kcodecs",
"ColorScheme": "kcolorscheme",
"Completion": "kcompletion",
"Config": "kconfig",
"ConfigWidgets": "kconfigwidgets",
"CoreAddons": "kcoreaddons",
"Crash": "kcrash",
"DBusAddons": "kdbusaddons",
"Declarative": "kdeclarative",
"DocTools": "kdoctools",
"GuiAddons": "kguiaddons",
"GlobalAccel": "kglobalaccel",
"I18n": "ki18n",
"IconThemes": "kiconthemes",
"IdleTime": "kidletime",
"ImageFormats": "kimageformats",
"ItemModels": "kitemmodels",
"ItemViews": "kitemviews",
"JobWidgets": "kjobwidgets",
"KCMUtils": "kcmutils",
"KDED": "kded6",
"KIO": "kio",
"KNewStuff": "knewstuff",
"KNotifyConfig": "notifyconfig",
"Notifications": "knotifications",
"KPackage": "kpackage",
"Parts": "parts",
"Plasma": "plasma",
"Prison": "prison",
"Pty": "pty",
"Service": "kservice",
"Solid": "solid",
"Sonnet": "sonnet",
"Svg": "ksvg",
"SyntaxHighlighting": "syntaxhighlighting",
"TextEditor": "ktexteditor",
"TextWidgets": "ktextwidgets",
"Wallet": "kwallet",
"Wayland": "kwayland",
"WidgetsAddons": "kwidgetsaddons",
"WindowSystem": "kwindowsystem",
"XmlGui": "kxmlgui",
"ExtraCMakeModules": "extra-cmake-modules",
}
def normalize_dep_name(component: str) -> str:
"""Map a CMake KF6::Foo / Qt6Bar reference to a Red Bear OS recipe name.
Examples:
KF6::KIO -> kf6-kio
KF6::KCMUtils -> kf6-kcmutils
KF6::IconThemes -> kf6-kiconthemes
Qt6Core -> qtbase
Qt6Gui -> qtbase
Qt6GuiPrivate -> qtbase
Qt6Qml -> qtdeclarative
"""
if component.startswith("KF6::"):
rest = component[len("KF6::"):]
if rest in KF6_RECIPE_OVERRIDES:
return f"kf6-{KF6_RECIPE_OVERRIDES[rest]}"
s = re.sub(r"(?<!^)(?=[A-Z])", "-", rest).lower()
return f"kf6-{s}"
if component.startswith("Qt6"):
rest = component[len("Qt6"):]
rest_lower = rest.lower()
# Map specific Qt6 modules to Red Bear OS recipes
qtbase_modules = {"core", "gui", "guilib", "guifunctions",
"widgets", "network", "dbus", "test",
"concurrent", "printsupport", "qpa",
"xml", "opengl", "sql", "svgwidgets",
"testlib", "platform", "platformsupport",
"platformheaders", "platformheadersclean",
"guiprivate", "widgetstyles", "statemachine"}
qtdeclarative_modules = {"qml", "qmltest", "qmlintegration",
"qmlworkerscript", "qmlmodels", "qmlcore",
"quick", "quickcontrols", "quickcontrols2",
"quickparticles", "quickwidgets",
"quicktest"}
if rest_lower in qtbase_modules:
return "qtbase"
if rest_lower in qtdeclarative_modules:
return "qtdeclarative"
if rest_lower == "svg":
return "qtsvg"
if rest_lower == "wayland":
return "qtwayland"
return f"qt6-{rest_lower}" # generic
return component.lower()
def audit_recipe(recipe_toml: Path, verbose: bool = False):
"""Audit a single recipe. Return (missing_deps, unused_deps, error_msg)."""
fetched = fetch_source(recipe_toml)
if fetched[1]:
return set(), set(), fetched[1]
source_dir: Path = fetched[0] # type: ignore[assignment]
kf6_components, qt6_components = scan_source(source_dir)
if verbose:
print(f" {recipe_toml.relative_to(PROJECT_ROOT)}: "
f"uses KF6={sorted(kf6_components)}, "
f"Qt6={sorted(qt6_components)}")
recipe_deps, raw = read_recipe_deps(recipe_toml)
# Map cmake component names to recipe names
needed_recipes = set()
for c in kf6_components | qt6_components:
needed_recipes.add(normalize_dep_name(c))
# Standard transitive deps every KF6/Qt6 recipe needs (don't flag
# these as "unused" if the recipe omits them — they're common
# infrastructure):
standard_infra = {
"qtbase", # every Qt6 recipe needs qtbase
"qtdeclarative", # often transitive via Qt6Core
"kf6-extra-cmake-modules", # KDE/Qt6 build system
"kf6-kf6", # kf6-umbrella, only a build marker
}
missing = needed_recipes - recipe_deps - standard_infra
unused = (recipe_deps - needed_recipes) - standard_infra
# Cleanup: remove the recipe's own name (some recipes list themselves)
own_name = recipe_toml.parent.name
missing.discard(own_name)
unused.discard(own_name)
# Cleanup
shutil.rmtree(source_dir.parent, ignore_errors=True)
return missing, unused, None
def discover_kf6_recipes(component_filter=None):
"""Yield every KF6 recipe path (local + mainline, excluding WIP).
Per local/AGENTS.md "Local recipe priority vs upstream WIP", the
local fork in `local/recipes/` is the source of truth when both
exist. The mainline tree under `recipes/wip/` is transitional and
not part of the durable shipping surface, so we skip it.
"""
for recipes_root in (LOCAL_RECIPES, MAINLINE_RECIPES):
if not recipes_root.is_dir():
continue
for recipe_toml in recipes_root.rglob("recipe.toml"):
if "source" in recipe_toml.parts or "target" in recipe_toml.parts:
continue
if "wip" in recipe_toml.parts:
continue
# Only KF6 + Qt recipes
name = recipe_toml.parent.name
if not (name.startswith("kf6-") or name.startswith("qt")):
continue
if component_filter and name != component_filter:
continue
yield recipe_toml
def main():
parser = argparse.ArgumentParser(
description=(
"Audit every KF6/Qt recipe's [build].dependencies against "
"what its source actually requires."
)
)
parser.add_argument("--component", help="Audit only this recipe")
parser.add_argument("--verbose", "-v", action="store_true",
help="Print every recipe's component usage")
parser.add_argument("--no-fetch", action="store_true",
help="Skip fetching (use cached or fail fast)")
parser.add_argument("--fix", action="store_true",
help="Apply a fix to recipe.toml's [build].dependencies")
parser.add_argument("--dry-run", action="store_true",
help="With --fix, only show what would change")
parser.add_argument("--json", action="store_true",
help="Emit a machine-readable JSON summary on stdout "
"(use for CI hooks or `make lint` integration).")
args = parser.parse_args()
if args.json and args.fix and args.dry_run:
print("ERROR: --json is incompatible with --fix --dry-run. "
"The combination is incoherent: --dry-run does not "
"produce diff data suitable for JSON output. Pick one.",
file=sys.stderr)
return 1
recipes = list(discover_kf6_recipes(args.component))
if not recipes:
if args.json:
import json
print(json.dumps({"recipes": [], "total_missing": 0,
"total_unused": 0}))
else:
print("No recipes found.", file=sys.stderr)
return 1
if not args.json:
print(f"Auditing {len(recipes)} recipe(s)...")
total_missing = 0
total_unused = 0
json_results = []
for recipe_toml in recipes:
entry = {
"recipe": recipe_toml.parent.name,
"missing": [],
"unused": [],
"skipped": False,
"error": None,
}
if args.no_fetch:
entry["skipped"] = True
json_results.append(entry)
continue
missing, unused, err = audit_recipe(recipe_toml,
verbose=args.verbose and not args.json)
if err:
entry["error"] = err[:120]
if not args.json:
print(f" {recipe_toml.parent.name}: SKIP ({err[:80]})")
json_results.append(entry)
continue
if missing or unused:
entry["missing"] = sorted(missing)
entry["unused"] = sorted(unused)
if not args.json:
print(f" {recipe_toml.parent.name}:")
if missing:
print(f" MISSING deps: {sorted(missing)}")
total_missing += len(missing)
if unused:
print(f" UNUSED deps: {sorted(unused)}")
total_unused += len(unused)
else:
total_missing += len(missing)
total_unused += len(unused)
if args.fix and missing:
apply_fix(recipe_toml, missing, dry_run=args.dry_run)
json_results.append(entry)
skipped_count = sum(1 for r in json_results if r.get("skipped"))
if args.json:
import json
print(json.dumps({
"recipes": json_results,
"total": len(recipes),
"skipped_count": skipped_count,
"total_missing": total_missing,
"total_unused": total_unused,
}, indent=2))
# Refuse exit 0 when every entry was skipped — no audit
# was actually performed, even though no errors were seen.
if skipped_count == len(recipes):
return 2
return 0 if not (total_missing or total_unused) else 1
print(f"\nSummary: {total_missing} missing dep(s), "
f"{total_unused} unused dep(s) across {len(recipes)} recipes.")
if skipped_count == len(recipes):
return 2
if total_missing or total_unused:
return 1
return 0
def apply_fix(recipe_toml: Path, missing: set, dry_run: bool = False):
"""Add missing deps to recipe.toml's [build].dependencies.
Uses tomllib (read-only, stdlib in py3.11+) to parse the existing
list safely — we never touch the source if a dep is already present,
even with weird quoting or inline tables. A timestamped .bak file
is written before any in-place edit so the change is reversible.
"""
try:
import tomllib
except ImportError:
import tomli as tomllib # type: ignore
with open(recipe_toml, "rb") as f:
data = tomllib.load(f)
build = data.get("build")
if not isinstance(build, dict):
print(f" SKIP fix: no [build] block found in {recipe_toml.name}")
return
deps = build.get("dependencies")
if not isinstance(deps, list):
print(f" SKIP fix: no [build].dependencies list in {recipe_toml.name}")
return
# Normalize existing entries to strings for comparison
existing = {d for d in deps if isinstance(d, str)}
to_add = sorted(set(missing) - existing)
if not to_add:
return
new_deps = list(deps) + to_add
if dry_run:
print(f" DRY-RUN: would add {to_add}")
return
# Hand-roll the rewrite so we preserve as much of the original
# formatting as possible. We only rewrite the [build] section
# line that holds the dependencies list. The rest of the file is
# untouched.
text = recipe_toml.read_text()
# Build the new dependency list literal in canonical form.
quoted = ",\n ".join(f'"{d}"' for d in new_deps)
new_block = f"dependencies = [\n {quoted},\n]"
# Match the OLD dependencies = [ ... ] block, including the
# multi-line form, and replace it with the new one. This is
# narrowly scoped to the [build] section so we don't accidentally
# touch a separate [package].dependencies list.
pattern = re.compile(
r"(?ms)(\[build\][^\[]*?)^[ \t]*dependencies[ \t]*=\s*\[.*?\]",
)
if not pattern.search(text):
print(f" SKIP fix: regex did not match [build].dependencies in {recipe_toml.name}")
return
new_text = pattern.sub(lambda m: m.group(1) + new_block, text, count=1)
if new_text == text:
print(f" SKIP fix: no change produced for {recipe_toml.name}")
return
# Backup before write. Use a unique suffix so consecutive --fix
# runs do NOT clobber prior backups. Falls back to a counter when
# multiple backups land in the same wall-clock second.
backup = None
for attempt in range(64):
suffix = ".bak.{}.{}".format(
int(time.time()), attempt if attempt else ""
).rstrip(".")
candidate = recipe_toml.with_name(recipe_toml.name + suffix)
if not candidate.exists():
backup = candidate
break
if backup is None:
print(f" SKIP fix: too many backups already exist for {recipe_toml.name}")
return
backup.write_text(text)
recipe_toml.write_text(new_text)
print(f" FIXED: added {to_add} (backup at {backup.name})")
if __name__ == "__main__":
sys.exit(main())