RedBear-OS/local/scripts/audit-kf6-deps.py

#!/usr/bin/env python3
"""Audit every KF6/Qt recipe's [build].dependencies against what its source
actually requires.

For each recipe under local/recipes/kde/ and recipes/, this script:
  1. Resolves the upstream source (git or tar) at the pinned rev
  2. Extracts/reads the source's CMakeLists.txt + all .cmake files
  3. Greps for `find_package(KF6::* COMPONENTS ...)` and `find_package(Qt6* ...)` calls
  4. Reads the recipe's [build].dependencies array
  5. Reports any KF6::/Qt* component referenced in the source but missing
     from the recipe's dependencies, AND any recipe dependency that is
     unused (i.e. not referenced by the source)
  6. Optionally: emits a fixed [build].dependencies array as a patch

Per AGENTS.md "BUILD DURABILITY" policy, the recipe.toml is the durable
artifact. This audit ensures the recipe matches what the source actually
needs, preventing "Package 'X' not found" failures at cook time.

Usage:
  ./local/scripts/audit-kf6-deps.py --verbose          # audit all 46 recipes
  ./local/scripts/audit-kf6-deps.py --component kf6-kio
  ./local/scripts/audit-kf6-deps.py --fix --dry-run    # show proposed fix
  ./local/scripts/audit-kf6-deps.py --fix              # write the fix in place
"""
import argparse
import re
import shutil
import subprocess
import sys
import tempfile
import time
import tomllib
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parents[2]
LOCAL_RECIPES = PROJECT_ROOT / "local/recipes"
MAINLINE_RECIPES = PROJECT_ROOT / "recipes"

# KF6 components appear in three forms in upstream KDE source. The
# dominant form, used by ~99% of KF6 code, is the named form: one
# find_package call per line, with the component name spelled out in
# full. The other two forms are occasional variants we still need
# to catch.
KF6_DIRECT_RE = re.compile(
    r"find_package\s*\(\s*(KF6[A-Za-z]*(?:::[A-Za-z0-9]+)+)"
)
KF6_COMPONENTS_BLOCK_RE = re.compile(
    r"find_package\s*\(\s*KF6\b[^\)]*?COMPONENTS[^\)]*\)"
)
KF6_NAMED_RE = re.compile(
    r"find_package\s*\(\s*(KF6[A-Z][A-Za-z0-9]+)\b"
)
# Form 4: find_package(KF6 <Name> REQUIRED) — rare in modern KDE code but
# exists in some older Plasma components and a handful of KF6 addons.
KF6_PLAIN_NAME_RE = re.compile(
    r"find_package\s*\(\s*KF6\s+([A-Z][A-Za-z0-9]+)\b"
)
KF6_COMPONENT_TOKEN_RE = re.compile(
    r"\b([A-Z][A-Za-z0-9]+)\b"
)

# Qt6 components: find_package(Qt6Foo REQUIRED) — usually individual modules
QT6_COMPONENT_RE = re.compile(
    r"find_package\s*\(\s*Qt6([A-Z][A-Za-z0-9]*)"
)
# Also catch the more explicit form
#   find_package(Qt6 5.15.0 COMPONENTS Core Network DBus)
QT6_GENERIC_RE = re.compile(
    r"find_package\s*\(\s*Qt6\b"
)
QT6_COMPONENTS_BLOCK_RE = re.compile(
    r"find_package\s*\(\s*Qt6\b[^\)]*?COMPONENTS[^\n]+"
)


def run(cmd, **kwargs):
    proc = subprocess.run(cmd, capture_output=True, text=True,
                           check=False, **kwargs)
    return proc.returncode, proc.stdout, proc.stderr


def _strip_cmake_noise(text: str) -> str:
    """Strip CMake comments and string literals from source before regex.

    CMake comments are introduced by `#` and run to end of line; string
    literals are double-quoted with backslash escapes. Without this
    pass, code like `set(MY_NOTE "needs find_package(KF6::Foo) later")`
    or `# find_package(KF6::FakeUsedOnlyInComment)` would be falsely
    classified as a real dependency reference.
    """
    text = re.sub(r"(?m)#.*$", "", text)
    text = re.sub(r'"(?:\\.|[^"\\])*"', '""', text)
    return text


def fetch_source(recipe_toml: Path):
    """Fetch the upstream source at the pinned rev into a tempdir.
    Returns (Path, error_msg)."""
    with open(recipe_toml, "rb") as f:
        data = tomllib.load(f)
    source = data.get("source") or {}
    url = source.get("git")
    rev = source.get("rev")
    tar = source.get("tar")

    tmp = Path(tempfile.mkdtemp(prefix="audit-kf6-"))
    if tar:
        # tar-based: download to tmp/tarball, extract into tmp/src
        tarball = tmp / "src.tar.xz"
        rc, _, err = run(["curl", "-sSL", "-o", str(tarball), tar])
        if rc != 0:
            return None, f"download failed: {err}"
        extract_dir = tmp / "src"
        extract_dir.mkdir()
        rc, _, err = run(["tar", "-xJf", str(tarball), "-C", str(extract_dir)])
        if rc != 0:
            return None, f"extract failed: {err}"
        # The tarball may have a top-level dir; find it
        candidates = list(extract_dir.iterdir())
        if len(candidates) == 1 and candidates[0].is_dir():
            return candidates[0], None
        return extract_dir, None
    elif url and rev:
        # git-based: clone at the pinned rev
        rc, _, err = run(["git", "clone", "--quiet", "--no-checkout",
                          url, str(tmp / "src")])
        if rc != 0:
            return None, f"clone failed: {err}"
        rc, _, err = run(["git", "-C", str(tmp / "src"), "checkout", "--quiet", rev])
        if rc != 0:
            return None, f"checkout failed: {err}"
        return tmp / "src", None
    else:
        return None, "no git or tar source"


def scan_source(source_dir: Path):
    """Walk the source tree and extract every KF6:: and Qt6 component used.

    Returns (set_of_kf6_components, set_of_qt6_components).
    """
    kf6 = set()
    qt6 = set()
    for cmake_file in list(source_dir.rglob("CMakeLists.txt")) + \
                      list(source_dir.rglob("*.cmake")):
        try:
            text = cmake_file.read_text(errors="replace")
        except OSError:
            continue
        text = _strip_cmake_noise(text)

        # Form 1: find_package(KF6::Foo REQUIRED) — rare, mostly Plasma
        for m in KF6_DIRECT_RE.finditer(text):
            kf6.add(m.group(1))

        # Form 2: find_package(KF6 COMPONENTS Foo Bar Baz) — all in one
        for m in KF6_COMPONENTS_BLOCK_RE.finditer(text):
            line = m.group(0)
            for tok in KF6_COMPONENT_TOKEN_RE.findall(line):
                if tok in ("REQUIRED", "QUIET", "COMPONENTS", "CONFIG",
                           "VERSION", "EXACT", "QUIETLY", "MODULE", "KF6"):
                    continue
                kf6.add(f"KF6::{tok}")

        # Form 3 (the dominant KDE form): find_package(KF6Xxx REQUIRED).
        # The full name is captured without the "KF6" prefix, then
        # normalized to the KF6::Foo form so normalize_dep_name handles
        # it like every other KF6 component. We filter out self-references
        # like "KF6KF6" (which the regex would otherwise mis-capture).
        for m in KF6_NAMED_RE.finditer(text):
            rest = m.group(1)[len("KF6"):]
            if rest.startswith("KF6") or not rest:
                continue
            kf6.add(f"KF6::{rest}")

        # Form 4: find_package(KF6 <Name> REQUIRED) — rare, but emitted
        # by some older Plasma components. The capture is the bare name.
        for m in KF6_PLAIN_NAME_RE.finditer(text):
            kf6.add(f"KF6::{m.group(1)}")

        # Qt6 individual modules: find_package(Qt6Foo REQUIRED)
        for m in QT6_COMPONENT_RE.finditer(text):
            qt6.add(f"Qt6{m.group(1)}")

        # Qt6 block form: find_package(Qt6 5.15.0 COMPONENTS Core Network DBus)
        for m in QT6_COMPONENTS_BLOCK_RE.finditer(text):
            line = m.group(0)
            for tok in KF6_COMPONENT_TOKEN_RE.findall(line):
                if tok in ("REQUIRED", "QUIET", "COMPONENTS", "CONFIG",
                           "VERSION", "EXACT", "QUIETLY", "MODULE",
                           "Qt6"):
                    continue
                qt6.add(f"Qt6{tok}")

        # Plain find_package(Qt6 ...) without components — minimal
        if QT6_GENERIC_RE.search(text):
            qt6.add("Qt6Core")
    return kf6, qt6


def read_recipe_deps(recipe_toml: Path):
    """Return (set_of_dep_names, raw_deps_text)."""
    with open(recipe_toml, "rb") as f:
        data = tomllib.load(f)
    build = data.get("build") or {}
    raw = build.get("dependencies") or []
    return {d.strip() for d in raw}, raw


KF6_RECIPE_OVERRIDES = {
    "Archive": "karchive",
    "Attica": "attica",
    "Auth": "kauth",
    "Bookmarks": "kbookmarks",
    "Codecs": "kcodecs",
    "ColorScheme": "kcolorscheme",
    "Completion": "kcompletion",
    "Config": "kconfig",
    "ConfigWidgets": "kconfigwidgets",
    "CoreAddons": "kcoreaddons",
    "Crash": "kcrash",
    "DBusAddons": "kdbusaddons",
    "Declarative": "kdeclarative",
    "DocTools": "kdoctools",
    "GuiAddons": "kguiaddons",
    "GlobalAccel": "kglobalaccel",
    "I18n": "ki18n",
    "IconThemes": "kiconthemes",
    "IdleTime": "kidletime",
    "ImageFormats": "kimageformats",
    "ItemModels": "kitemmodels",
    "ItemViews": "kitemviews",
    "JobWidgets": "kjobwidgets",
    "KCMUtils": "kcmutils",
    "KDED": "kded6",
    "KIO": "kio",
    "KNewStuff": "knewstuff",
    "KNotifyConfig": "notifyconfig",
    "Notifications": "knotifications",
    "KPackage": "kpackage",
    "Parts": "parts",
    "Plasma": "plasma",
    "Prison": "prison",
    "Pty": "pty",
    "Service": "kservice",
    "Solid": "solid",
    "Sonnet": "sonnet",
    "Svg": "ksvg",
    "SyntaxHighlighting": "syntaxhighlighting",
    "TextEditor": "ktexteditor",
    "TextWidgets": "ktextwidgets",
    "Wallet": "kwallet",
    "Wayland": "kwayland",
    "WidgetsAddons": "kwidgetsaddons",
    "WindowSystem": "kwindowsystem",
    "XmlGui": "kxmlgui",
    "ExtraCMakeModules": "extra-cmake-modules",
}


def normalize_dep_name(component: str) -> str:
    """Map a CMake KF6::Foo / Qt6Bar reference to a Red Bear OS recipe name.

    Examples:
      KF6::KIO      -> kf6-kio
      KF6::KCMUtils -> kf6-kcmutils
      KF6::IconThemes -> kf6-kiconthemes
      Qt6Core       -> qtbase
      Qt6Gui        -> qtbase
      Qt6GuiPrivate -> qtbase
      Qt6Qml        -> qtdeclarative
    """
    if component.startswith("KF6::"):
        rest = component[len("KF6::"):]
        if rest in KF6_RECIPE_OVERRIDES:
            return f"kf6-{KF6_RECIPE_OVERRIDES[rest]}"
        s = re.sub(r"(?<!^)(?=[A-Z])", "-", rest).lower()
        return f"kf6-{s}"
    if component.startswith("Qt6"):
        rest = component[len("Qt6"):]
        rest_lower = rest.lower()
        # Map specific Qt6 modules to Red Bear OS recipes
        qtbase_modules = {"core", "gui", "guilib", "guifunctions",
                          "widgets", "network", "dbus", "test",
                          "concurrent", "printsupport", "qpa",
                          "xml", "opengl", "sql", "svgwidgets",
                          "testlib", "platform", "platformsupport",
                          "platformheaders", "platformheadersclean",
                          "guiprivate", "widgetstyles", "statemachine"}
        qtdeclarative_modules = {"qml", "qmltest", "qmlintegration",
                                 "qmlworkerscript", "qmlmodels", "qmlcore",
                                 "quick", "quickcontrols", "quickcontrols2",
                                 "quickparticles", "quickwidgets",
                                 "quicktest"}
        if rest_lower in qtbase_modules:
            return "qtbase"
        if rest_lower in qtdeclarative_modules:
            return "qtdeclarative"
        if rest_lower == "svg":
            return "qtsvg"
        if rest_lower == "wayland":
            return "qtwayland"
        return f"qt6-{rest_lower}"  # generic
    return component.lower()


def audit_recipe(recipe_toml: Path, verbose: bool = False):
    """Audit a single recipe. Return (missing_deps, unused_deps, error_msg)."""
    fetched = fetch_source(recipe_toml)
    if fetched[1]:
        return set(), set(), fetched[1]
    source_dir: Path = fetched[0]  # type: ignore[assignment]

    kf6_components, qt6_components = scan_source(source_dir)
    if verbose:
        print(f"  {recipe_toml.relative_to(PROJECT_ROOT)}: "
              f"uses KF6={sorted(kf6_components)}, "
              f"Qt6={sorted(qt6_components)}")

    recipe_deps, raw = read_recipe_deps(recipe_toml)

    # Map cmake component names to recipe names
    needed_recipes = set()
    for c in kf6_components | qt6_components:
        needed_recipes.add(normalize_dep_name(c))

    # Standard transitive deps every KF6/Qt6 recipe needs (don't flag
    # these as "unused" if the recipe omits them — they're common
    # infrastructure):
    standard_infra = {
        "qtbase",                # every Qt6 recipe needs qtbase
        "qtdeclarative",          # often transitive via Qt6Core
        "kf6-extra-cmake-modules", # KDE/Qt6 build system
        "kf6-kf6",               # kf6-umbrella, only a build marker
    }

    missing = needed_recipes - recipe_deps - standard_infra
    unused = (recipe_deps - needed_recipes) - standard_infra

    # Cleanup: remove the recipe's own name (some recipes list themselves)
    own_name = recipe_toml.parent.name
    missing.discard(own_name)
    unused.discard(own_name)

    # Cleanup
    shutil.rmtree(source_dir.parent, ignore_errors=True)

    return missing, unused, None


def discover_kf6_recipes(component_filter=None):
    """Yield every KF6 recipe path (local + mainline, excluding WIP).

    Per local/AGENTS.md "Local recipe priority vs upstream WIP", the
    local fork in `local/recipes/` is the source of truth when both
    exist. The mainline tree under `recipes/wip/` is transitional and
    not part of the durable shipping surface, so we skip it.
    """
    for recipes_root in (LOCAL_RECIPES, MAINLINE_RECIPES):
        if not recipes_root.is_dir():
            continue
        for recipe_toml in recipes_root.rglob("recipe.toml"):
            if "source" in recipe_toml.parts or "target" in recipe_toml.parts:
                continue
            if "wip" in recipe_toml.parts:
                continue
            # Only KF6 + Qt recipes
            name = recipe_toml.parent.name
            if not (name.startswith("kf6-") or name.startswith("qt")):
                continue
            if component_filter and name != component_filter:
                continue
            yield recipe_toml


def main():
    parser = argparse.ArgumentParser(
        description=(
            "Audit every KF6/Qt recipe's [build].dependencies against "
            "what its source actually requires."
        )
    )
    parser.add_argument("--component", help="Audit only this recipe")
    parser.add_argument("--verbose", "-v", action="store_true",
                        help="Print every recipe's component usage")
    parser.add_argument("--no-fetch", action="store_true",
                        help="Skip fetching (use cached or fail fast)")
    parser.add_argument("--fix", action="store_true",
                        help="Apply a fix to recipe.toml's [build].dependencies")
    parser.add_argument("--dry-run", action="store_true",
                        help="With --fix, only show what would change")
    parser.add_argument("--json", action="store_true",
                        help="Emit a machine-readable JSON summary on stdout "
                             "(use for CI hooks or `make lint` integration).")
    args = parser.parse_args()

    if args.json and args.fix and args.dry_run:
        print("ERROR: --json is incompatible with --fix --dry-run. "
              "The combination is incoherent: --dry-run does not "
              "produce diff data suitable for JSON output. Pick one.",
              file=sys.stderr)
        return 1

    recipes = list(discover_kf6_recipes(args.component))
    if not recipes:
        if args.json:
            import json
            print(json.dumps({"recipes": [], "total_missing": 0,
                              "total_unused": 0}))
        else:
            print("No recipes found.", file=sys.stderr)
        return 1

    if not args.json:
        print(f"Auditing {len(recipes)} recipe(s)...")

    total_missing = 0
    total_unused = 0
    json_results = []
    for recipe_toml in recipes:
        entry = {
            "recipe": recipe_toml.parent.name,
            "missing": [],
            "unused": [],
            "skipped": False,
            "error": None,
        }
        if args.no_fetch:
            entry["skipped"] = True
            json_results.append(entry)
            continue
        missing, unused, err = audit_recipe(recipe_toml,
                                            verbose=args.verbose and not args.json)
        if err:
            entry["error"] = err[:120]
            if not args.json:
                print(f"  {recipe_toml.parent.name}: SKIP ({err[:80]})")
            json_results.append(entry)
            continue
        if missing or unused:
            entry["missing"] = sorted(missing)
            entry["unused"] = sorted(unused)
            if not args.json:
                print(f"  {recipe_toml.parent.name}:")
                if missing:
                    print(f"    MISSING deps: {sorted(missing)}")
                    total_missing += len(missing)
                if unused:
                    print(f"    UNUSED  deps: {sorted(unused)}")
                    total_unused += len(unused)
            else:
                total_missing += len(missing)
                total_unused += len(unused)
            if args.fix and missing:
                apply_fix(recipe_toml, missing, dry_run=args.dry_run)
        json_results.append(entry)

    skipped_count = sum(1 for r in json_results if r.get("skipped"))
    if args.json:
        import json
        print(json.dumps({
            "recipes": json_results,
            "total": len(recipes),
            "skipped_count": skipped_count,
            "total_missing": total_missing,
            "total_unused": total_unused,
        }, indent=2))
        # Refuse exit 0 when every entry was skipped — no audit
        # was actually performed, even though no errors were seen.
        if skipped_count == len(recipes):
            return 2
        return 0 if not (total_missing or total_unused) else 1

    print(f"\nSummary: {total_missing} missing dep(s), "
          f"{total_unused} unused dep(s) across {len(recipes)} recipes.")
    if skipped_count == len(recipes):
        return 2
    if total_missing or total_unused:
        return 1
    return 0


def apply_fix(recipe_toml: Path, missing: set, dry_run: bool = False):
    """Add missing deps to recipe.toml's [build].dependencies.

    Uses tomllib (read-only, stdlib in py3.11+) to parse the existing
    list safely — we never touch the source if a dep is already present,
    even with weird quoting or inline tables. A timestamped .bak file
    is written before any in-place edit so the change is reversible.
    """
    try:
        import tomllib
    except ImportError:
        import tomli as tomllib  # type: ignore
    with open(recipe_toml, "rb") as f:
        data = tomllib.load(f)
    build = data.get("build")
    if not isinstance(build, dict):
        print(f"    SKIP fix: no [build] block found in {recipe_toml.name}")
        return
    deps = build.get("dependencies")
    if not isinstance(deps, list):
        print(f"    SKIP fix: no [build].dependencies list in {recipe_toml.name}")
        return
    # Normalize existing entries to strings for comparison
    existing = {d for d in deps if isinstance(d, str)}
    to_add = sorted(set(missing) - existing)
    if not to_add:
        return
    new_deps = list(deps) + to_add
    if dry_run:
        print(f"    DRY-RUN: would add {to_add}")
        return
    # Hand-roll the rewrite so we preserve as much of the original
    # formatting as possible. We only rewrite the [build] section
    # line that holds the dependencies list. The rest of the file is
    # untouched.
    text = recipe_toml.read_text()
    # Build the new dependency list literal in canonical form.
    quoted = ",\n    ".join(f'"{d}"' for d in new_deps)
    new_block = f"dependencies = [\n    {quoted},\n]"
    # Match the OLD dependencies = [ ... ] block, including the
    # multi-line form, and replace it with the new one. This is
    # narrowly scoped to the [build] section so we don't accidentally
    # touch a separate [package].dependencies list.
    pattern = re.compile(
        r"(?ms)(\[build\][^\[]*?)^[ \t]*dependencies[ \t]*=\s*\[.*?\]",
    )
    if not pattern.search(text):
        print(f"    SKIP fix: regex did not match [build].dependencies in {recipe_toml.name}")
        return
    new_text = pattern.sub(lambda m: m.group(1) + new_block, text, count=1)
    if new_text == text:
        print(f"    SKIP fix: no change produced for {recipe_toml.name}")
        return
    # Backup before write. Use a unique suffix so consecutive --fix
    # runs do NOT clobber prior backups. Falls back to a counter when
    # multiple backups land in the same wall-clock second.
    backup = None
    for attempt in range(64):
        suffix = ".bak.{}.{}".format(
            int(time.time()), attempt if attempt else ""
        ).rstrip(".")
        candidate = recipe_toml.with_name(recipe_toml.name + suffix)
        if not candidate.exists():
            backup = candidate
            break
    if backup is None:
        print(f"    SKIP fix: too many backups already exist for {recipe_toml.name}")
        return
    backup.write_text(text)
    recipe_toml.write_text(new_text)
    print(f"    FIXED: added {to_add} (backup at {backup.name})")


if __name__ == "__main__":
    sys.exit(main())