vasilito
cb424d7448
verify-patch-sanity.py validates every active recipe .patch has internally- consistent hunk line counts — catching the 'malformed patch at line N' failure at commit/CI/preflight time instead of hours into a cook. This cycle hit that class three times (qtwaylandscanner, sddm, xwayland), each only discovered when cookbook tried to apply the patch. Running it across the repo found 29 latent malformed patches (validated against GNU patch: e.g. relibc/P3-sysv-ipc reproduces 'malformed patch at line 22'). They were harmless only because they sit in vendored recipes (baked, not re- applied) — but would fail on any version-bump re-derivation. --fix recounts the hunk headers (body untouched) and repaired all 29. Wired into build-preflight.sh (Phase 1.0D) and redbear-ci.yml, with a unit test (test-patch-sanity.sh). Skips archived/legacy trees and unvalidatable formats (empty placeholders, bare-@@ git hunks).
484 lines
14 KiB
Python
Executable File
484 lines
14 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
import argparse
|
|
import sys
|
|
import os
|
|
|
|
from subprocess import call
|
|
|
|
SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects")
|
|
DEFAULT_LLVM_DIR = os.path.realpath(
|
|
os.path.join(SCRIPTS_DIR, os.path.pardir, os.path.pardir, os.path.pardir)
|
|
)
|
|
|
|
|
|
def add(parser, args):
|
|
import SATestAdd
|
|
from ProjectMap import ProjectInfo
|
|
|
|
if args.source == "git" and (args.origin == "" or args.commit == ""):
|
|
parser.error("Please provide both --origin and --commit if source is 'git'")
|
|
|
|
if args.source != "git" and (args.origin != "" or args.commit != ""):
|
|
parser.error(
|
|
"Options --origin and --commit don't make sense when " "source is not 'git'"
|
|
)
|
|
|
|
project = ProjectInfo(
|
|
args.name[0], args.mode, args.source, args.origin, args.commit
|
|
)
|
|
|
|
SATestAdd.add_new_project(project)
|
|
|
|
|
|
def build(parser, args):
|
|
import SATestBuild
|
|
|
|
SATestBuild.VERBOSE = args.verbose
|
|
|
|
projects = get_projects(parser, args)
|
|
tester = SATestBuild.RegressionTester(
|
|
args.jobs,
|
|
projects,
|
|
args.override_compiler,
|
|
args.extra_analyzer_config,
|
|
args.extra_checkers,
|
|
args.regenerate,
|
|
args.strictness,
|
|
)
|
|
tests_passed = tester.test_all()
|
|
|
|
if not tests_passed:
|
|
sys.stderr.write("ERROR: Tests failed.\n")
|
|
sys.exit(42)
|
|
|
|
|
|
def compare(parser, args):
|
|
import CmpRuns
|
|
|
|
choices = [
|
|
CmpRuns.HistogramType.RELATIVE.value,
|
|
CmpRuns.HistogramType.LOG_RELATIVE.value,
|
|
CmpRuns.HistogramType.ABSOLUTE.value,
|
|
]
|
|
|
|
if args.histogram is not None and args.histogram not in choices:
|
|
parser.error(
|
|
"Incorrect histogram type, available choices are {}".format(choices)
|
|
)
|
|
|
|
dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
|
|
dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
|
|
|
|
CmpRuns.dump_scan_build_results_diff(
|
|
dir_old,
|
|
dir_new,
|
|
show_stats=args.show_stats,
|
|
stats_only=args.stats_only,
|
|
histogram=args.histogram,
|
|
verbose_log=args.verbose_log,
|
|
)
|
|
|
|
|
|
def update(parser, args):
|
|
import SATestUpdateDiffs
|
|
from ProjectMap import ProjectMap
|
|
|
|
project_map = ProjectMap()
|
|
for project in project_map.projects:
|
|
SATestUpdateDiffs.update_reference_results(project, args.git)
|
|
|
|
|
|
def benchmark(parser, args):
|
|
from SATestBenchmark import Benchmark
|
|
|
|
projects = get_projects(parser, args)
|
|
benchmark = Benchmark(projects, args.iterations, args.output)
|
|
benchmark.run()
|
|
|
|
|
|
def benchmark_compare(parser, args):
|
|
import SATestBenchmark
|
|
|
|
SATestBenchmark.compare(args.old, args.new, args.output)
|
|
|
|
|
|
def get_projects(parser, args):
|
|
from ProjectMap import ProjectMap, Size
|
|
|
|
project_map = ProjectMap()
|
|
projects = project_map.projects
|
|
|
|
def filter_projects(projects, predicate, force=False):
|
|
return [
|
|
project.with_fields(
|
|
enabled=(force or project.enabled) and predicate(project)
|
|
)
|
|
for project in projects
|
|
]
|
|
|
|
if args.projects:
|
|
projects_arg = args.projects.split(",")
|
|
available_projects = [project.name for project in projects]
|
|
|
|
# validate that given projects are present in the project map file
|
|
for manual_project in projects_arg:
|
|
if manual_project not in available_projects:
|
|
parser.error(
|
|
"Project '{project}' is not found in "
|
|
"the project map file. Available projects are "
|
|
"{all}.".format(project=manual_project, all=available_projects)
|
|
)
|
|
|
|
projects = filter_projects(
|
|
projects, lambda project: project.name in projects_arg, force=True
|
|
)
|
|
|
|
try:
|
|
max_size = Size.from_str(args.max_size)
|
|
except ValueError as e:
|
|
parser.error("{}".format(e))
|
|
|
|
projects = filter_projects(projects, lambda project: project.size <= max_size)
|
|
|
|
return projects
|
|
|
|
|
|
def docker(parser, args):
|
|
if len(args.rest) > 0:
|
|
if args.rest[0] != "--":
|
|
parser.error("REST arguments should start with '--'")
|
|
args.rest = args.rest[1:]
|
|
|
|
if args.build_image:
|
|
docker_build_image()
|
|
elif args.shell:
|
|
docker_shell(args)
|
|
else:
|
|
sys.exit(docker_run(args, " ".join(args.rest)))
|
|
|
|
|
|
def docker_build_image():
|
|
sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), shell=True))
|
|
|
|
|
|
def docker_shell(args):
|
|
try:
|
|
# First we need to start the docker container in a waiting mode,
|
|
# so it doesn't do anything, but most importantly keeps working
|
|
# while the shell session is in progress.
|
|
docker_run(args, "--wait", "--detach")
|
|
# Since the docker container is running, we can actually connect to it
|
|
call("docker exec -it satest bash", shell=True)
|
|
|
|
except KeyboardInterrupt:
|
|
pass
|
|
|
|
finally:
|
|
docker_cleanup()
|
|
|
|
|
|
def docker_run(args, command, docker_args=""):
|
|
try:
|
|
return call(
|
|
"docker run --rm --name satest "
|
|
"-v {llvm}:/llvm-project "
|
|
"-v {build}:/build "
|
|
"-v {clang}:/analyzer "
|
|
"-v {scripts}:/scripts "
|
|
"-v {projects}:/projects "
|
|
"{docker_args} "
|
|
"satest-image:latest {command}".format(
|
|
llvm=args.llvm_project_dir,
|
|
build=args.build_dir,
|
|
clang=args.clang_dir,
|
|
scripts=SCRIPTS_DIR,
|
|
projects=PROJECTS_DIR,
|
|
docker_args=docker_args,
|
|
command=command,
|
|
),
|
|
shell=True,
|
|
)
|
|
|
|
except KeyboardInterrupt:
|
|
docker_cleanup()
|
|
|
|
|
|
def docker_cleanup():
|
|
print("Please wait for docker to clean up")
|
|
call("docker stop satest", shell=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers()
|
|
|
|
# add subcommand
|
|
add_parser = subparsers.add_parser(
|
|
"add", help="Add a new project for the analyzer testing."
|
|
)
|
|
# TODO: Add an option not to build.
|
|
# TODO: Set the path to the Repository directory.
|
|
add_parser.add_argument("name", nargs=1, help="Name of the new project")
|
|
add_parser.add_argument(
|
|
"--mode",
|
|
action="store",
|
|
default=1,
|
|
type=int,
|
|
choices=[0, 1, 2],
|
|
help="Build mode: 0 for single file project, "
|
|
"1 for scan_build, "
|
|
"2 for single file c++11 project",
|
|
)
|
|
add_parser.add_argument(
|
|
"--source",
|
|
action="store",
|
|
default="script",
|
|
choices=["script", "git", "zip"],
|
|
help="Source type of the new project: "
|
|
"'git' for getting from git "
|
|
"(please provide --origin and --commit), "
|
|
"'zip' for unpacking source from a zip file, "
|
|
"'script' for downloading source by running "
|
|
"a custom script",
|
|
)
|
|
add_parser.add_argument(
|
|
"--origin", action="store", default="", help="Origin link for a git repository"
|
|
)
|
|
add_parser.add_argument(
|
|
"--commit", action="store", default="", help="Git hash for a commit to checkout"
|
|
)
|
|
add_parser.set_defaults(func=add)
|
|
|
|
# build subcommand
|
|
build_parser = subparsers.add_parser(
|
|
"build",
|
|
help="Build projects from the project map and compare results with "
|
|
"the reference.",
|
|
)
|
|
build_parser.add_argument(
|
|
"--strictness",
|
|
dest="strictness",
|
|
type=int,
|
|
default=0,
|
|
help="0 to fail on runtime errors, 1 to fail "
|
|
"when the number of found bugs are different "
|
|
"from the reference, 2 to fail on any "
|
|
"difference from the reference. Default is 0.",
|
|
)
|
|
build_parser.add_argument(
|
|
"-r",
|
|
dest="regenerate",
|
|
action="store_true",
|
|
default=False,
|
|
help="Regenerate reference output.",
|
|
)
|
|
build_parser.add_argument(
|
|
"--override-compiler",
|
|
action="store_true",
|
|
default=False,
|
|
help="Call scan-build with " "--override-compiler option.",
|
|
)
|
|
build_parser.add_argument(
|
|
"-j",
|
|
"--jobs",
|
|
dest="jobs",
|
|
type=int,
|
|
default=0,
|
|
help="Number of projects to test concurrently",
|
|
)
|
|
build_parser.add_argument(
|
|
"--extra-analyzer-config",
|
|
dest="extra_analyzer_config",
|
|
type=str,
|
|
default="",
|
|
help="Arguments passed to to -analyzer-config",
|
|
)
|
|
build_parser.add_argument(
|
|
"--extra-checkers",
|
|
dest="extra_checkers",
|
|
type=str,
|
|
default="",
|
|
help="Extra checkers to enable",
|
|
)
|
|
build_parser.add_argument(
|
|
"--projects",
|
|
action="store",
|
|
default="",
|
|
help="Comma-separated list of projects to test",
|
|
)
|
|
build_parser.add_argument(
|
|
"--max-size",
|
|
action="store",
|
|
default=None,
|
|
help="Maximum size for the projects to test",
|
|
)
|
|
build_parser.add_argument("-v", "--verbose", action="count", default=0)
|
|
build_parser.set_defaults(func=build)
|
|
|
|
# compare subcommand
|
|
cmp_parser = subparsers.add_parser(
|
|
"compare",
|
|
help="Comparing two static analyzer runs in terms of "
|
|
"reported warnings and execution time statistics.",
|
|
)
|
|
cmp_parser.add_argument(
|
|
"--root-old",
|
|
dest="root_old",
|
|
help="Prefix to ignore on source files for " "OLD directory",
|
|
action="store",
|
|
type=str,
|
|
default="",
|
|
)
|
|
cmp_parser.add_argument(
|
|
"--root-new",
|
|
dest="root_new",
|
|
help="Prefix to ignore on source files for " "NEW directory",
|
|
action="store",
|
|
type=str,
|
|
default="",
|
|
)
|
|
cmp_parser.add_argument(
|
|
"--verbose-log",
|
|
dest="verbose_log",
|
|
help="Write additional information to LOG " "[default=None]",
|
|
action="store",
|
|
type=str,
|
|
default=None,
|
|
metavar="LOG",
|
|
)
|
|
cmp_parser.add_argument(
|
|
"--stats-only",
|
|
action="store_true",
|
|
dest="stats_only",
|
|
default=False,
|
|
help="Only show statistics on reports",
|
|
)
|
|
cmp_parser.add_argument(
|
|
"--show-stats",
|
|
action="store_true",
|
|
dest="show_stats",
|
|
default=False,
|
|
help="Show change in statistics",
|
|
)
|
|
cmp_parser.add_argument(
|
|
"--histogram",
|
|
action="store",
|
|
default=None,
|
|
help="Show histogram of paths differences. " "Requires matplotlib",
|
|
)
|
|
cmp_parser.add_argument("old", nargs=1, help="Directory with old results")
|
|
cmp_parser.add_argument("new", nargs=1, help="Directory with new results")
|
|
cmp_parser.set_defaults(func=compare)
|
|
|
|
# update subcommand
|
|
upd_parser = subparsers.add_parser(
|
|
"update",
|
|
help="Update static analyzer reference results based on the previous "
|
|
"run of SATest build. Assumes that SATest build was just run.",
|
|
)
|
|
upd_parser.add_argument(
|
|
"--git", action="store_true", help="Stage updated results using git."
|
|
)
|
|
upd_parser.set_defaults(func=update)
|
|
|
|
# docker subcommand
|
|
dock_parser = subparsers.add_parser(
|
|
"docker", help="Run regression system in the docker."
|
|
)
|
|
|
|
dock_parser.add_argument(
|
|
"--build-image",
|
|
action="store_true",
|
|
help="Build docker image for running tests.",
|
|
)
|
|
dock_parser.add_argument(
|
|
"--shell", action="store_true", help="Start a shell on docker."
|
|
)
|
|
dock_parser.add_argument(
|
|
"--llvm-project-dir",
|
|
action="store",
|
|
default=DEFAULT_LLVM_DIR,
|
|
help="Path to LLVM source code. Defaults "
|
|
"to the repo where this script is located. ",
|
|
)
|
|
dock_parser.add_argument(
|
|
"--build-dir",
|
|
action="store",
|
|
default="",
|
|
help="Path to a directory where docker should " "build LLVM code.",
|
|
)
|
|
dock_parser.add_argument(
|
|
"--clang-dir",
|
|
action="store",
|
|
default="",
|
|
help="Path to find/install LLVM installation.",
|
|
)
|
|
dock_parser.add_argument(
|
|
"rest",
|
|
nargs=argparse.REMAINDER,
|
|
default=[],
|
|
help="Additional args that will be forwarded " "to the docker's entrypoint.",
|
|
)
|
|
dock_parser.set_defaults(func=docker)
|
|
|
|
# benchmark subcommand
|
|
bench_parser = subparsers.add_parser(
|
|
"benchmark", help="Run benchmarks by building a set of projects multiple times."
|
|
)
|
|
|
|
bench_parser.add_argument(
|
|
"-i",
|
|
"--iterations",
|
|
action="store",
|
|
type=int,
|
|
default=20,
|
|
help="Number of iterations for building each " "project.",
|
|
)
|
|
bench_parser.add_argument(
|
|
"-o",
|
|
"--output",
|
|
action="store",
|
|
default="benchmark.csv",
|
|
help="Output csv file for the benchmark results",
|
|
)
|
|
bench_parser.add_argument(
|
|
"--projects",
|
|
action="store",
|
|
default="",
|
|
help="Comma-separated list of projects to test",
|
|
)
|
|
bench_parser.add_argument(
|
|
"--max-size",
|
|
action="store",
|
|
default=None,
|
|
help="Maximum size for the projects to test",
|
|
)
|
|
bench_parser.set_defaults(func=benchmark)
|
|
|
|
bench_subparsers = bench_parser.add_subparsers()
|
|
bench_compare_parser = bench_subparsers.add_parser(
|
|
"compare", help="Compare benchmark runs."
|
|
)
|
|
bench_compare_parser.add_argument(
|
|
"--old",
|
|
action="store",
|
|
required=True,
|
|
help="Benchmark reference results to " "compare agains.",
|
|
)
|
|
bench_compare_parser.add_argument(
|
|
"--new", action="store", required=True, help="New benchmark results to check."
|
|
)
|
|
bench_compare_parser.add_argument(
|
|
"-o", "--output", action="store", required=True, help="Output file for plots."
|
|
)
|
|
bench_compare_parser.set_defaults(func=benchmark_compare)
|
|
|
|
args = parser.parse_args()
|
|
args.func(parser, args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|