vasilito
cb424d7448
verify-patch-sanity.py validates every active recipe .patch has internally- consistent hunk line counts — catching the 'malformed patch at line N' failure at commit/CI/preflight time instead of hours into a cook. This cycle hit that class three times (qtwaylandscanner, sddm, xwayland), each only discovered when cookbook tried to apply the patch. Running it across the repo found 29 latent malformed patches (validated against GNU patch: e.g. relibc/P3-sysv-ipc reproduces 'malformed patch at line 22'). They were harmless only because they sit in vendored recipes (baked, not re- applied) — but would fail on any version-bump re-derivation. --fix recounts the hunk headers (body untouched) and repaired all 29. Wired into build-preflight.sh (Phase 1.0D) and redbear-ci.yml, with a unit test (test-patch-sanity.sh). Skips archived/legacy trees and unvalidatable formats (empty placeholders, bare-@@ git hunks).
524 lines
19 KiB
Python
Executable File
524 lines
19 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# type: ignore
|
|
|
|
"""
|
|
compare.py - versatile benchmark output compare tool
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
import unittest
|
|
from argparse import ArgumentParser
|
|
|
|
import gbench
|
|
from gbench import report, util
|
|
|
|
|
|
def check_inputs(in1, in2, flags):
|
|
"""
|
|
Perform checking on the user provided inputs and diagnose any abnormalities
|
|
"""
|
|
in1_kind, in1_err = util.classify_input_file(in1)
|
|
in2_kind, in2_err = util.classify_input_file(in2)
|
|
output_file = util.find_benchmark_flag("--benchmark_out=", flags)
|
|
output_type = util.find_benchmark_flag("--benchmark_out_format=", flags)
|
|
if (
|
|
in1_kind == util.IT_Executable
|
|
and in2_kind == util.IT_Executable
|
|
and output_file
|
|
):
|
|
print(
|
|
(
|
|
"WARNING: '--benchmark_out=%s' will be passed to both "
|
|
"benchmarks causing it to be overwritten"
|
|
)
|
|
% output_file
|
|
)
|
|
if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON:
|
|
# When both sides are JSON the only supported flag is
|
|
# --benchmark_filter=
|
|
for flag in util.remove_benchmark_flags("--benchmark_filter=", flags):
|
|
print(
|
|
"WARNING: passing %s has no effect since both "
|
|
"inputs are JSON" % flag
|
|
)
|
|
if output_type is not None and output_type != "json":
|
|
print(
|
|
(
|
|
"ERROR: passing '--benchmark_out_format=%s' to 'compare.py`"
|
|
" is not supported."
|
|
)
|
|
% output_type
|
|
)
|
|
sys.exit(1)
|
|
|
|
|
|
def create_parser():
|
|
parser = ArgumentParser(
|
|
description="versatile benchmark output compare tool"
|
|
)
|
|
|
|
parser.add_argument(
|
|
"-a",
|
|
"--display_aggregates_only",
|
|
dest="display_aggregates_only",
|
|
action="store_true",
|
|
help="If there are repetitions, by default, we display everything - the"
|
|
" actual runs, and the aggregates computed. Sometimes, it is "
|
|
"desirable to only view the aggregates. E.g. when there are a lot "
|
|
"of repetitions. Do note that only the display is affected. "
|
|
"Internally, all the actual runs are still used, e.g. for U test.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--no-color",
|
|
dest="color",
|
|
default=True,
|
|
action="store_false",
|
|
help="Do not use colors in the terminal output",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"-d",
|
|
"--dump_to_json",
|
|
dest="dump_to_json",
|
|
help="Additionally, dump benchmark comparison output to this file in JSON format.",
|
|
)
|
|
|
|
utest = parser.add_argument_group()
|
|
utest.add_argument(
|
|
"--no-utest",
|
|
dest="utest",
|
|
default=True,
|
|
action="store_false",
|
|
help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(
|
|
report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS
|
|
),
|
|
)
|
|
alpha_default = 0.05
|
|
utest.add_argument(
|
|
"--alpha",
|
|
dest="utest_alpha",
|
|
default=alpha_default,
|
|
type=float,
|
|
help=(
|
|
"significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)"
|
|
)
|
|
% alpha_default,
|
|
)
|
|
|
|
subparsers = parser.add_subparsers(
|
|
help="This tool has multiple modes of operation:", dest="mode"
|
|
)
|
|
|
|
parser_a = subparsers.add_parser(
|
|
"benchmarks",
|
|
help="The most simple use-case, compare all the output of these two benchmarks",
|
|
)
|
|
baseline = parser_a.add_argument_group("baseline", "The benchmark baseline")
|
|
baseline.add_argument(
|
|
"test_baseline",
|
|
metavar="test_baseline",
|
|
type=argparse.FileType("r"),
|
|
nargs=1,
|
|
help="A benchmark executable or JSON output file",
|
|
)
|
|
contender = parser_a.add_argument_group(
|
|
"contender", "The benchmark that will be compared against the baseline"
|
|
)
|
|
contender.add_argument(
|
|
"test_contender",
|
|
metavar="test_contender",
|
|
type=argparse.FileType("r"),
|
|
nargs=1,
|
|
help="A benchmark executable or JSON output file",
|
|
)
|
|
parser_a.add_argument(
|
|
"benchmark_options",
|
|
metavar="benchmark_options",
|
|
nargs=argparse.REMAINDER,
|
|
help="Arguments to pass when running benchmark executables",
|
|
)
|
|
|
|
parser_b = subparsers.add_parser(
|
|
"filters", help="Compare filter one with the filter two of benchmark"
|
|
)
|
|
baseline = parser_b.add_argument_group("baseline", "The benchmark baseline")
|
|
baseline.add_argument(
|
|
"test",
|
|
metavar="test",
|
|
type=argparse.FileType("r"),
|
|
nargs=1,
|
|
help="A benchmark executable or JSON output file",
|
|
)
|
|
baseline.add_argument(
|
|
"filter_baseline",
|
|
metavar="filter_baseline",
|
|
type=str,
|
|
nargs=1,
|
|
help="The first filter, that will be used as baseline",
|
|
)
|
|
contender = parser_b.add_argument_group(
|
|
"contender", "The benchmark that will be compared against the baseline"
|
|
)
|
|
contender.add_argument(
|
|
"filter_contender",
|
|
metavar="filter_contender",
|
|
type=str,
|
|
nargs=1,
|
|
help="The second filter, that will be compared against the baseline",
|
|
)
|
|
parser_b.add_argument(
|
|
"benchmark_options",
|
|
metavar="benchmark_options",
|
|
nargs=argparse.REMAINDER,
|
|
help="Arguments to pass when running benchmark executables",
|
|
)
|
|
|
|
parser_c = subparsers.add_parser(
|
|
"benchmarksfiltered",
|
|
help="Compare filter one of first benchmark with filter two of the second benchmark",
|
|
)
|
|
baseline = parser_c.add_argument_group("baseline", "The benchmark baseline")
|
|
baseline.add_argument(
|
|
"test_baseline",
|
|
metavar="test_baseline",
|
|
type=argparse.FileType("r"),
|
|
nargs=1,
|
|
help="A benchmark executable or JSON output file",
|
|
)
|
|
baseline.add_argument(
|
|
"filter_baseline",
|
|
metavar="filter_baseline",
|
|
type=str,
|
|
nargs=1,
|
|
help="The first filter, that will be used as baseline",
|
|
)
|
|
contender = parser_c.add_argument_group(
|
|
"contender", "The benchmark that will be compared against the baseline"
|
|
)
|
|
contender.add_argument(
|
|
"test_contender",
|
|
metavar="test_contender",
|
|
type=argparse.FileType("r"),
|
|
nargs=1,
|
|
help="The second benchmark executable or JSON output file, that will be compared against the baseline",
|
|
)
|
|
contender.add_argument(
|
|
"filter_contender",
|
|
metavar="filter_contender",
|
|
type=str,
|
|
nargs=1,
|
|
help="The second filter, that will be compared against the baseline",
|
|
)
|
|
parser_c.add_argument(
|
|
"benchmark_options",
|
|
metavar="benchmark_options",
|
|
nargs=argparse.REMAINDER,
|
|
help="Arguments to pass when running benchmark executables",
|
|
)
|
|
|
|
return parser
|
|
|
|
|
|
def main():
|
|
# Parse the command line flags
|
|
parser = create_parser()
|
|
args, unknown_args = parser.parse_known_args()
|
|
if args.mode is None:
|
|
parser.print_help()
|
|
exit(1)
|
|
assert not unknown_args
|
|
benchmark_options = args.benchmark_options
|
|
|
|
if args.mode == "benchmarks":
|
|
test_baseline = args.test_baseline[0].name
|
|
test_contender = args.test_contender[0].name
|
|
filter_baseline = ""
|
|
filter_contender = ""
|
|
|
|
# NOTE: if test_baseline == test_contender, you are analyzing the stdev
|
|
|
|
description = "Comparing %s to %s" % (test_baseline, test_contender)
|
|
elif args.mode == "filters":
|
|
test_baseline = args.test[0].name
|
|
test_contender = args.test[0].name
|
|
filter_baseline = args.filter_baseline[0]
|
|
filter_contender = args.filter_contender[0]
|
|
|
|
# NOTE: if filter_baseline == filter_contender, you are analyzing the
|
|
# stdev
|
|
|
|
description = "Comparing %s to %s (from %s)" % (
|
|
filter_baseline,
|
|
filter_contender,
|
|
args.test[0].name,
|
|
)
|
|
elif args.mode == "benchmarksfiltered":
|
|
test_baseline = args.test_baseline[0].name
|
|
test_contender = args.test_contender[0].name
|
|
filter_baseline = args.filter_baseline[0]
|
|
filter_contender = args.filter_contender[0]
|
|
|
|
# NOTE: if test_baseline == test_contender and
|
|
# filter_baseline == filter_contender, you are analyzing the stdev
|
|
|
|
description = "Comparing %s (from %s) to %s (from %s)" % (
|
|
filter_baseline,
|
|
test_baseline,
|
|
filter_contender,
|
|
test_contender,
|
|
)
|
|
else:
|
|
# should never happen
|
|
print("Unrecognized mode of operation: '%s'" % args.mode)
|
|
parser.print_help()
|
|
exit(1)
|
|
|
|
check_inputs(test_baseline, test_contender, benchmark_options)
|
|
|
|
if args.display_aggregates_only:
|
|
benchmark_options += ["--benchmark_display_aggregates_only=true"]
|
|
|
|
options_baseline = []
|
|
options_contender = []
|
|
|
|
if filter_baseline and filter_contender:
|
|
options_baseline = ["--benchmark_filter=%s" % filter_baseline]
|
|
options_contender = ["--benchmark_filter=%s" % filter_contender]
|
|
|
|
# Run the benchmarks and report the results
|
|
json1 = json1_orig = gbench.util.sort_benchmark_results(
|
|
gbench.util.run_or_load_benchmark(
|
|
test_baseline, benchmark_options + options_baseline
|
|
)
|
|
)
|
|
json2 = json2_orig = gbench.util.sort_benchmark_results(
|
|
gbench.util.run_or_load_benchmark(
|
|
test_contender, benchmark_options + options_contender
|
|
)
|
|
)
|
|
|
|
# Now, filter the benchmarks so that the difference report can work
|
|
if filter_baseline and filter_contender:
|
|
replacement = "[%s vs. %s]" % (filter_baseline, filter_contender)
|
|
json1 = gbench.report.filter_benchmark(
|
|
json1_orig, filter_baseline, replacement
|
|
)
|
|
json2 = gbench.report.filter_benchmark(
|
|
json2_orig, filter_contender, replacement
|
|
)
|
|
|
|
diff_report = gbench.report.get_difference_report(json1, json2, args.utest)
|
|
output_lines = gbench.report.print_difference_report(
|
|
diff_report,
|
|
args.display_aggregates_only,
|
|
args.utest,
|
|
args.utest_alpha,
|
|
args.color,
|
|
)
|
|
print(description)
|
|
for ln in output_lines:
|
|
print(ln)
|
|
|
|
# Optionally, diff and output to JSON
|
|
if args.dump_to_json is not None:
|
|
with open(args.dump_to_json, "w") as f_json:
|
|
json.dump(diff_report, f_json, indent=1)
|
|
|
|
|
|
class TestParser(unittest.TestCase):
|
|
def setUp(self):
|
|
self.parser = create_parser()
|
|
testInputs = os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)), "gbench", "Inputs"
|
|
)
|
|
self.testInput0 = os.path.join(testInputs, "test1_run1.json")
|
|
self.testInput1 = os.path.join(testInputs, "test1_run2.json")
|
|
|
|
def test_benchmarks_basic(self):
|
|
parsed = self.parser.parse_args(
|
|
["benchmarks", self.testInput0, self.testInput1]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, "benchmarks")
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarks_basic_without_utest(self):
|
|
parsed = self.parser.parse_args(
|
|
["--no-utest", "benchmarks", self.testInput0, self.testInput1]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertFalse(parsed.utest)
|
|
self.assertEqual(parsed.utest_alpha, 0.05)
|
|
self.assertEqual(parsed.mode, "benchmarks")
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarks_basic_display_aggregates_only(self):
|
|
parsed = self.parser.parse_args(
|
|
["-a", "benchmarks", self.testInput0, self.testInput1]
|
|
)
|
|
self.assertTrue(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, "benchmarks")
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarks_basic_with_utest_alpha(self):
|
|
parsed = self.parser.parse_args(
|
|
["--alpha=0.314", "benchmarks", self.testInput0, self.testInput1]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.utest_alpha, 0.314)
|
|
self.assertEqual(parsed.mode, "benchmarks")
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarks_basic_without_utest_with_utest_alpha(self):
|
|
parsed = self.parser.parse_args(
|
|
[
|
|
"--no-utest",
|
|
"--alpha=0.314",
|
|
"benchmarks",
|
|
self.testInput0,
|
|
self.testInput1,
|
|
]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertFalse(parsed.utest)
|
|
self.assertEqual(parsed.utest_alpha, 0.314)
|
|
self.assertEqual(parsed.mode, "benchmarks")
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarks_with_remainder(self):
|
|
parsed = self.parser.parse_args(
|
|
["benchmarks", self.testInput0, self.testInput1, "d"]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, "benchmarks")
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertEqual(parsed.benchmark_options, ["d"])
|
|
|
|
def test_benchmarks_with_remainder_after_doubleminus(self):
|
|
parsed = self.parser.parse_args(
|
|
["benchmarks", self.testInput0, self.testInput1, "--", "e"]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, "benchmarks")
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertEqual(parsed.benchmark_options, ["e"])
|
|
|
|
def test_filters_basic(self):
|
|
parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d"])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, "filters")
|
|
self.assertEqual(parsed.test[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], "c")
|
|
self.assertEqual(parsed.filter_contender[0], "d")
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_filters_with_remainder(self):
|
|
parsed = self.parser.parse_args(
|
|
["filters", self.testInput0, "c", "d", "e"]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, "filters")
|
|
self.assertEqual(parsed.test[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], "c")
|
|
self.assertEqual(parsed.filter_contender[0], "d")
|
|
self.assertEqual(parsed.benchmark_options, ["e"])
|
|
|
|
def test_filters_with_remainder_after_doubleminus(self):
|
|
parsed = self.parser.parse_args(
|
|
["filters", self.testInput0, "c", "d", "--", "f"]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, "filters")
|
|
self.assertEqual(parsed.test[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], "c")
|
|
self.assertEqual(parsed.filter_contender[0], "d")
|
|
self.assertEqual(parsed.benchmark_options, ["f"])
|
|
|
|
def test_benchmarksfiltered_basic(self):
|
|
parsed = self.parser.parse_args(
|
|
["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e"]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, "benchmarksfiltered")
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], "c")
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertEqual(parsed.filter_contender[0], "e")
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarksfiltered_with_remainder(self):
|
|
parsed = self.parser.parse_args(
|
|
[
|
|
"benchmarksfiltered",
|
|
self.testInput0,
|
|
"c",
|
|
self.testInput1,
|
|
"e",
|
|
"f",
|
|
]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, "benchmarksfiltered")
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], "c")
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertEqual(parsed.filter_contender[0], "e")
|
|
self.assertEqual(parsed.benchmark_options[0], "f")
|
|
|
|
def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
|
|
parsed = self.parser.parse_args(
|
|
[
|
|
"benchmarksfiltered",
|
|
self.testInput0,
|
|
"c",
|
|
self.testInput1,
|
|
"e",
|
|
"--",
|
|
"g",
|
|
]
|
|
)
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, "benchmarksfiltered")
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], "c")
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertEqual(parsed.filter_contender[0], "e")
|
|
self.assertEqual(parsed.benchmark_options[0], "g")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# unittest.main()
|
|
main()
|
|
|
|
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
|
|
# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
|
|
# kate: indent-mode python; remove-trailing-spaces modified;
|