vasilito
cb424d7448
verify-patch-sanity.py validates every active recipe .patch has internally- consistent hunk line counts — catching the 'malformed patch at line N' failure at commit/CI/preflight time instead of hours into a cook. This cycle hit that class three times (qtwaylandscanner, sddm, xwayland), each only discovered when cookbook tried to apply the patch. Running it across the repo found 29 latent malformed patches (validated against GNU patch: e.g. relibc/P3-sysv-ipc reproduces 'malformed patch at line 22'). They were harmless only because they sit in vendored recipes (baked, not re- applied) — but would fail on any version-bump re-derivation. --fix recounts the hunk headers (body untouched) and repaired all 29. Wired into build-preflight.sh (Phase 1.0D) and redbear-ci.yml, with a unit test (test-patch-sanity.sh). Skips archived/legacy trees and unvalidatable formats (empty placeholders, bare-@@ git hunks).
152 lines
5.6 KiB
Python
Executable File
152 lines
5.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
# See https://llvm.org/LICENSE.txt for license information.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
#
|
|
# ==------------------------------------------------------------------------==#
|
|
|
|
import argparse
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
|
|
# The purpose of this script is to measure the performance effect
|
|
# of an lld change in a statistically sound way, automating all the
|
|
# tedious parts of doing so. It copies the test case into /tmp as well as
|
|
# running the test binaries from /tmp to reduce the influence on the test
|
|
# machine's storage medium on the results. It accounts for measurement
|
|
# bias caused by binary layout (using the --randomize-section-padding
|
|
# flag to link the test binaries) and by environment variable size
|
|
# (implemented by hyperfine [1]). Runs of the base and test case are
|
|
# interleaved to account for environmental factors which may influence
|
|
# the result due to the passage of time. The results of running hyperfine
|
|
# are collected into a results.csv file in the output directory and may
|
|
# be analyzed by the user with a tool such as ministat.
|
|
#
|
|
# Requirements: Linux host, hyperfine [2] in $PATH, run from a build directory
|
|
# configured to use ninja and a recent version of lld that supports
|
|
# --randomize-section-padding, /tmp is tmpfs.
|
|
#
|
|
# [1] https://github.com/sharkdp/hyperfine/blob/3cedcc38d0c430cbf38b4364b441c43a938d2bf3/src/util/randomized_environment_offset.rs#L1
|
|
# [2] https://github.com/sharkdp/hyperfine
|
|
#
|
|
# Example invocation for comparing the performance of the current commit
|
|
# against the previous commit which is treated as the baseline, without
|
|
# linking debug info:
|
|
#
|
|
# lld/utils/run_benchmark.py \
|
|
# --base-commit HEAD^ \
|
|
# --test-commit HEAD \
|
|
# --test-case lld/utils/speed-test-reproducers/result/firefox-x64/response.txt \
|
|
# --num-iterations 512 \
|
|
# --num-binary-variants 16 \
|
|
# --output-dir outdir \
|
|
# --ldflags=-S
|
|
#
|
|
# Then this bash command will compare the real time of the base and test cases.
|
|
#
|
|
# ministat -A \
|
|
# <(grep lld-base outdir/results.csv | cut -d, -f2) \
|
|
# <(grep lld-test outdir/results.csv | cut -d, -f2)
|
|
|
|
# We don't want to copy stat() information when we copy the reproducer
|
|
# to the temporary directory. Files in the Nix store are read-only so this will
|
|
# cause trouble when the linker writes the output file and when we want to clean
|
|
# up the temporary directory. Python doesn't provide a way to disable copying
|
|
# stat() information in shutil.copytree so we just monkeypatch shutil.copystat
|
|
# to do nothing.
|
|
shutil.copystat = lambda *args, **kwargs: 0
|
|
|
|
parser = argparse.ArgumentParser(prog="benchmark_change.py")
|
|
parser.add_argument("--base-commit", required=True)
|
|
parser.add_argument("--test-commit", required=True)
|
|
parser.add_argument("--test-case", required=True)
|
|
parser.add_argument("--num-iterations", type=int, required=True)
|
|
parser.add_argument("--num-binary-variants", type=int, required=True)
|
|
parser.add_argument("--output-dir", required=True)
|
|
parser.add_argument("--ldflags", required=False)
|
|
args = parser.parse_args()
|
|
|
|
test_dir = tempfile.mkdtemp()
|
|
print(f"Using {test_dir} as temporary directory")
|
|
|
|
os.makedirs(args.output_dir)
|
|
print(f"Using {args.output_dir} as output directory")
|
|
|
|
|
|
def extract_link_command(target):
|
|
# We assume that the last command printed by "ninja -t commands" containing a
|
|
# "-o" flag is the link command (we need to check for -o because subsequent
|
|
# commands create symlinks for ld.lld and so on). This is true for CMake and
|
|
# gn.
|
|
link_command = None
|
|
for line in subprocess.Popen(
|
|
["ninja", "-t", "commands", target], stdout=subprocess.PIPE
|
|
).stdout.readlines():
|
|
commands = line.decode("utf-8").split("&&")
|
|
for command in commands:
|
|
if " -o " in command:
|
|
link_command = command.strip()
|
|
return link_command
|
|
|
|
|
|
def generate_binary_variants(case_name):
|
|
subprocess.run(["ninja", "lld"])
|
|
link_command = extract_link_command("lld")
|
|
|
|
for i in range(0, args.num_binary_variants):
|
|
print(f"Generating binary variant {i} for {case_name} case")
|
|
command = f"{link_command} -o {test_dir}/lld-{case_name}{i} -Wl,--randomize-section-padding={i}"
|
|
subprocess.run(command, check=True, shell=True)
|
|
|
|
|
|
# Make sure that there are no local changes.
|
|
subprocess.run(["git", "diff", "--exit-code", "HEAD"], check=True)
|
|
|
|
# Resolve the base and test commit, since if they are relative to HEAD we will
|
|
# check out the wrong commit below.
|
|
resolved_base_commit = subprocess.check_output(
|
|
["git", "rev-parse", args.base_commit]
|
|
).strip()
|
|
resolved_test_commit = subprocess.check_output(
|
|
["git", "rev-parse", args.test_commit]
|
|
).strip()
|
|
|
|
test_case_dir = os.path.dirname(args.test_case)
|
|
test_case_respfile = os.path.basename(args.test_case)
|
|
|
|
test_dir_test_case_dir = f"{test_dir}/testcase"
|
|
shutil.copytree(test_case_dir, test_dir_test_case_dir)
|
|
|
|
subprocess.run(["git", "checkout", resolved_base_commit], check=True)
|
|
generate_binary_variants("base")
|
|
|
|
subprocess.run(["git", "checkout", resolved_test_commit], check=True)
|
|
generate_binary_variants("test")
|
|
|
|
|
|
def hyperfine_link_command(case_name):
|
|
return f'../lld-{case_name}$(({{iter}}%{args.num_binary_variants})) -flavor ld.lld @{test_case_respfile} {args.ldflags or ""}'
|
|
|
|
|
|
results_csv = f"{args.output_dir}/results.csv"
|
|
subprocess.run(
|
|
[
|
|
"hyperfine",
|
|
"--export-csv",
|
|
os.path.abspath(results_csv),
|
|
"-P",
|
|
"iter",
|
|
"0",
|
|
str(args.num_iterations - 1),
|
|
hyperfine_link_command("base"),
|
|
hyperfine_link_command("test"),
|
|
],
|
|
check=True,
|
|
cwd=test_dir_test_case_dir,
|
|
)
|
|
|
|
shutil.rmtree(test_dir)
|