vasilito
cb424d7448
verify-patch-sanity.py validates every active recipe .patch has internally- consistent hunk line counts — catching the 'malformed patch at line N' failure at commit/CI/preflight time instead of hours into a cook. This cycle hit that class three times (qtwaylandscanner, sddm, xwayland), each only discovered when cookbook tried to apply the patch. Running it across the repo found 29 latent malformed patches (validated against GNU patch: e.g. relibc/P3-sysv-ipc reproduces 'malformed patch at line 22'). They were harmless only because they sit in vendored recipes (baked, not re- applied) — but would fail on any version-bump re-derivation. --fix recounts the hunk headers (body untouched) and repaired all 29. Wired into build-preflight.sh (Phase 1.0D) and redbear-ci.yml, with a unit test (test-patch-sanity.sh). Skips archived/legacy trees and unvalidatable formats (empty placeholders, bare-@@ git hunks).
257 lines
10 KiB
Python
Executable File
257 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# ===----------------------------------------------------------------------===##
|
|
#
|
|
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
# See https://llvm.org/LICENSE.txt for license information.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
#
|
|
# ===----------------------------------------------------------------------===##
|
|
|
|
"""adb_run.py is a utility for running a libc++ test program via adb.
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import shlex
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
from typing import List, Tuple
|
|
|
|
|
|
# Sync a host file /path/to/dir/file to ${REMOTE_BASE_DIR}/run-${HASH}/dir/file.
|
|
REMOTE_BASE_DIR = "/data/local/tmp/adb_run"
|
|
|
|
g_job_limit_socket = None
|
|
g_verbose = False
|
|
|
|
|
|
def run_adb_sync_command(command: List[str]) -> None:
|
|
"""Run an adb command and discard the output, unless the command fails. If
|
|
the command fails, dump the output instead, and exit the script with
|
|
failure.
|
|
"""
|
|
if g_verbose:
|
|
sys.stderr.write(f"running: {shlex.join(command)}\n")
|
|
proc = subprocess.run(command, universal_newlines=True,
|
|
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, encoding="utf-8")
|
|
if proc.returncode != 0:
|
|
# adb's stdout (e.g. for adb push) should normally be discarded, but
|
|
# on failure, it should be shown. Print it to stderr because it's
|
|
# unrelated to the test program's stdout output. A common error caught
|
|
# here is "No space left on device".
|
|
sys.stderr.write(f"{proc.stdout}\n"
|
|
f"error: adb command exited with {proc.returncode}: "
|
|
f"{shlex.join(command)}\n")
|
|
sys.exit(proc.returncode)
|
|
|
|
|
|
def sync_test_dir(local_dir: str, remote_dir: str) -> None:
|
|
"""Sync the libc++ test directory on the host to the remote device."""
|
|
|
|
# Optimization: The typical libc++ test directory has only a single
|
|
# *.tmp.exe file in it. In that case, skip the `mkdir` command, which is
|
|
# normally necessary because we don't know if the target directory already
|
|
# exists on the device.
|
|
local_files = os.listdir(local_dir)
|
|
if len(local_files) == 1:
|
|
local_file = os.path.join(local_dir, local_files[0])
|
|
remote_file = os.path.join(remote_dir, local_files[0])
|
|
if not os.path.islink(local_file) and os.path.isfile(local_file):
|
|
run_adb_sync_command(["adb", "push", "--sync", local_file,
|
|
remote_file])
|
|
return
|
|
|
|
assert os.path.basename(local_dir) == os.path.basename(remote_dir)
|
|
run_adb_sync_command(["adb", "shell", "mkdir", "-p", remote_dir])
|
|
run_adb_sync_command(["adb", "push", "--sync", local_dir,
|
|
os.path.dirname(remote_dir)])
|
|
|
|
|
|
def build_env_arg(env_args: List[str], prepend_path_args: List[Tuple[str, str]]) -> str:
|
|
components = []
|
|
for arg in env_args:
|
|
k, v = arg.split("=", 1)
|
|
components.append(f"export {k}={shlex.quote(v)}; ")
|
|
for k, v in prepend_path_args:
|
|
components.append(f"export {k}={shlex.quote(v)}${{{k}:+:${k}}}; ")
|
|
return "".join(components)
|
|
|
|
|
|
def run_command(args: argparse.Namespace) -> int:
|
|
local_dir = args.execdir
|
|
assert local_dir.startswith("/")
|
|
assert not local_dir.endswith("/")
|
|
|
|
# Copy each execdir to a subdir of REMOTE_BASE_DIR. Name the directory using
|
|
# a hash of local_dir so that concurrent adb_run invocations don't create
|
|
# the same intermediate parent directory. At least `adb push` has trouble
|
|
# with concurrent mkdir syscalls on common parent directories. (Somehow
|
|
# mkdir fails with EAGAIN/EWOULDBLOCK, see internal Google bug,
|
|
# b/289311228.)
|
|
local_dir_hash = hashlib.sha1(local_dir.encode()).hexdigest()
|
|
remote_dir = f"{REMOTE_BASE_DIR}/run-{local_dir_hash}/{os.path.basename(local_dir)}"
|
|
sync_test_dir(local_dir, remote_dir)
|
|
|
|
adb_shell_command = (
|
|
# Set the environment early so that PATH can be overridden. Overriding
|
|
# PATH is useful for:
|
|
# - Replacing older shell utilities with toybox (e.g. on old devices).
|
|
# - Adding a `bash` command that delegates to `sh` (mksh).
|
|
f"{build_env_arg(args.env, args.prepend_path_env)}"
|
|
|
|
# Set a high oom_score_adj so that, if the test program uses too much
|
|
# memory, it is killed before anything else on the device. The default
|
|
# oom_score_adj is -1000, so a test using too much memory typically
|
|
# crashes the device.
|
|
"echo 1000 >/proc/self/oom_score_adj; "
|
|
|
|
# If we're running as root, switch to the shell user. The libc++
|
|
# filesystem tests require running without root permissions. Some x86
|
|
# emulator devices (before Android N) do not have a working `adb unroot`
|
|
# and always run as root. Non-debug builds typically lack `su` and only
|
|
# run as the shell user.
|
|
#
|
|
# Some libc++ tests create temporary files in the working directory,
|
|
# which might be owned by root. Before switching to shell, make the
|
|
# cwd writable (and readable+executable) to every user.
|
|
#
|
|
# N.B.:
|
|
# - Avoid "id -u" because it wasn't supported until Android M.
|
|
# - The `env` and `which` commands were also added in Android M.
|
|
# - Starting in Android M, su from root->shell resets PATH, so we need
|
|
# to modify it again in the new environment.
|
|
# - Avoid chmod's "a+rwx" syntax because it's not supported until
|
|
# Android N.
|
|
# - Defining this function allows specifying the arguments to the test
|
|
# program (i.e. "$@") only once.
|
|
"run_without_root() {"
|
|
" chmod 777 .;"
|
|
" case \"$(id)\" in"
|
|
" *\"uid=0(root)\"*)"
|
|
" if command -v env >/dev/null; then"
|
|
" su shell \"$(command -v env)\" PATH=\"$PATH\" \"$@\";"
|
|
" else"
|
|
" su shell \"$@\";"
|
|
" fi;;"
|
|
" *) \"$@\";;"
|
|
" esac;"
|
|
"}; "
|
|
)
|
|
|
|
# Older versions of Bionic limit the length of argv[0] to 127 bytes
|
|
# (SOINFO_NAME_LEN-1), and the path to libc++ tests tend to exceed this
|
|
# limit. Changing the working directory works around this limit. The limit
|
|
# is increased to 4095 (PATH_MAX-1) in Android M (API 23).
|
|
command_line = [arg.replace(local_dir + "/", "./") for arg in args.command]
|
|
|
|
# Prior to the adb feature "shell_v2" (added in Android N), `adb shell`
|
|
# always created a pty:
|
|
# - This merged stdout and stderr together.
|
|
# - The pty converts LF to CRLF.
|
|
# - The exit code of the shell command wasn't propagated.
|
|
# Work around all three limitations, unless "shell_v2" is present.
|
|
proc = subprocess.run(["adb", "features"], check=True,
|
|
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
encoding="utf-8")
|
|
adb_features = set(proc.stdout.strip().split())
|
|
has_shell_v2 = "shell_v2" in adb_features
|
|
if has_shell_v2:
|
|
adb_shell_command += (
|
|
f"cd {remote_dir} && run_without_root {shlex.join(command_line)}"
|
|
)
|
|
else:
|
|
adb_shell_command += (
|
|
f"{{"
|
|
f" stdout=$("
|
|
f" cd {remote_dir} && run_without_root {shlex.join(command_line)};"
|
|
f" echo -n __libcxx_adb_exit__=$?"
|
|
f" ); "
|
|
f"}} 2>&1; "
|
|
f"echo -n __libcxx_adb_stdout__\"$stdout\""
|
|
)
|
|
|
|
adb_command_line = ["adb", "shell", adb_shell_command]
|
|
if g_verbose:
|
|
sys.stderr.write(f"running: {shlex.join(adb_command_line)}\n")
|
|
|
|
if has_shell_v2:
|
|
proc = subprocess.run(adb_command_line, shell=False, check=False,
|
|
encoding="utf-8")
|
|
return proc.returncode
|
|
else:
|
|
proc = subprocess.run(adb_command_line, shell=False, check=False,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
encoding="utf-8")
|
|
# The old `adb shell` mode used a pty, which converted LF to CRLF.
|
|
# Convert it back.
|
|
output = proc.stdout.replace("\r\n", "\n")
|
|
|
|
if proc.returncode:
|
|
sys.stderr.write(f"error: adb failed:\n"
|
|
f" command: {shlex.join(adb_command_line)}\n"
|
|
f" output: {output}\n")
|
|
return proc.returncode
|
|
|
|
match = re.match(r"(.*)__libcxx_adb_stdout__(.*)__libcxx_adb_exit__=(\d+)$",
|
|
output, re.DOTALL)
|
|
if not match:
|
|
sys.stderr.write(f"error: could not parse adb output:\n"
|
|
f" command: {shlex.join(adb_command_line)}\n"
|
|
f" output: {output}\n")
|
|
return 1
|
|
|
|
sys.stderr.write(match.group(1))
|
|
sys.stdout.write(match.group(2))
|
|
return int(match.group(3))
|
|
|
|
|
|
def connect_to_job_limiter_server(sock_addr: str) -> None:
|
|
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
|
|
try:
|
|
sock.connect(sock_addr)
|
|
except (FileNotFoundError, ConnectionRefusedError) as e:
|
|
# Copying-and-pasting an adb_run.py command-line from a lit test failure
|
|
# is likely to fail because the socket no longer exists (or is
|
|
# inactive), so just give a warning.
|
|
sys.stderr.write(f"warning: could not connect to {sock_addr}: {e}\n")
|
|
return
|
|
|
|
# The connect call can succeed before the server has called accept, because
|
|
# of the listen backlog, so wait for the server to send a byte.
|
|
sock.recv(1)
|
|
|
|
# Keep the socket open until this process ends, then let the OS close the
|
|
# connection automatically.
|
|
global g_job_limit_socket
|
|
g_job_limit_socket = sock
|
|
|
|
|
|
def main() -> int:
|
|
"""Main function (pylint wants this docstring)."""
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--execdir", type=str, required=True)
|
|
parser.add_argument("--env", type=str, required=False, action="append",
|
|
default=[], metavar="NAME=VALUE")
|
|
parser.add_argument("--prepend-path-env", type=str, nargs=2, required=False,
|
|
action="append", default=[],
|
|
metavar=("NAME", "PATH"))
|
|
parser.add_argument("--job-limit-socket")
|
|
parser.add_argument("--verbose", "-v", default=False, action="store_true")
|
|
parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
|
|
args = parser.parse_args()
|
|
|
|
global g_verbose
|
|
g_verbose = args.verbose
|
|
if args.job_limit_socket is not None:
|
|
connect_to_job_limiter_server(args.job_limit_socket)
|
|
return run_command(args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|