Improve test running on Redox

Makefile

@@ -63,6 +63,7 @@ all: | headers libs
 
 # TODO: can sed be removed now that cbindgen iirc supports varargs?
 headers: $(HEADERS_DEPS)
+	set -e ; \
 	for header in $(HEADERS_UNPARSED); do \
 		echo "Header $$header"; \
 		if test -f "src/header/$$header/cbindgen.toml"; then \

tests/run_tests.sh

@@ -0,0 +1,210 @@
+#!/usr/bin/env bash
+
+# Binaries that should generate the same output every time
+EXPECT_NAMES=(\
+	alloca \
+	args \
+	arpainet \
+	assert \
+	constructor \
+	ctype \
+	crypt/blowfish \
+	crypt/md5 \
+	crypt/pbkdf2 \
+	crypt/scrypt \
+	crypt/sha256 \
+	crypt/sha512 \
+	destructor \
+	dirent/scandir \
+	endian \
+	errno \
+	error \
+	fcntl/create \
+	fcntl/fcntl \
+	fnmatch \
+	futimens \
+	libgen \
+	locale \
+	math \
+	netdb/getaddrinfo \
+	ptrace \
+	regex \
+	select \
+	setjmp \
+	sigaction \
+	sigaltstack \
+	signal \
+	stdio/all \
+	stdio/buffer \
+	stdio/dprintf \
+	stdio/fgets \
+	stdio/fputs \
+	stdio/fread \
+	stdio/freopen \
+	stdio/fseek \
+	stdio/fwrite \
+	stdio/getc_unget \
+  	stdio/getline \
+	stdio/mutex \
+	stdio/popen \
+	stdio/printf \
+	stdio/rename \
+	stdio/scanf \
+	stdio/setvbuf \
+	stdio/sprintf \
+	stdio/printf_space_pad \
+	stdio/ungetc_ftell \
+	stdio/ungetc_multiple \
+	stdio/fscanf_offby1 \
+	stdio/fscanf \
+	stdio/printf_neg_pad \
+	stdlib/a64l \
+	stdlib/alloc \
+	stdlib/atof \
+	stdlib/atoi \
+	stdlib/div \
+	stdlib/env \
+	stdlib/mkostemps \
+	stdlib/ptsname \
+	stdlib/qsort \
+	stdlib/rand \
+	stdlib/rand48 \
+	stdlib/random \
+	stdlib/strtod \
+	stdlib/strtol \
+	stdlib/strtoul \
+	stdlib/system \
+	string/mem \
+	string/strcat \
+	string/strchr \
+	string/strcpy \
+	string/strcspn \
+	string/strlen \
+	string/strncmp \
+	string/strpbrk \
+	string/strrchr \
+	string/strspn \
+	string/strstr \
+	string/strtok \
+	string/strtok_r \
+	string/strsignal \
+	strings \
+	sys_mman \
+	time/asctime \
+	time/constants \
+	time/gmtime \
+	time/localtime \
+	time/macros \
+	time/mktime \
+	time/strftime \
+	time/time \
+	time/tzset \
+	tls \
+	unistd/access \
+	unistd/brk \
+	unistd/constants \
+	unistd/dup \
+	unistd/exec \
+	unistd/fchdir \
+	unistd/fork \
+	unistd/fsync \
+	unistd/ftruncate \
+	unistd/getopt \
+	unistd/getopt_long \
+	unistd/pipe \
+	unistd/rmdir \
+	unistd/sleep \
+	unistd/swab \
+	unistd/write \
+	waitpid \
+	wchar/fgetwc \
+	wchar/fwide \
+	wchar/mbrtowc \
+	wchar/mbsrtowcs \
+	wchar/printf-on-wchars \
+	wchar/putwchar \
+	wchar/wscanf \
+	wchar/ungetwc \
+	wchar/wprintf \
+	wchar/wcrtomb \
+	wchar/wcpcpy \
+	wchar/wcpncpy \
+	wchar/wcscspn \
+	wchar/wcsdup \
+	wchar/wcsrchr \
+	wchar/wcsrtombs \
+	wchar/wcsstr \
+	wchar/wcstod \
+	wchar/wcstok \
+	wchar/wcstol \
+	wchar/wcstoimax \
+	wchar/wcstoumax \
+	wchar/wcscasecmp \
+	wchar/wcsncasecmp \
+	wchar/wcsnlen \
+	wchar/wcsnrtombs \
+	wchar/wcswidth \
+	wctype/towlower \
+	wctype/towupper \
+	mkfifo \
+	mknod \
+	mknodat \
+)
+	# TODO: Fix these
+	# netdb/netdb \
+
+# Binaries that may generate varied output
+STATUS_NAMES=(\
+	dirent/main \
+	net/if \
+	pty/forkpty \
+	psignal \
+	pwd \
+	sa_restart \
+	sigchld \
+	stdio/ctermid \
+	sigqueue \
+	stdio/tempnam \
+	stdio/tmpnam \
+	stdlib/bsearch \
+	stdlib/mktemp \
+	stdlib/realpath \
+	sys_epoll/epoll \
+	sys_resource/constants \
+	sys_utsname/uname \
+	time/gettimeofday \
+	unistd/chdir \
+	unistd/getcwd \
+	unistd/gethostname \
+	unistd/getid \
+	unistd/getpagesize \
+	unistd/isatty \
+	unistd/link \
+	unistd/pathconf \
+	unistd/setid \
+	unistd/stat \
+	unistd/sysconf \
+	pthread/main \
+	pthread/cleanup \
+	pthread/extjoin \
+	pthread/once \
+	pthread/customstack \
+	pthread/barrier \
+	pthread/rwlock_trylock \
+	pthread/rwlock_randtest \
+	pthread/mutex_recursive \
+	pthread/timeout \
+	grp/getgrouplist \
+	grp/getgrgid_r \
+	grp/getgrnam_r \
+	grp/gr_iter \
+)
+    #	resource/getrusage
+    #	time/times
+
+EXPECT_BINS=(${EXPECT_NAMES[@]/#/.\/bins_static\/})
+STATUS_BINS=(${STATUS_NAMES[@]/#/.\/bins_static\/})
+
+bins_verify/relibc-tests --status-only ${STATUS_BINS[@]}
+
+bins_verify/relibc-tests ${EXPECT_BINS[@]}

tests/src/main.rs

@@ -44,21 +44,37 @@ fn expected(bin: &str, kind: &str, generated: &[u8], status: ExitStatus) -> Resu
 }
 
 fn main() {
+    
+    let (status_only, bins) = {
+        let mut bins = vec![];
+        let mut status_only = false;
+        for name in env::args().skip(1) {
+            if name == "--status-only" {
+                status_only = true;
+            } else {
+                bins.push(name);
+            }
+        }
+        (status_only, bins)
+    };
+
     let mut failures = Vec::new();
 
-    for bin in env::args().skip(1) {
+    for bin in bins {
         println!("# {} #", bin);
 
         match Command::new(&bin).arg("test").arg("args").output() {
             Ok(output) => {
-                if let Err(failure) = expected(&bin, "stdout", &output.stdout, output.status) {
-                    println!("{}", failure);
-                    failures.push(failure);
-                }
+                if !status_only {
+                    if let Err(failure) = expected(&bin, "stdout", &output.stdout, output.status) {
+                        println!("{}", failure);
+                        failures.push(failure);
+                    }
 
-                if let Err(failure) = expected(&bin, "stderr", &output.stderr, output.status) {
-                    println!("{}", failure);
-                    failures.push(failure);
+                    if let Err(failure) = expected(&bin, "stderr", &output.stderr, output.status) {
+                        println!("{}", failure);
+                        failures.push(failure);
+                    }
                 }
             }
             Err(err) => {