From 17eac6c2db8b2cdfe33d40229bdda2acd86b304a Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Thu, 26 Sep 2019 16:40:14 -0600 Subject: [PATCH 1/6] selftests: Add kselftest-all and kselftest-install targets Add kselftest-all target to build tests from the top level Makefile. This is to simplify kselftest use-cases for CI and distributions where build and test systems are different. Current kselftest target builds and runs tests on a development system which is a developer use-case. Add kselftest-install target to install tests from the top level Makefile. This is to simplify kselftest use-cases for CI and distributions where build and test systems are different. This change addresses requests from developers and testers to add support for installing kselftest from the main Makefile. In addition, make the install directory the same when install is run using "make kselftest-install" or by running kselftest_install.sh. Also fix the INSTALL_PATH variable conflict between main Makefile and selftests Makefile. Signed-off-by: Shuah Khan Acked-by: Masahiro Yamada Signed-off-by: Shuah Khan --- Makefile | 5 ++--- tools/testing/selftests/Makefile | 8 ++++++-- tools/testing/selftests/kselftest_install.sh | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 6f54f2f95743..02f7ede2f344 100644 --- a/Makefile +++ b/Makefile @@ -1237,9 +1237,8 @@ PHONY += kselftest kselftest: $(Q)$(MAKE) -C $(srctree)/tools/testing/selftests run_tests -PHONY += kselftest-clean -kselftest-clean: - $(Q)$(MAKE) -C $(srctree)/tools/testing/selftests clean +kselftest-%: FORCE + $(Q)$(MAKE) -C $(srctree)/tools/testing/selftests $* PHONY += kselftest-merge kselftest-merge: diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index c3feccb99ff5..bad18145ed1a 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -171,9 +171,12 @@ run_pstore_crash: # 1. output_dir=kernel_src # 2. a separate output directory is specified using O= KBUILD_OUTPUT # 3. a separate output directory is specified using KBUILD_OUTPUT +# Avoid conflict with INSTALL_PATH set by the main Makefile # -INSTALL_PATH ?= $(BUILD)/install -INSTALL_PATH := $(abspath $(INSTALL_PATH)) +KSFT_INSTALL_PATH ?= $(BUILD)/kselftest_install +KSFT_INSTALL_PATH := $(abspath $(KSFT_INSTALL_PATH)) +# Avoid changing the rest of the logic here and lib.mk. +INSTALL_PATH := $(KSFT_INSTALL_PATH) ALL_SCRIPT := $(INSTALL_PATH)/run_kselftest.sh install: all @@ -203,6 +206,7 @@ ifdef INSTALL_PATH echo "[ -w /dev/kmsg ] && echo \"kselftest: Running tests in $$TARGET\" >> /dev/kmsg" >> $(ALL_SCRIPT); \ echo "cd $$TARGET" >> $(ALL_SCRIPT); \ echo -n "run_many" >> $(ALL_SCRIPT); \ + echo -n "Emit Tests for $$TARGET\n"; \ $(MAKE) -s --no-print-directory OUTPUT=$$BUILD_TARGET -C $$TARGET emit_tests >> $(ALL_SCRIPT); \ echo "" >> $(ALL_SCRIPT); \ echo "cd \$$ROOT" >> $(ALL_SCRIPT); \ diff --git a/tools/testing/selftests/kselftest_install.sh b/tools/testing/selftests/kselftest_install.sh index ec304463883c..e2e1911d62d5 100755 --- a/tools/testing/selftests/kselftest_install.sh +++ b/tools/testing/selftests/kselftest_install.sh @@ -24,12 +24,12 @@ main() echo "$0: Installing in specified location - $install_loc ..." fi - install_dir=$install_loc/kselftest + install_dir=$install_loc/kselftest_install # Create install directory mkdir -p $install_dir # Build tests - INSTALL_PATH=$install_dir make install + KSFT_INSTALL_PATH=$install_dir make install } main "$@" From 3a24f7f6b610910ee328df11392e21925cefdecb Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Thu, 26 Sep 2019 18:52:18 +0100 Subject: [PATCH 2/6] kselftest: add capability to skip chosen TARGETS Let the user specify an optional TARGETS skiplist through the new optional SKIP_TARGETS Makefile variable. It is easier to skip at will using a reduced and well defined list of possibly problematic targets with SKIP_TARGETS than to provide a partially stripped down list of good targets using the usual TARGETS variable. Signed-off-by: Cristian Marussi Signed-off-by: Shuah Khan --- Documentation/dev-tools/kselftest.rst | 16 ++++++++++++++++ tools/testing/selftests/Makefile | 7 +++++++ 2 files changed, 23 insertions(+) diff --git a/Documentation/dev-tools/kselftest.rst b/Documentation/dev-tools/kselftest.rst index 25604904fa6e..ecdfdc9d4b03 100644 --- a/Documentation/dev-tools/kselftest.rst +++ b/Documentation/dev-tools/kselftest.rst @@ -89,6 +89,22 @@ To build, save output files in a separate directory with KBUILD_OUTPUT :: $ export KBUILD_OUTPUT=/tmp/kselftest; make TARGETS="size timers" kselftest +Additionally you can use the "SKIP_TARGETS" variable on the make command +line to specify one or more targets to exclude from the TARGETS list. + +To run all tests but a single subsystem:: + + $ make -C tools/testing/selftests SKIP_TARGETS=ptrace run_tests + +You can specify multiple tests to skip:: + + $ make SKIP_TARGETS="size timers" kselftest + +You can also specify a restricted list of tests to run together with a +dedicated skiplist:: + + $ make TARGETS="bpf breakpoints size timers" SKIP_TARGETS=bpf kselftest + See the top-level tools/testing/selftests/Makefile for the list of all possible targets. diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index bad18145ed1a..e08f26ca4071 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -63,6 +63,13 @@ TARGETS += zram TARGETS_HOTPLUG = cpu-hotplug TARGETS_HOTPLUG += memory-hotplug +# User can optionally provide a TARGETS skiplist. +SKIP_TARGETS ?= +ifneq ($(SKIP_TARGETS),) + TMP := $(filter-out $(SKIP_TARGETS), $(TARGETS)) + override TARGETS := $(TMP) +endif + # Clear LDFLAGS and MAKEFLAGS if called from main # Makefile to avoid test build failures when test # Makefile doesn't have explicit build rules. From 131b30c94fbc0adb15f911609884dd39dada8f00 Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Thu, 26 Sep 2019 18:52:19 +0100 Subject: [PATCH 3/6] kselftest: exclude failed TARGETS from runlist A TARGET which failed to be built/installed should not be included in the runlist generated inside the run_kselftest.sh script. Signed-off-by: Cristian Marussi Signed-off-by: Shuah Khan --- tools/testing/selftests/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index e08f26ca4071..4cdbae6f4e61 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -208,8 +208,12 @@ ifdef INSTALL_PATH echo " cat /dev/null > \$$logfile" >> $(ALL_SCRIPT) echo "fi" >> $(ALL_SCRIPT) + @# While building run_kselftest.sh skip also non-existent TARGET dirs: + @# they could be the result of a build failure and should NOT be + @# included in the generated runlist. for TARGET in $(TARGETS); do \ BUILD_TARGET=$$BUILD/$$TARGET; \ + [ ! -d $$INSTALL_PATH/$$TARGET ] && echo "Skipping non-existent dir: $$TARGET" && continue; \ echo "[ -w /dev/kmsg ] && echo \"kselftest: Running tests in $$TARGET\" >> /dev/kmsg" >> $(ALL_SCRIPT); \ echo "cd $$TARGET" >> $(ALL_SCRIPT); \ echo -n "run_many" >> $(ALL_SCRIPT); \ From 852c8cbf34d3b3130a05c38064dd98614f97d3a8 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 19 Sep 2019 11:06:44 -0700 Subject: [PATCH 4/6] selftests/kselftest/runner.sh: Add 45 second timeout per test Commit a745f7af3cbd ("selftests/harness: Add 30 second timeout per test") solves the problem of kselftest_harness.h-using binary tests possibly hanging forever. However, scripts and other binaries can still hang forever. This adds a global timeout to each test script run. To make this configurable (e.g. as needed in the "rtc" test case), include a new per-test-directory "settings" file (similar to "config") that can contain kselftest-specific settings. The first recognized field is "timeout". Additionally, this splits the reporting for timeouts into a specific "TIMEOUT" not-ok (and adds exit code reporting in the remaining case). Signed-off-by: Kees Cook Signed-off-by: Shuah Khan --- tools/testing/selftests/kselftest/runner.sh | 36 +++++++++++++++++++-- tools/testing/selftests/rtc/settings | 1 + 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 tools/testing/selftests/rtc/settings diff --git a/tools/testing/selftests/kselftest/runner.sh b/tools/testing/selftests/kselftest/runner.sh index 00c9020bdda8..84de7bc74f2c 100644 --- a/tools/testing/selftests/kselftest/runner.sh +++ b/tools/testing/selftests/kselftest/runner.sh @@ -3,9 +3,14 @@ # # Runs a set of tests in a given subdirectory. export skip_rc=4 +export timeout_rc=124 export logfile=/dev/stdout export per_test_logging= +# Defaults for "settings" file fields: +# "timeout" how many seconds to let each test run before failing. +export kselftest_default_timeout=45 + # There isn't a shell-agnostic way to find the path of a sourced file, # so we must rely on BASE_DIR being set to find other tools. if [ -z "$BASE_DIR" ]; then @@ -24,6 +29,16 @@ tap_prefix() fi } +tap_timeout() +{ + # Make sure tests will time out if utility is available. + if [ -x /usr/bin/timeout ] ; then + /usr/bin/timeout "$kselftest_timeout" "$1" + else + "$1" + fi +} + run_one() { DIR="$1" @@ -32,6 +47,18 @@ run_one() BASENAME_TEST=$(basename $TEST) + # Reset any "settings"-file variables. + export kselftest_timeout="$kselftest_default_timeout" + # Load per-test-directory kselftest "settings" file. + settings="$BASE_DIR/$DIR/settings" + if [ -r "$settings" ] ; then + while read line ; do + field=$(echo "$line" | cut -d= -f1) + value=$(echo "$line" | cut -d= -f2-) + eval "kselftest_$field"="$value" + done < "$settings" + fi + TEST_HDR_MSG="selftests: $DIR: $BASENAME_TEST" echo "# $TEST_HDR_MSG" if [ ! -x "$TEST" ]; then @@ -44,14 +71,17 @@ run_one() echo "not ok $test_num $TEST_HDR_MSG" else cd `dirname $TEST` > /dev/null - (((((./$BASENAME_TEST 2>&1; echo $? >&3) | + ((((( tap_timeout ./$BASENAME_TEST 2>&1; echo $? >&3) | tap_prefix >&4) 3>&1) | (read xs; exit $xs)) 4>>"$logfile" && echo "ok $test_num $TEST_HDR_MSG") || - (if [ $? -eq $skip_rc ]; then \ + (rc=$?; \ + if [ $rc -eq $skip_rc ]; then \ echo "not ok $test_num $TEST_HDR_MSG # SKIP" + elif [ $rc -eq $timeout_rc ]; then \ + echo "not ok $test_num $TEST_HDR_MSG # TIMEOUT" else - echo "not ok $test_num $TEST_HDR_MSG" + echo "not ok $test_num $TEST_HDR_MSG # exit=$rc" fi) cd - >/dev/null fi diff --git a/tools/testing/selftests/rtc/settings b/tools/testing/selftests/rtc/settings new file mode 100644 index 000000000000..ba4d85f74cd6 --- /dev/null +++ b/tools/testing/selftests/rtc/settings @@ -0,0 +1 @@ +timeout=90 From 8f9577eda5b74de4bd8b186c67f20736bdea6e07 Mon Sep 17 00:00:00 2001 From: "George G. Davis" Date: Tue, 17 Sep 2019 20:40:22 +0200 Subject: [PATCH 5/6] selftests: watchdog: Validate optional file argument The newly added optional file argument does not validate if the file is indeed a watchdog, e.g.: ./watchdog-test -f /dev/zero Watchdog Ticking Away! Fix it by confirming that the WDIOC_GETSUPPORT ioctl succeeds. Fixes: a4864a33f56caa ("selftests: watchdog: Add optional file argument") Reported-by: Eugeniu Rosca Signed-off-by: George G. Davis Signed-off-by: Eugeniu Rosca Signed-off-by: Shuah Khan --- tools/testing/selftests/watchdog/watchdog-test.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/testing/selftests/watchdog/watchdog-test.c b/tools/testing/selftests/watchdog/watchdog-test.c index afff120c7be6..6ed822dc2222 100644 --- a/tools/testing/selftests/watchdog/watchdog-test.c +++ b/tools/testing/selftests/watchdog/watchdog-test.c @@ -97,6 +97,7 @@ int main(int argc, char *argv[]) int c; int oneshot = 0; char *file = "/dev/watchdog"; + struct watchdog_info info; setbuf(stdout, NULL); @@ -118,6 +119,16 @@ int main(int argc, char *argv[]) exit(-1); } + /* + * Validate that `file` is a watchdog device + */ + ret = ioctl(fd, WDIOC_GETSUPPORT, &info); + if (ret) { + printf("WDIOC_GETSUPPORT error '%s'\n", strerror(errno)); + close(fd); + exit(ret); + } + optind = 0; while ((c = getopt_long(argc, argv, sopts, lopts, NULL)) != -1) { From ce3a677802121e038d2f062e90f96f84e7351da0 Mon Sep 17 00:00:00 2001 From: "George G. Davis" Date: Tue, 17 Sep 2019 20:40:23 +0200 Subject: [PATCH 6/6] selftests: watchdog: Add command line option to show watchdog_info With the new ioctl(WDIOC_GETSUPPORT) call in place, add a command line option to show the watchdog_info. Suggested-by: Eugeniu Rosca Signed-off-by: George G. Davis Signed-off-by: Eugeniu Rosca Signed-off-by: Shuah Khan --- tools/testing/selftests/watchdog/watchdog-test.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/watchdog/watchdog-test.c b/tools/testing/selftests/watchdog/watchdog-test.c index 6ed822dc2222..f45e510500c0 100644 --- a/tools/testing/selftests/watchdog/watchdog-test.c +++ b/tools/testing/selftests/watchdog/watchdog-test.c @@ -19,7 +19,7 @@ int fd; const char v = 'V'; -static const char sopts[] = "bdehp:t:Tn:NLf:"; +static const char sopts[] = "bdehp:t:Tn:NLf:i"; static const struct option lopts[] = { {"bootstatus", no_argument, NULL, 'b'}, {"disable", no_argument, NULL, 'd'}, @@ -32,6 +32,7 @@ static const struct option lopts[] = { {"getpretimeout", no_argument, NULL, 'N'}, {"gettimeleft", no_argument, NULL, 'L'}, {"file", required_argument, NULL, 'f'}, + {"info", no_argument, NULL, 'i'}, {NULL, no_argument, NULL, 0x0} }; @@ -72,6 +73,7 @@ static void usage(char *progname) printf("Usage: %s [options]\n", progname); printf(" -f, --file\t\tOpen watchdog device file\n"); printf("\t\t\tDefault is /dev/watchdog\n"); + printf(" -i, --info\t\tShow watchdog_info\n"); printf(" -b, --bootstatus\tGet last boot status (Watchdog/POR)\n"); printf(" -d, --disable\t\tTurn off the watchdog timer\n"); printf(" -e, --enable\t\tTurn on the watchdog timer\n"); @@ -216,6 +218,18 @@ int main(int argc, char *argv[]) case 'f': /* Handled above */ break; + case 'i': + /* + * watchdog_info was obtained as part of file open + * validation. So we just show it here. + */ + oneshot = 1; + printf("watchdog_info:\n"); + printf(" identity:\t\t%s\n", info.identity); + printf(" firmware_version:\t%u\n", + info.firmware_version); + printf(" options:\t\t%08x\n", info.options); + break; default: usage(argv[0]);