From 6112ca0261d54f710e9895a414dfcabd53ccaa93 Mon Sep 17 00:00:00 2001
From: Thomas Haller <thaller@redhat.com>
Date: Fri, 8 Sep 2023 17:07:25 +0200
Subject: tests/shell: add "--quick" option to skip slow tests (via
 NFT_TEST_SKIP_slow=y)

It's important to run (a part) of the tests in a timely manner.
Add an option to skip long running tests.

Thereby, add a more general NFT_TEST_SKIP_* mechanism.

This is related and inverse from "NFT_TEST_HAVE_json", where a test
can require [ "$NFT_TEST_HAVE_json" != n ] to run, but is skipped when
[ "$NFT_TEST_SKIP_slow" = y ].

Currently only NFT_TEST_SKIP_slow is supported. The user can set such
environment variables (or use the -Q|--quick command line option). The
configuration is printed in the test info.

Tests should check for [ "$NFT_TEST_SKIP_slow" = y ] so that the
variable has to be explicitly set to opt-out. For convenience, tests can
also add a

    # NFT_TEST_SKIP(NFT_TEST_SKIP_slow)

tag, which is evaluated by test-wrapper.sh. Or they can run a quick, reduced
part of the test, but then should still indicate to be skipped.

Mark 8 tests are as slow, that take longer than 5 seconds on my machine.
With this, a parallel wall time for the non-slow tests is only 7 seconds
(on my machine).

The ultimate point is to integrate a call to "tests/shell/run-tests.sh"
in a `make check` target. For development, you can then export
NFT_TEST_SKIP_slow=y and have a fast `make check`.

Signed-off-by: Thomas Haller <thaller@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
---
 tests/shell/run-tests.sh | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

(limited to 'tests/shell/run-tests.sh')

diff --git a/tests/shell/run-tests.sh b/tests/shell/run-tests.sh
index 51d285e3..f20a2bec 100755
--- a/tests/shell/run-tests.sh
+++ b/tests/shell/run-tests.sh
@@ -118,6 +118,7 @@ usage() {
 	echo " -U|--no-unshare : Sets NFT_TEST_UNSHARE_CMD=\"\"."
 	echo " -k|--keep-logs  : Sets NFT_TEST_KEEP_LOGS=y."
 	echo " -s|--sequential : Sets NFT_TEST_JOBS=0, which also enables global cleanups."
+	echo " -Q|--quick      : Sets NFT_TEST_SKIP_slow=y."
 	echo " --              : Separate options from tests."
 	echo " [TESTS...]      : Other options are treated as test names,"
 	echo "                   that is, executables that are run by the runner."
@@ -174,6 +175,8 @@ usage() {
 	echo " NFT_TEST_HAVE_<FEATURE>=*|y: Some tests requires certain features or will be skipped."
 	echo "                 The features are autodetected, but you can force it by setting the variable."
 	echo "                 Supported <FEATURE>s are: ${_HAVE_OPTS[@]}."
+	echo " NFT_TEST_SKIP_<OPTION>=*|y: if set, certain tests are skipped."
+	echo "                 Supported <OPTION>s are: ${_SKIP_OPTS[@]}."
 }
 
 NFT_TEST_BASEDIR="$(dirname "$0")"
@@ -188,6 +191,13 @@ for KEY in $(compgen -v | grep '^NFT_TEST_HAVE_' | sort) ; do
 	fi
 done
 
+_SKIP_OPTS=( slow )
+for KEY in $(compgen -v | grep '^NFT_TEST_SKIP_' | sort) ; do
+	if ! array_contains "${KEY#NFT_TEST_SKIP_}" "${_SKIP_OPTS[@]}" ; then
+		unset "$KEY"
+	fi
+done
+
 _NFT_TEST_JOBS_DEFAULT="$(nproc)"
 [ "$_NFT_TEST_JOBS_DEFAULT" -gt 0 ] 2>/dev/null || _NFT_TEST_JOBS_DEFAULT=1
 _NFT_TEST_JOBS_DEFAULT="$(( _NFT_TEST_JOBS_DEFAULT + (_NFT_TEST_JOBS_DEFAULT + 1) / 2 ))"
@@ -199,6 +209,7 @@ KMEMLEAK="$(bool_y "$KMEMLEAK")"
 NFT_TEST_KEEP_LOGS="$(bool_y "$NFT_TEST_KEEP_LOGS")"
 NFT_TEST_HAS_REALROOT="$NFT_TEST_HAS_REALROOT"
 NFT_TEST_JOBS="${NFT_TEST_JOBS:-$_NFT_TEST_JOBS_DEFAULT}"
+NFT_TEST_SKIP_slow="$(bool_y "$NFT_TEST_SKIP_slow")"
 DO_LIST_TESTS=
 
 TESTS=()
@@ -238,6 +249,9 @@ while [ $# -gt 0 ] ; do
 		-s|--sequential)
 			NFT_TEST_JOBS=0
 			;;
+		-Q|--quick)
+			NFT_TEST_SKIP_slow=y
+			;;
 		--)
 			TESTS+=( "$@" )
 			shift $#
@@ -438,6 +452,10 @@ msg_info "conf: NFT_TEST_KEEP_LOGS=$(printf '%q' "$NFT_TEST_KEEP_LOGS")"
 msg_info "conf: NFT_TEST_JOBS=$NFT_TEST_JOBS"
 msg_info "conf: TMPDIR=$(printf '%q' "$_TMPDIR")"
 echo
+for KEY in $(compgen -v | grep '^NFT_TEST_SKIP_' | sort) ; do
+	msg_info "conf: $KEY=$(printf '%q' "${!KEY}")"
+	export "$KEY"
+done
 for KEY in $(compgen -v | grep '^NFT_TEST_HAVE_' | sort) ; do
 	msg_info "conf: $KEY=$(printf '%q' "${!KEY}")"
 	export "$KEY"
-- 
cgit v1.2.3