From 7d803cd5fbf70ca55b213cd4a7fa2c624482c3ca Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Mon, 18 Sep 2023 22:27:07 +0200 Subject: tests/shell: honor NFT_TEST_FAIL_ON_SKIP variable to fail on any skipped tests The test suite should pass with various kernels and build configurations. Of course, that means, that some tests will be gracefully skipped, and we don't treat that as an overall failure. However, it should be possible to run a specific kernel (net-next?) and build configuration, where we expect that all tests pass. Add an option to fail the run, if any tests were skipped. This is to ensure that we don't have broken tests that never pass. This will make more sense with automated CI is running, to enable on a test system and ensure that at least on that system, all tests pass. Signed-off-by: Thomas Haller Signed-off-by: Florian Westphal --- tests/shell/run-tests.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'tests/shell/run-tests.sh') diff --git a/tests/shell/run-tests.sh b/tests/shell/run-tests.sh index 5ef6359f..4ff0b55a 100755 --- a/tests/shell/run-tests.sh +++ b/tests/shell/run-tests.sh @@ -217,6 +217,7 @@ usage() { echo " Setting this to \"0\" means also to perform global cleanups between tests (remove" echo " kernel modules)." echo " Parallel jobs requires unshare and are disabled with NFT_TEST_UNSHARE_CMD=\"\"." + echo " NFT_TEST_FAIL_ON_SKIP=*|y: if any jobs are skipped, exit with error." echo " NFT_TEST_RANDOM_SEED=: The test runner will export the environment variable NFT_TEST_RANDOM_SEED" echo " set to a random number. This can be used as a stable seed for tests to randomize behavior." echo " Set this to a fixed value to get reproducible behavior." @@ -277,6 +278,7 @@ KMEMLEAK="$(bool_y "$KMEMLEAK")" NFT_TEST_KEEP_LOGS="$(bool_y "$NFT_TEST_KEEP_LOGS")" NFT_TEST_HAS_REALROOT="$NFT_TEST_HAS_REALROOT" NFT_TEST_JOBS="${NFT_TEST_JOBS:-$_NFT_TEST_JOBS_DEFAULT}" +NFT_TEST_FAIL_ON_SKIP="$(bool_y "$NFT_TEST_FAIL_ON_SKIP")" NFT_TEST_RANDOM_SEED="$NFT_TEST_RANDOM_SEED" NFT_TEST_SHUFFLE_TESTS="$NFT_TEST_SHUFFLE_TESTS" NFT_TEST_SKIP_slow="$(bool_y "$NFT_TEST_SKIP_slow")" @@ -590,6 +592,7 @@ msg_info "conf: NFT_TEST_HAS_UNSHARED=$(printf '%q' "$NFT_TEST_HAS_UNSHARED")" msg_info "conf: NFT_TEST_HAS_UNSHARED_MOUNT=$(printf '%q' "$NFT_TEST_HAS_UNSHARED_MOUNT")" msg_info "conf: NFT_TEST_KEEP_LOGS=$(printf '%q' "$NFT_TEST_KEEP_LOGS")" msg_info "conf: NFT_TEST_JOBS=$NFT_TEST_JOBS" +msg_info "conf: NFT_TEST_FAIL_ON_SKIP=$NFT_TEST_FAIL_ON_SKIP" msg_info "conf: NFT_TEST_RANDOM_SEED=$NFT_TEST_RANDOM_SEED" msg_info "conf: NFT_TEST_SHUFFLE_TESTS=$NFT_TEST_SHUFFLE_TESTS" msg_info "conf: TMPDIR=$(printf '%q' "$_TMPDIR")" @@ -847,7 +850,7 @@ echo "" kmemleak_found=0 check_kmemleak_force -if [ "$failed" -gt 0 ] ; then +if [ "$failed" -gt 0 ] || [ "$NFT_TEST_FAIL_ON_SKIP" = y -a "$skipped" -gt 0 ] ; then RR="$RED" elif [ "$skipped" -gt 0 ] ; then RR="$YELLOW" @@ -881,6 +884,9 @@ fi if [ "$failed" -gt 0 ] ; then exit 1 +elif [ "$NFT_TEST_FAIL_ON_SKIP" = y -a "$skipped" -gt 0 ] ; then + msg_info "some tests were skipped. Fail due to NFT_TEST_FAIL_ON_SKIP=y" + exit 1 elif [ "$ok" -eq 0 -a "$skipped" -gt 0 ] ; then exit 77 else -- cgit v1.2.3