diff options
| author | Thomas Haller <thaller@redhat.com> | 2023-09-18 22:27:07 +0200 |
|---|---|---|
| committer | Florian Westphal <fw@strlen.de> | 2023-09-21 11:59:10 +0200 |
| commit | 7d803cd5fbf70ca55b213cd4a7fa2c624482c3ca (patch) | |
| tree | 4ebcfcadd4095730e7665457012c2501915564cf /tests/shell/run-tests.sh | |
| parent | a44ea3e5675078954740c54ec0400bb6d56397b2 (diff) | |
tests/shell: honor NFT_TEST_FAIL_ON_SKIP variable to fail on any skipped tests
The test suite should pass with various kernels and build
configurations. Of course, that means, that some tests will be
gracefully skipped, and we don't treat that as an overall failure.
However, it should be possible to run a specific kernel (net-next?) and
build configuration, where we expect that all tests pass.
Add an option to fail the run, if any tests were skipped. This is to
ensure that we don't have broken tests that never pass.
This will make more sense with automated CI is running, to enable on a
test system and ensure that at least on that system, all tests pass.
Signed-off-by: Thomas Haller <thaller@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Diffstat (limited to 'tests/shell/run-tests.sh')
| -rwxr-xr-x | tests/shell/run-tests.sh | 8 |
1 files changed, 7 insertions, 1 deletions
diff --git a/tests/shell/run-tests.sh b/tests/shell/run-tests.sh index 5ef6359f..4ff0b55a 100755 --- a/tests/shell/run-tests.sh +++ b/tests/shell/run-tests.sh @@ -217,6 +217,7 @@ usage() { echo " Setting this to \"0\" means also to perform global cleanups between tests (remove" echo " kernel modules)." echo " Parallel jobs requires unshare and are disabled with NFT_TEST_UNSHARE_CMD=\"\"." + echo " NFT_TEST_FAIL_ON_SKIP=*|y: if any jobs are skipped, exit with error." echo " NFT_TEST_RANDOM_SEED=<SEED>: The test runner will export the environment variable NFT_TEST_RANDOM_SEED" echo " set to a random number. This can be used as a stable seed for tests to randomize behavior." echo " Set this to a fixed value to get reproducible behavior." @@ -277,6 +278,7 @@ KMEMLEAK="$(bool_y "$KMEMLEAK")" NFT_TEST_KEEP_LOGS="$(bool_y "$NFT_TEST_KEEP_LOGS")" NFT_TEST_HAS_REALROOT="$NFT_TEST_HAS_REALROOT" NFT_TEST_JOBS="${NFT_TEST_JOBS:-$_NFT_TEST_JOBS_DEFAULT}" +NFT_TEST_FAIL_ON_SKIP="$(bool_y "$NFT_TEST_FAIL_ON_SKIP")" NFT_TEST_RANDOM_SEED="$NFT_TEST_RANDOM_SEED" NFT_TEST_SHUFFLE_TESTS="$NFT_TEST_SHUFFLE_TESTS" NFT_TEST_SKIP_slow="$(bool_y "$NFT_TEST_SKIP_slow")" @@ -590,6 +592,7 @@ msg_info "conf: NFT_TEST_HAS_UNSHARED=$(printf '%q' "$NFT_TEST_HAS_UNSHARED")" msg_info "conf: NFT_TEST_HAS_UNSHARED_MOUNT=$(printf '%q' "$NFT_TEST_HAS_UNSHARED_MOUNT")" msg_info "conf: NFT_TEST_KEEP_LOGS=$(printf '%q' "$NFT_TEST_KEEP_LOGS")" msg_info "conf: NFT_TEST_JOBS=$NFT_TEST_JOBS" +msg_info "conf: NFT_TEST_FAIL_ON_SKIP=$NFT_TEST_FAIL_ON_SKIP" msg_info "conf: NFT_TEST_RANDOM_SEED=$NFT_TEST_RANDOM_SEED" msg_info "conf: NFT_TEST_SHUFFLE_TESTS=$NFT_TEST_SHUFFLE_TESTS" msg_info "conf: TMPDIR=$(printf '%q' "$_TMPDIR")" @@ -847,7 +850,7 @@ echo "" kmemleak_found=0 check_kmemleak_force -if [ "$failed" -gt 0 ] ; then +if [ "$failed" -gt 0 ] || [ "$NFT_TEST_FAIL_ON_SKIP" = y -a "$skipped" -gt 0 ] ; then RR="$RED" elif [ "$skipped" -gt 0 ] ; then RR="$YELLOW" @@ -881,6 +884,9 @@ fi if [ "$failed" -gt 0 ] ; then exit 1 +elif [ "$NFT_TEST_FAIL_ON_SKIP" = y -a "$skipped" -gt 0 ] ; then + msg_info "some tests were skipped. Fail due to NFT_TEST_FAIL_ON_SKIP=y" + exit 1 elif [ "$ok" -eq 0 -a "$skipped" -gt 0 ] ; then exit 77 else |