tests: shell: change all test scripts to return 0

The shell-based tests currently encode a return value in the
file name, i.e. foo_1 expects that the script should return '1'
for the test case to pass.

This is very error prone, and one test case is even broken (i.e.,
it returns 1, but because of a different, earlier error).

do_something || exit 1
or
'set -e'

are both pretty common patterns, in both cases tests should fail.

In those test-cases that deliberately test for an error,
nft something_should_fail || exit 0
nft something_should_fail && exit 1

or a similar constructs should be used.

This initial commit modififies all '_1' scripts to return 0 on
success, usually via 'nft wrong || exit 0'.

All tests pass, except the one broken test case that hasn't worked
before either, but where 'set -e' use made it pass (the failing command
is supposed to work, and the command that is supposed to fail is never
run).

Signed-off-by: Florian Westphal <fw@strlen.de>

47 files changed, 130 insertions, 87 deletions

diff --git a/tests/shell/run-tests.sh b/tests/shell/run-tests.sh
index fdca5fb3..6b693cc1 100755
--- a/tests/shell/run-tests.sh
+++ b/tests/shell/run-tests.sh
@@ -4,7 +4,6 @@
 TESTDIR="./$(dirname $0)/"
 RETURNCODE_SEPARATOR="_"
 SRC_NFT="$(dirname $0)/../../src/nft"
-POSITIVE_RET=0
 DIFF=$(which diff)
 
 msg_error() {
@@ -102,29 +101,27 @@ for testfile in $(find_tests)
 do
 	kernel_cleanup
 
-	rc_spec=$(awk -F${RETURNCODE_SEPARATOR} '{print $NF}' <<< $testfile)
-
 	msg_info "[EXECUTING]	$testfile"
 	test_output=$(NFT=$NFT ${testfile} 2>&1)
 	rc_got=$?
 	echo -en "\033[1A\033[K" # clean the [EXECUTING] foobar line
 
-	if [ "$rc_got" == "$rc_spec" ] ; then
+	if [ "$rc_got" -eq 0 ] ; then
 		# check nft dump only for positive tests
-		rc_spec="${POSITIVE_RET}"
 		dumppath="$(dirname ${testfile})/dumps"
 		dumpfile="${dumppath}/$(basename ${testfile}).nft"
-		if [ "$rc_got" == "${POSITIVE_RET}" ] && [ -f ${dumpfile} ]; then
+		rc_spec=0
+		if [ "$rc_got" -eq 0 ] && [ -f ${dumpfile} ]; then
 			test_output=$(${DIFF} ${dumpfile} <($NFT list ruleset) 2>&1)
 			rc_spec=$?
 		fi
 
-		if [ "$rc_spec" == "${POSITIVE_RET}" ]; then
+		if [ "$rc_spec" -eq 0 ]; then
 			msg_info "[OK]		$testfile"
 			[ "$VERBOSE" == "y" ] && [ ! -z "$test_output" ] && echo "$test_output"
 			((ok++))
 
-			if [ "$DUMPGEN" == "y" ] && [ "$rc_got" == "${POSITIVE_RET}" ] && [ ! -f "${dumpfile}" ]; then
+			if [ "$DUMPGEN" == "y" ] && [ "$rc_got" == 0 ] && [ ! -f "${dumpfile}" ]; then
 				mkdir -p "${dumppath}"
 				nft list ruleset > "${dumpfile}"
 			fi
@@ -140,7 +137,7 @@ do
 	else
 		((failed++))
 		if [ "$VERBOSE" == "y" ] ; then
-			msg_warn "[FAILED]	$testfile: expected $rc_spec but got $rc_got"
+			msg_warn "[FAILED]	$testfile: got $rc_got"
 			[ ! -z "$test_output" ] && echo "$test_output"
 		else
 			msg_warn "[FAILED]	$testfile"
diff --git a/tests/shell/testcases/chains/0002jumps_1 b/tests/shell/testcases/chains/0002jumps_1
index 4d163b05..aa70037f 100755
--- a/tests/shell/testcases/chains/0002jumps_1
+++ b/tests/shell/testcases/chains/0002jumps_1
@@ -20,5 +20,7 @@ done
 
 # this last jump should fail: too many links
 $NFT add chain t c$((MAX_JUMPS + 1))
-$NFT add rule t c${MAX_JUMPS} jump c$((MAX_JUMPS + 1)) 2>/dev/null
+
+$NFT add rule t c${MAX_JUMPS} jump c$((MAX_JUMPS + 1)) 2>/dev/null || exit 0
 echo "E: max jumps ignored?" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0003jump_loop_1 b/tests/shell/testcases/chains/0003jump_loop_1
index f74361f2..80e243f0 100755
--- a/tests/shell/testcases/chains/0003jump_loop_1
+++ b/tests/shell/testcases/chains/0003jump_loop_1
@@ -17,5 +17,6 @@ do
 done
 
 # this last jump should fail: loop
-$NFT add rule t c${MAX_JUMPS} jump c1 2>/dev/null
+$NFT add rule t c${MAX_JUMPS} jump c1 2>/dev/null || exit 0
 echo "E: loop of jumps ignored?" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0004busy_1 b/tests/shell/testcases/chains/0004busy_1
index cc9a0dad..e68d1baa 100755
--- a/tests/shell/testcases/chains/0004busy_1
+++ b/tests/shell/testcases/chains/0004busy_1
@@ -6,6 +6,8 @@ $NFT add table t
 $NFT add chain t c1
 $NFT add chain t c2
 $NFT add rule t c1 jump c2
+
 # kernel should return EBUSY
-$NFT delete chain t c2 2>/dev/null
+$NFT delete chain t c2 2>/dev/null || exit 0
 echo "E: deleted a busy chain?" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0005busy_map_1 b/tests/shell/testcases/chains/0005busy_map_1
index 93eca827..c800f193 100755
--- a/tests/shell/testcases/chains/0005busy_map_1
+++ b/tests/shell/testcases/chains/0005busy_map_1
@@ -6,6 +6,8 @@ $NFT add table t
 $NFT add chain t c1
 $NFT add chain t c2
 $NFT add rule t c1 tcp dport vmap { 1 : jump c2 }
+
 # kernel should return EBUSY
-$NFT delete chain t c2 2>/dev/null
+$NFT delete chain t c2 2>/dev/null || exit 0
 echo "E: deleted a busy chain?" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0007masquerade_1 b/tests/shell/testcases/chains/0007masquerade_1
index 4e98d106..4434c898 100755
--- a/tests/shell/testcases/chains/0007masquerade_1
+++ b/tests/shell/testcases/chains/0007masquerade_1
@@ -4,6 +4,8 @@ set -e
 
 $NFT add table t
 $NFT add chain t c1 {type filter hook output priority 0 \; }
+
 # wrong hook output, only postrouting is valid
-$NFT add rule t c1 masquerade 2>/dev/null
+$NFT add rule t c1 masquerade 2>/dev/null || exit 0
 echo "E: accepted masquerade in output hook" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0008masquerade_jump_1 b/tests/shell/testcases/chains/0008masquerade_jump_1
index 7754ed03..aee1475f 100755
--- a/tests/shell/testcases/chains/0008masquerade_jump_1
+++ b/tests/shell/testcases/chains/0008masquerade_jump_1
@@ -6,6 +6,8 @@ $NFT add table t
 $NFT add chain t output {type nat hook output priority 0 \; }
 $NFT add chain t c1
 $NFT add rule t c1 masquerade
+
 # kernel should return EOPNOTSUPP
-$NFT add rule t output jump c1 2>/dev/null
+$NFT add rule t output jump c1 2>/dev/null || exit 0
 echo "E: accepted masquerade in output hook" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0009masquerade_jump_1 b/tests/shell/testcases/chains/0009masquerade_jump_1
index 684d4417..2b931eeb 100755
--- a/tests/shell/testcases/chains/0009masquerade_jump_1
+++ b/tests/shell/testcases/chains/0009masquerade_jump_1
@@ -6,6 +6,8 @@ $NFT add table t
 $NFT add chain t output {type nat hook output priority 0 \; }
 $NFT add chain t c1
 $NFT add rule t c1 masquerade
+
 # kernel should return EOPNOTSUPP
-$NFT add rule t output tcp dport vmap {1 :jump c1 } 2>/dev/null
+$NFT add rule t output tcp dport vmap {1 :jump c1 } 2>/dev/null || exit 0
 echo "E: accepted masquerade in output hook in a vmap" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0010endless_jump_loop_1 b/tests/shell/testcases/chains/0010endless_jump_loop_1
index dba70e14..5d3ef239 100755
--- a/tests/shell/testcases/chains/0010endless_jump_loop_1
+++ b/tests/shell/testcases/chains/0010endless_jump_loop_1
@@ -4,6 +4,8 @@ set -e
 
 $NFT add table t
 $NFT add chain t c
+
 # kernel should return ELOOP
-$NFT add rule t c tcp dport vmap {1 : jump c} 2>/dev/null
+$NFT add rule t c tcp dport vmap {1 : jump c} 2>/dev/null || exit 0
 echo "E: accepted endless jump loop in a vmap" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0011endless_jump_loop_1 b/tests/shell/testcases/chains/0011endless_jump_loop_1
index adbff8d4..d75932d7 100755
--- a/tests/shell/testcases/chains/0011endless_jump_loop_1
+++ b/tests/shell/testcases/chains/0011endless_jump_loop_1
@@ -10,5 +10,6 @@ $NFT add element t m {2 : jump c2}
 $NFT add rule t c1 tcp dport vmap @m
 
 # kernel should return ELOOP
-$NFT add element t m {1 : jump c1} 2>/dev/null
+$NFT add element t m {1 : jump c1} 2>/dev/null || exit 0
 echo "E: accepted endless jump loop in a vmap" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0012reject_in_prerouting_1 b/tests/shell/testcases/chains/0012reject_in_prerouting_1
index 81cda0c4..0ee86c11 100755
--- a/tests/shell/testcases/chains/0012reject_in_prerouting_1
+++ b/tests/shell/testcases/chains/0012reject_in_prerouting_1
@@ -4,6 +4,8 @@ set -e
 
 $NFT add table t
 $NFT add chain t prerouting {type filter hook prerouting priority 0 \; }
+
 # wrong hook prerouting, only input/forward/output is valid
-$NFT add rule t prerouting reject 2>/dev/null
+$NFT add rule t prerouting reject 2>/dev/null || exit 0
 echo "E: accepted reject in prerouting hook" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0015check_jump_loop_1 b/tests/shell/testcases/chains/0015check_jump_loop_1
index ba40ddb9..a59bb3bf 100755
--- a/tests/shell/testcases/chains/0015check_jump_loop_1
+++ b/tests/shell/testcases/chains/0015check_jump_loop_1
@@ -7,5 +7,7 @@ $NFT add chain t c1
 $NFT add chain t c2
 $NFT add t c1 jump c2
 # kernel should return ENOENT
-$NFT add t c2 ip daddr vmap { 1 : jump c3 }
+
+$NFT add t c2 ip daddr vmap { 1 : jump c3 } || exit 0
 echo "E: Jumped to non existing chain" >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0017masquerade_jump_1 b/tests/shell/testcases/chains/0017masquerade_jump_1
index a57675f5..209e6d48 100755
--- a/tests/shell/testcases/chains/0017masquerade_jump_1
+++ b/tests/shell/testcases/chains/0017masquerade_jump_1
@@ -6,5 +6,9 @@ $NFT add table t
 $NFT add chain t input {type filter hook input priority 4 \; }
 $NFT add chain t c1
 $NFT add rule t input jump c1
+
 # kernel should return EOPNOTSUPP
-$NFT add rule t c1 masquerade 2>/dev/null >&2
+$NFT add rule t c1 masquerade 2>/dev/null >&2 || exit 0
+
+echo "E: Accepted masquerade rule in non-nat type base chain" 1>&2
+exit 1
diff --git a/tests/shell/testcases/chains/0018check_jump_loop_1 b/tests/shell/testcases/chains/0018check_jump_loop_1
index d1443dab..b87520f2 100755
--- a/tests/shell/testcases/chains/0018check_jump_loop_1
+++ b/tests/shell/testcases/chains/0018check_jump_loop_1
@@ -6,5 +6,8 @@ $NFT add table ip filter
 $NFT add chain ip filter ap1
 $NFT add chain ip filter ap2
 $NFT add rule ip filter ap1 jump ap2
+
 # kernel should return EOPNOTSUPP
-$NFT add rule ip filter ap1 jump ap1 2>/dev/null >&2
+$NFT add rule ip filter ap1 jump ap1 2>/dev/null >&2 || exit 0
+echo "E: Accepted jump-to-self"
+exit 1
diff --git a/tests/shell/testcases/chains/0019masquerade_jump_1 b/tests/shell/testcases/chains/0019masquerade_jump_1
index 4fe68c84..0ff1ac3f 100755
--- a/tests/shell/testcases/chains/0019masquerade_jump_1
+++ b/tests/shell/testcases/chains/0019masquerade_jump_1
@@ -6,5 +6,8 @@ $NFT add table t
 $NFT add chain t input {type filter hook input priority 4 \; }
 $NFT add chain t c1
 $NFT add rule t input ip saddr vmap { 1.1.1.1 : jump c1 }
+
 # kernel should return EOPNOTSUPP
-$NFT add rule t c1 masquerade 2>/dev/null >&2
+$NFT add rule t c1 masquerade 2>/dev/null >&2 || exit 0
+echo "E: accepted masquerade in chain from non-nat type basechain" 1>&2
+exit 1
diff --git a/tests/shell/testcases/chains/0020depth_1 b/tests/shell/testcases/chains/0020depth_1
index fa539c8f..23e1f826 100755
--- a/tests/shell/testcases/chains/0020depth_1
+++ b/tests/shell/testcases/chains/0020depth_1
@@ -1,7 +1,6 @@
 #!/bin/bash
 
 set -e
-
 $NFT add table ip filter
 $NFT add chain ip filter input { type filter hook input priority 0\; }
 
@@ -19,4 +18,6 @@ for ((i=11;i<19;i++)); do
 	$NFT add rule ip filter a$i jump a$((i+1))
 done
 
-$NFT add rule ip filter a10 jump a11
+$NFT add rule ip filter a10 jump a11 || exit 0
+echo "E: Expected 20th jump to fail due to jump stack exhaustion" 1>&2
+exit 1
diff --git a/tests/shell/testcases/chains/0022prio_dummy_1 b/tests/shell/testcases/chains/0022prio_dummy_1
index ecdd9456..66c44074 100755
--- a/tests/shell/testcases/chains/0022prio_dummy_1
+++ b/tests/shell/testcases/chains/0022prio_dummy_1
@@ -3,5 +3,7 @@
 set -e
 
 $NFT add table ip x
-$NFT add chain ip x y "{ type filter hook input priority dummy+1; }" &> /dev/null
+
+$NFT add chain ip x y "{ type filter hook input priority dummy+1; }" &> /dev/null || exit 0
 echo "E: dummy should not be a valid priority." >&2
+exit 1
diff --git a/tests/shell/testcases/chains/0023prio_inet_srcnat_1 b/tests/shell/testcases/chains/0023prio_inet_srcnat_1
index fa53f7a7..d2b1fa43 100755
--- a/tests/shell/testcases/chains/0023prio_inet_srcnat_1
+++ b/tests/shell/testcases/chains/0023prio_inet_srcnat_1
@@ -9,8 +9,8 @@ do
 		if (($? == 0))
 		then
 			echo "E: srcnat should not be a valid priority name in $family $hook chains." >&2
-			exit 0
+			exit 1
 		fi
 	done
 done
-exit 1
+exit 0
diff --git a/tests/shell/testcases/chains/0024prio_inet_dstnat_1 b/tests/shell/testcases/chains/0024prio_inet_dstnat_1
index a9a7264a..d112f2c9 100755
--- a/tests/shell/testcases/chains/0024prio_inet_dstnat_1
+++ b/tests/shell/testcases/chains/0024prio_inet_dstnat_1
@@ -9,8 +9,8 @@ do
 		if (($? == 0))
 		then
 			echo "E: dstnat should not be a valid priority name in $family $hook chains." >&2
-			exit 0
+			exit 1
 		fi
 	done
 done
-exit 1
+exit 0
diff --git a/tests/shell/testcases/chains/0025prio_arp_1 b/tests/shell/testcases/chains/0025prio_arp_1
index 8c671d55..1a172629 100755
--- a/tests/shell/testcases/chains/0025prio_arp_1
+++ b/tests/shell/testcases/chains/0025prio_arp_1
@@ -10,8 +10,8 @@ family=arp
 			if (($? == 0))
 			then
 				echo "E: $prioname should not be a valid priority name for arp family chains." >&2
-				exit 0
+				exit 1
 			fi
 		done
 	done
-exit 1
+exit 0
diff --git a/tests/shell/testcases/chains/0026prio_netdev_1 b/tests/shell/testcases/chains/0026prio_netdev_1
index ae022830..aa902e9b 100755
--- a/tests/shell/testcases/chains/0026prio_netdev_1
+++ b/tests/shell/testcases/chains/0026prio_netdev_1
@@ -4,12 +4,12 @@ family=netdev
 	hook=ingress
 		for prioname in raw mangle dstnat security srcnat
 		do
-			$NFT add table $family x
+			$NFT add table $family x || exit 1
 			$NFT add chain $family x y "{ type filter hook $hook device lo priority $prioname; }" &> /dev/null
 			if (($? == 0))
 			then
 				echo "E: $prioname should not be a valid priority name for netdev family chains." >&2
-				exit 0
+				exit 1
 			fi
 		done
-exit 1
+exit 0
diff --git a/tests/shell/testcases/chains/0027prio_bridge_dstnat_1 b/tests/shell/testcases/chains/0027prio_bridge_dstnat_1
index df0b6950..52c73e65 100755
--- a/tests/shell/testcases/chains/0027prio_bridge_dstnat_1
+++ b/tests/shell/testcases/chains/0027prio_bridge_dstnat_1
@@ -9,7 +9,7 @@ family=bridge
 			if (($? == 0))
 			then
 				echo "E: $prioname should not be a valid priority name for bridge $hook chains." >&2
-				exit 0
+				exit 1
 			fi
 	done
-exit 1
+exit 0
diff --git a/tests/shell/testcases/chains/0028prio_bridge_out_1 b/tests/shell/testcases/chains/0028prio_bridge_out_1
index 06fdbebb..63aa296c 100755
--- a/tests/shell/testcases/chains/0028prio_bridge_out_1
+++ b/tests/shell/testcases/chains/0028prio_bridge_out_1
@@ -9,7 +9,7 @@ family=bridge
 			if (($? == 0))
 			then
 				echo "E: $prioname should not be a valid priority name for bridge $hook chains." >&2
-				exit 0
+				exit 1
 			fi
 	done
-exit 1
+exit 0
diff --git a/tests/shell/testcases/chains/0029prio_bridge_srcnat_1 b/tests/shell/testcases/chains/0029prio_bridge_srcnat_1
index 8896a7cf..38917119 100755
--- a/tests/shell/testcases/chains/0029prio_bridge_srcnat_1
+++ b/tests/shell/testcases/chains/0029prio_bridge_srcnat_1
@@ -9,7 +9,7 @@ family=bridge
 			if (($? == 0))
 			then
 				echo "E: $prioname should not be a valid priority name for bridge $hook chains." >&2
-				exit 0
+				exit 1
 			fi
 	done
-exit 1
+exit 0
diff --git a/tests/shell/testcases/flowtable/0005delete_in_use_1 b/tests/shell/testcases/flowtable/0005delete_in_use_1
index 1b239f41..149d6444 100755
--- a/tests/shell/testcases/flowtable/0005delete_in_use_1
+++ b/tests/shell/testcases/flowtable/0005delete_in_use_1
@@ -5,5 +5,7 @@ $NFT add table x
 $NFT add chain x x
 $NFT add flowtable x y { hook ingress priority 0\; devices = { lo }\;}
 $NFT add rule x x flow offload @y
-$NFT delete flowtable x y
+
+$NFT delete flowtable x y || exit 0
 echo "E: delete flowtable in use"
+exit 1
diff --git a/tests/shell/testcases/flowtable/0008prio_1 b/tests/shell/testcases/flowtable/0008prio_1
index 87084b93..48953d79 100755
--- a/tests/shell/testcases/flowtable/0008prio_1
+++ b/tests/shell/testcases/flowtable/0008prio_1
@@ -7,8 +7,8 @@ do
 	if (($? == 0))
 	then
 		echo "E: $prioname should not be a valid priority name for flowtables" >&2
-		exit 0
+		exit 1
 	fi
 done
 
-exit 1
+exit 0
diff --git a/tests/shell/testcases/include/0004endlessloop_1 b/tests/shell/testcases/include/0004endlessloop_1
index c4aba0c4..3e6789d3 100755
--- a/tests/shell/testcases/include/0004endlessloop_1
+++ b/tests/shell/testcases/include/0004endlessloop_1
@@ -14,5 +14,6 @@ RULESET="include \"$tmpfile\""
 
 echo "$RULESET" > $tmpfile
 
-$NFT -f $tmpfile 2>/dev/null
+$NFT -f $tmpfile 2>/dev/null || exit 0
 echo "E: endless include loop" >&2
+exit 1
diff --git a/tests/shell/testcases/include/0009glob_nofile_1 b/tests/shell/testcases/include/0009glob_nofile_1
index bab58305..d769155a 100755
--- a/tests/shell/testcases/include/0009glob_nofile_1
+++ b/tests/shell/testcases/include/0009glob_nofile_1
@@ -26,8 +26,6 @@ RULESET1="include \"$tmpdir/non_existent_file.nft\""
 
 echo "$RULESET1" > $tmpfile1
 
-$NFT -f $tmpfile1
-if [ $? -eq 0 ] ; then
-        echo "E: Failed to catch a missing include directory/file" >&2
-        exit 1
-fi
+$NFT -f $tmpfile1 || exit 0
+echo "E: Failed to catch a missing include directory/file" >&2
+exit 1
diff --git a/tests/shell/testcases/include/0010glob_broken_file_1 b/tests/shell/testcases/include/0010glob_broken_file_1
index 9027f189..a00babf1 100755
--- a/tests/shell/testcases/include/0010glob_broken_file_1
+++ b/tests/shell/testcases/include/0010glob_broken_file_1
@@ -41,9 +41,6 @@ echo "$RULESET1" > $tmpfile1
 echo "$RULESET2" > $tmpfile2
 echo "$RULESET3" > $tmpfile3
 
-$NFT -f $tmpfile3
-
-if [ $? -eq 0 ] ; then
-        echo "E: didn't catch a broken file in directory" >&2
-        exit 1
-fi
+$NFT -f $tmpfile3 || exit 0
+echo "E: didn't catch a broken file in directory" >&2
+exit 1
diff --git a/tests/shell/testcases/include/0012glob_dependency_1 b/tests/shell/testcases/include/0012glob_dependency_1
index 6cf4ba17..e4e12e27 100755
--- a/tests/shell/testcases/include/0012glob_dependency_1
+++ b/tests/shell/testcases/include/0012glob_dependency_1
@@ -44,9 +44,6 @@ echo "$RULESET1" > $tmpfile2
 echo "$RULESET2" > $tmpfile1
 echo "$RULESET3" > $tmpfile3
 
-$NFT -f $tmpfile3
-
-if [ $? -eq 0 ] ; then
-        echo "E: did not catch wrong file order in include directory" >&2
-        exit 1
-fi
+$NFT -f $tmpfile3 || exit 0
+echo "E: did not catch wrong file order in include directory" >&2
+exit 1
diff --git a/tests/shell/testcases/maps/different_map_types_1 b/tests/shell/testcases/maps/different_map_types_1
index b0a09d02..a7e831ff 100755
--- a/tests/shell/testcases/maps/different_map_types_1
+++ b/tests/shell/testcases/maps/different_map_types_1
@@ -6,5 +6,8 @@ set -e
 
 $NFT add table ip filter
 $NFT add chain ip filter output { type filter hook output priority 0 \; }
-$NFT add rule ip filter output meta mark set tcp dport map { 22 : 1, 23 : 192.168.0.1 }
+
+$NFT add rule ip filter output meta mark set tcp dport map { 22 : 1, 23 : 192.168.0.1 } || exit 0
+
 echo "E: Added two different types of expression to map"
+exit 1
diff --git a/tests/shell/testcases/nft-f/0007action_object_set_segfault_1 b/tests/shell/testcases/nft-f/0007action_object_set_segfault_1
index 933a2f62..6cbd3869 100755
--- a/tests/shell/testcases/nft-f/0007action_object_set_segfault_1
+++ b/tests/shell/testcases/nft-f/0007action_object_set_segfault_1
@@ -10,4 +10,5 @@ add set t s {type ipv4_addr\;}
 add rule t c ip saddr @s
 "
 
-$NFT -f - <<< "$RULESET" 2>/dev/null
+$NFT -f - <<< "$RULESET" 2>/dev/null && exit 1
+exit 0
diff --git a/tests/shell/testcases/nft-f/0013defines_1 b/tests/shell/testcases/nft-f/0013defines_1
index 1dd5b569..b6330884 100755
--- a/tests/shell/testcases/nft-f/0013defines_1
+++ b/tests/shell/testcases/nft-f/0013defines_1
@@ -14,4 +14,5 @@ table ip t {
 	}
 }"
 
-$NFT -f - <<< "$RULESET"
+$NFT -f - <<< "$RULESET" && exit 1
+exit 0
diff --git a/tests/shell/testcases/nft-f/0014defines_1 b/tests/shell/testcases/nft-f/0014defines_1
index c8e73c24..35f2536f 100755
--- a/tests/shell/testcases/nft-f/0014defines_1
+++ b/tests/shell/testcases/nft-f/0014defines_1
@@ -14,4 +14,5 @@ table ip t {
 	}
 }"
 
-$NFT -f - <<< "$RULESET"
+$NFT -f - <<< "$RULESET" && exit 1
+exit 0
diff --git a/tests/shell/testcases/nft-f/0015defines_1 b/tests/shell/testcases/nft-f/0015defines_1
index 489c65b5..935cb458 100755
--- a/tests/shell/testcases/nft-f/0015defines_1
+++ b/tests/shell/testcases/nft-f/0015defines_1
@@ -13,4 +13,5 @@ table ip t {
 	}
 }"
 
-$NFT -f - <<< "$RULESET"
+$NFT -f - <<< "$RULESET" && exit 1
+exit 0
diff --git a/tests/shell/testcases/nft-f/0016redefines_1 b/tests/shell/testcases/nft-f/0016redefines_1
index ed702c90..9f6b56fe 100755
--- a/tests/shell/testcases/nft-f/0016redefines_1
+++ b/tests/shell/testcases/nft-f/0016redefines_1
@@ -30,3 +30,5 @@ if [ "$EXPECTED" != "$GET" ] ; then
         [ -x $DIFF ] && $DIFF -u <(echo "$EXPECTED") <(echo "$GET")
         exit 1
 fi
+
+exit 0
diff --git a/tests/shell/testcases/optionals/handles_1 b/tests/shell/testcases/optionals/handles_1
index a3ae1a7f..c00abfe8 100755
--- a/tests/shell/testcases/optionals/handles_1
+++ b/tests/shell/testcases/optionals/handles_1
@@ -5,4 +5,6 @@
 $NFT add table test
 $NFT add chain test test
 $NFT add rule test test tcp dport 22 counter accept
-$NFT list table test | grep 'accept # handle '[[:digit:]]$ >/dev/null
+( $NFT list table test | grep 'accept # handle '[[:digit:]]$ >/dev/null ) && exit 1
+
+exit 0
diff --git a/tests/shell/testcases/rule_management/0002addinsertlocation_1 b/tests/shell/testcases/rule_management/0002addinsertlocation_1
index b48d3d66..920032f2 100755
--- a/tests/shell/testcases/rule_management/0002addinsertlocation_1
+++ b/tests/shell/testcases/rule_management/0002addinsertlocation_1
@@ -17,7 +17,7 @@ for cmd in add insert; do
 		$NFT $cmd rule t c $keyword 5 drop 2>/dev/null || continue
 
 		echo "E: invalid $keyword value allowed in $cmd command" >&2
-		exit 0
+		exit 1
 	done
 done
-exit 1
+exit 0
diff --git a/tests/shell/testcases/rule_management/0005replace_1 b/tests/shell/testcases/rule_management/0005replace_1
index e82995a5..d8d64477 100755
--- a/tests/shell/testcases/rule_management/0005replace_1
+++ b/tests/shell/testcases/rule_management/0005replace_1
@@ -7,5 +7,7 @@ set -e
 $NFT add table t
 $NFT add chain t c
 # kernel should return ENOENT
-$NFT replace rule t c handle 2 drop 2>/dev/null
+
+$NFT replace rule t c handle 2 drop 2>/dev/null || exit 0
 echo "E: missing kernel ENOENT" >&2
+exit 1
diff --git a/tests/shell/testcases/rule_management/0006replace_1 b/tests/shell/testcases/rule_management/0006replace_1
index 5dfcba02..b728310f 100755
--- a/tests/shell/testcases/rule_management/0006replace_1
+++ b/tests/shell/testcases/rule_management/0006replace_1
@@ -6,6 +6,8 @@
 set -e
 $NFT add table t
 $NFT add chain t c
+
 # position keyword with replace action is not allowed, this should fail
-$NFT replace rule t c position 2 drop 2>/dev/null
+$NFT replace rule t c position 2 drop 2>/dev/null || exit 0
 echo "E: allowed replace with position specification" >&2
+exit 1
diff --git a/tests/shell/testcases/rule_management/0008delete_1 b/tests/shell/testcases/rule_management/0008delete_1
index 3dce2191..d1900d66 100755
--- a/tests/shell/testcases/rule_management/0008delete_1
+++ b/tests/shell/testcases/rule_management/0008delete_1
@@ -6,6 +6,8 @@
 set -e
 $NFT add table t
 $NFT add chain t c
+
 # this should fail, we don't allow delete with position
-$NFT delete rule t c position 2 drop 2>/dev/null
+$NFT delete rule t c position 2 drop 2>/dev/null || exit 0
 echo "E: allowed position spec with delete action" >&2
+exit 1
diff --git a/tests/shell/testcases/rule_management/0009delete_1 b/tests/shell/testcases/rule_management/0009delete_1
index 87fec605..8751fec3 100755
--- a/tests/shell/testcases/rule_management/0009delete_1
+++ b/tests/shell/testcases/rule_management/0009delete_1
@@ -6,6 +6,8 @@
 set -e
 $NFT add table t
 $NFT add chain t c
+
 # kernel ENOENT
-$NFT delete rule t c handle 3333 2>/dev/null
+$NFT delete rule t c handle 3333 2>/dev/null || exit 0
 echo "E: missing kernel ENOENT" >&2
+exit 1
diff --git a/tests/shell/testcases/sets/0018set_check_size_1 b/tests/shell/testcases/sets/0018set_check_size_1
index 833b8e2b..bc705605 100755
--- a/tests/shell/testcases/sets/0018set_check_size_1
+++ b/tests/shell/testcases/sets/0018set_check_size_1
@@ -5,4 +5,7 @@ $NFT add table x
 $NFT add set x s {type ipv4_addr\; size 2\;}
 $NFT add element x s {1.1.1.1}
 $NFT add element x s {1.1.1.2}
-$NFT add element x s {1.1.1.3}
+
+$NFT add element x s {1.1.1.3} || exit 0
+echo "E: Accepted 3rd element in a table with max size of 2" 1>&2
+exit 1
diff --git a/tests/shell/testcases/transactions/0014chain_1 b/tests/shell/testcases/transactions/0014chain_1
index 802a7e63..cddc8a2e 100755
--- a/tests/shell/testcases/transactions/0014chain_1
+++ b/tests/shell/testcases/transactions/0014chain_1
@@ -1,11 +1,10 @@
 #!/bin/bash
 
-set -e
-
 RULESET="add table x
 add chain x y
 delete chain x y
 delete chain x y"
 
-$NFT -f - <<< "$RULESET" 2>/dev/null
+$NFT -f - <<< "$RULESET" 2>/dev/null || exit 0
 echo "E: allowing double-removal of chain" >&2
+exit 1
diff --git a/tests/shell/testcases/transactions/0022rule_1 b/tests/shell/testcases/transactions/0022rule_1
index 0e7c9a6f..07be53f2 100755
--- a/tests/shell/testcases/transactions/0022rule_1
+++ b/tests/shell/testcases/transactions/0022rule_1
@@ -1,12 +1,11 @@
 #!/bin/bash
 
-set -e
-
 RULESET="add table x
 add chain x y
 delete chain x y
 add rule x y jump y"
 
 # kernel must return ENOENT
-$NFT -f - <<< "$RULESET" 2>/dev/null
+$NFT -f - <<< "$RULESET" 2>/dev/null || exit 0
 echo "E: allowing jump loop to unexisting chain"
+exit 1
diff --git a/tests/shell/testcases/transactions/0023rule_1 b/tests/shell/testcases/transactions/0023rule_1
index edc4e8d2..e58c088c 100755
--- a/tests/shell/testcases/transactions/0023rule_1
+++ b/tests/shell/testcases/transactions/0023rule_1
@@ -1,11 +1,10 @@
 #!/bin/bash
 
-set -e
-
 RULESET="add table x
 add chain x y
 add rule x y jump y"
 
 # kernel must return ELOOP
-$NFT -f - <<< "$RULESET" 2>/dev/null
+$NFT -f - <<< "$RULESET" 2>/dev/null || exit 0
 echo "E: allowing jump to chain loop"
+exit 1
diff --git a/tests/shell/testcases/transactions/0036set_1 b/tests/shell/testcases/transactions/0036set_1
index e691fa7f..45d922eb 100755
--- a/tests/shell/testcases/transactions/0036set_1
+++ b/tests/shell/testcases/transactions/0036set_1
@@ -1,13 +1,12 @@
 #!/bin/bash
 
-set -e
-
 RULESET="add table x
 add set x y { type ipv4_addr; }
 add element x y { 1.1.1.1, 2.2.2.2 }
 delete element x y { 1.1.1.1 }
 delete element x y { 1.1.1.1 }"
 
-$NFT -f - <<< "$RULESET" 2> /dev/null
+$NFT -f - <<< "$RULESET" 2> /dev/null || exit 0
 # Kernel must return ENOENT
 echo "E: allowing double-removal of element"
+exit 1