tests: monitor: Test JSON output as well

Enhance monitor test suite to test check JSON output as well. Note that
for now there is no support for --echo output testing with JSON.

Signed-off-by: Phil Sutter <phil@nwl.cc>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>

8 files changed, 117 insertions, 11 deletions

diff --git a/tests/monitor/README b/tests/monitor/README
index 9c5e37f5..39096a7f 100644
--- a/tests/monitor/README
+++ b/tests/monitor/README
@@ -15,13 +15,14 @@ to be established manually, i.e. in order to test monitor output when adding a
 chain, the table containing it has to be created first. In between each
 testcase, rule set is flushed completely.
 
-Input and output lines are prefixed by 'I' and 'O', respectively. The prefix has
-to be separated from the rest of the line by whitespace. Consecutive input lines
-are passed to 'nft' together, hence lead to a single transaction.
+Input lines are prefixed by 'I'. Multiple consecutive input lines are passed to
+'nft' together, hence lead to a single transaction.
 
-Since in most cases output should be equal to input, there is a shortcut: If a
-line consists of 'O -' only, the test script uses all previous input lines as
-expected output directly.
+There are two types of output lines: Those for standard syntax, prefixed by 'O'
+and those for JSON output, prefixed by 'J'. For standard syntax output lines,
+there is a shortcut: If a line consists of 'O -' only, the test script uses all
+previous input lines as expected output directly. Of course this is not
+available for JSON output lines.
 
 Empty lines and those starting with '#' are ignored.
 
@@ -29,8 +30,8 @@ Test Script Semantics
 ---------------------
 
 The script iterates over all test case files, reading them line by line. It
-assumes that sections of 'I' lines alternate with sections of 'O' lines. After
-stripping the prefix, each line is appended to a temporary file. There are
+assumes that sections of 'I' lines alternate with sections of 'O'/'J' lines.
+After stripping the prefix, each line is appended to a temporary file. There are
 separate files for input and output lines.
 
 If a set of input and output lines is complete (i.e. upon encountering either a
@@ -46,3 +47,13 @@ Note: Running 'nft monitor' in background is prone to race conditions. Hence
 an artificial delay is introduced before calling 'nft -f' to allow for 'nft
 monitor' to complete initialization and another one before comparing the output
 to allow for 'nft monitor' to process the netlink events.
+
+By default, only standard syntax is being tested for, i.e. 'J'-prefixed lines
+are simply ignored. If JSON testing was requested (by passing '-j' flag to the
+test script), 'O'-prefixed lines in turn are ignored.
+
+There is one caveat with regards to JSON output: Since it always contains handle
+properties (if the given object possesses such) which is supposed to be
+arbitrary, there is a filter script which normalizes all handle values in
+monitor output to zero before comparison. Therefore expected output must have
+all handle properties present but with a value of zero.
diff --git a/tests/monitor/run-tests.sh b/tests/monitor/run-tests.sh
index 1adabda1..f4089887 100755
--- a/tests/monitor/run-tests.sh
+++ b/tests/monitor/run-tests.sh
@@ -3,6 +3,7 @@
 cd $(dirname $0)
 nft=../../src/nft
 debug=false
+test_json=false
 
 mydiff() {
 	diff -w -I '^# ' "$@"
@@ -47,9 +48,16 @@ echo_output_append() {
 	}
 	[[ "$*" =~ ^add|replace|insert ]] && echo "$*" >>$output_file
 }
+json_output_filter() { # (filename)
+	# unify handle values
+	sed -i -e 's/\("handle":\) [0-9][0-9]*/\1 0/g' "$1"
+}
 monitor_run_test() {
 	monitor_output=$(mktemp -p $testdir)
-	$nft -nn monitor >$monitor_output &
+	monitor_args=""
+	$test_json && monitor_args="vm json"
+
+	$nft -nn monitor $monitor_args >$monitor_output &
 	monitor_pid=$!
 
 	sleep 0.5
@@ -67,6 +75,7 @@ monitor_run_test() {
 	sleep 0.5
 	kill $monitor_pid
 	wait >/dev/null 2>&1
+	$test_json && json_output_filter $monitor_output
 	if ! mydiff -q $monitor_output $output_file >/dev/null 2>&1; then
 		echo "monitor output differs!"
 		mydiff -u $output_file $monitor_output
@@ -99,7 +108,33 @@ echo_run_test() {
 	touch $output_file
 }
 
-for variant in monitor echo; do
+while [ -n "$1" ]; do
+	case "$1" in
+	-d|--debug)
+		debug=true
+		shift
+		;;
+	-j|--json)
+		test_json=true
+		shift
+		;;
+	*)
+		echo "unknown option '$1'"
+		;&
+	-h|--help)
+		echo "Usage: $(basename $0) [-j|--json] [-d|--debug]"
+		exit 1
+		;;
+	esac
+done
+
+if $test_json; then
+	variants="monitor"
+else
+	variants="monitor echo"
+fi
+
+for variant in $variants; do
 	run_test=${variant}_run_test
 	output_append=${variant}_output_append
 
@@ -124,7 +159,11 @@ for variant in monitor echo; do
 				;;
 			O)
 				input_complete=true
-				$output_append "$line"
+				$test_json || $output_append "$line"
+				;;
+			J)
+				input_complete=true
+				$test_json && $output_append "$line"
 				;;
 			'#'|'')
 				# ignore comments and empty lines
diff --git a/tests/monitor/testcases/object.t b/tests/monitor/testcases/object.t
index 6695b0f0..dacfed29 100644
--- a/tests/monitor/testcases/object.t
+++ b/tests/monitor/testcases/object.t
@@ -1,33 +1,46 @@
 # first the setup
 I add table ip t
 O -
+J {"add": {"table": {"family": "ip", "name": "t", "handle": 0}}}
 
 I add counter ip t c
 O add counter ip t c { packets 0 bytes 0 }
+J {"add": {"counter": {"family": "ip", "name": "c", "table": "t", "handle": 0, "packets": 0, "bytes": 0}}}
 
 I delete counter ip t c
 O -
+J {"delete": {"counter": {"family": "ip", "name": "c", "table": "t", "handle": 0, "packets": 0, "bytes": 0}}}
 
+# FIXME: input/output shouldn't be asynchronous here
 I add quota ip t q 25 mbytes
 O add quota ip t q { 25 mbytes }
+J {"add": {"quota": {"family": "ip", "name": "q", "table": "t", "handle": 0, "bytes": 26214400, "used": 0, "inv": false}}}
 
 I delete quota ip t q
 O -
+J {"delete": {"quota": {"family": "ip", "name": "q", "table": "t", "handle": 0, "bytes": 26214400, "used": 0, "inv": false}}}
 
+# FIXME: input/output shouldn't be asynchronous here
 I add limit ip t l rate 1/second
 O add limit ip t l { rate 1/second }
+J {"add": {"limit": {"family": "ip", "name": "l", "table": "t", "handle": 0, "rate": 1, "per": "second", "burst": 5}}}
 
 I delete limit ip t l
 O -
+J {"delete": {"limit": {"family": "ip", "name": "l", "table": "t", "handle": 0, "rate": 1, "per": "second", "burst": 5}}}
 
 I add ct helper ip t cth { type "sip" protocol tcp; l3proto ip; }
 O -
+J {"add": {"ct helper": {"family": "ip", "name": "cth", "table": "t", "handle": 0, "type": "sip", "protocol": "tcp", "l3proto": "ip"}}}
 
 I delete ct helper ip t cth
 O -
+J {"delete": {"ct helper": {"family": "ip", "name": "cth", "table": "t", "handle": 0, "type": "sip", "protocol": "tcp", "l3proto": "ip"}}}
 
 I add ct timeout ip t ctt { protocol udp; l3proto ip; policy = { unreplied: 15, replied: 12 }; }
 O -
+J {"add": {"ct timeout": {"family": "ip", "name": "ctt", "table": "t", "handle": 0, "protocol": "udp", "l3proto": "ip", "policy": {"unreplied": 15, "replied": 12}}}}
 
 I delete ct timeout ip t ctt
 O -
+J {"delete": {"ct timeout": {"family": "ip", "name": "ctt", "table": "t", "handle": 0, "protocol": "udp", "l3proto": "ip", "policy": {"unreplied": 15, "replied": 12}}}}
diff --git a/tests/monitor/testcases/set-maps.t b/tests/monitor/testcases/set-maps.t
index 3d86720e..acda480d 100644
--- a/tests/monitor/testcases/set-maps.t
+++ b/tests/monitor/testcases/set-maps.t
@@ -2,9 +2,13 @@
 I add table ip t
 I add map ip t portip { type inet_service: ipv4_addr; flags interval; }
 O -
+J {"add": {"table": {"family": "ip", "name": "t", "handle": 0}}}
+J {"add": {"map": {"family": "ip", "name": "portip", "table": "t", "type": "inet_service", "handle": 0, "map": "ipv4_addr", "flags": ["interval"]}}}
 
 I add element ip t portip { 80-100: 10.0.0.1 }
 O -
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portip", "elem": {"set": [[{"range": [80, 100]}, "10.0.0.1"]]}}}}
 
 I add element ip t portip { 1024-65535: 10.0.0.1 }
 O -
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portip", "elem": {"set": [[{"range": [1024, 65535]}, "10.0.0.1"]]}}}}
diff --git a/tests/monitor/testcases/set-mixed.t b/tests/monitor/testcases/set-mixed.t
index 9c1c5323..08c20116 100644
--- a/tests/monitor/testcases/set-mixed.t
+++ b/tests/monitor/testcases/set-mixed.t
@@ -3,13 +3,20 @@ I add table ip t
 I add set ip t portrange { type inet_service; flags interval; }
 I add set ip t ports { type inet_service; }
 O -
+J {"add": {"table": {"family": "ip", "name": "t", "handle": 0}}}
+J {"add": {"set": {"family": "ip", "name": "portrange", "table": "t", "type": "inet_service", "handle": 0, "flags": ["interval"]}}}
+J {"add": {"set": {"family": "ip", "name": "ports", "table": "t", "type": "inet_service", "handle": 0}}}
 
 # make sure concurrent adds work
 I add element ip t portrange { 1024-65535 }
 I add element ip t ports { 10 }
 O -
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [1024, 65535]}]}}}}
+J {"add": {"element": {"family": "ip", "table": "t", "name": "ports", "elem": {"set": [10]}}}}
 
 # delete items again
 I delete element ip t portrange { 1024-65535 }
 I delete element ip t ports { 10 }
 O -
+J {"delete": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [1024, 65535]}]}}}}
+J {"delete": {"element": {"family": "ip", "table": "t", "name": "ports", "elem": {"set": [10]}}}}
diff --git a/tests/monitor/testcases/set-multiple.t b/tests/monitor/testcases/set-multiple.t
index ad91fac0..bd7a6246 100644
--- a/tests/monitor/testcases/set-multiple.t
+++ b/tests/monitor/testcases/set-multiple.t
@@ -3,8 +3,13 @@ I add table ip t
 I add set ip t portrange { type inet_service; flags interval; }
 I add set ip t portrange2 { type inet_service; flags interval; }
 O -
+J {"add": {"table": {"family": "ip", "name": "t", "handle": 0}}}
+J {"add": {"set": {"family": "ip", "name": "portrange", "table": "t", "type": "inet_service", "handle": 0, "flags": ["interval"]}}}
+J {"add": {"set": {"family": "ip", "name": "portrange2", "table": "t", "type": "inet_service", "handle": 0, "flags": ["interval"]}}}
 
 # make sure concurrent adds work
 I add element ip t portrange { 1024-65535 }
 I add element ip t portrange2 { 10-20 }
 O -
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [1024, 65535]}]}}}}
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange2", "elem": {"set": [{"range": [10, 20]}]}}}}
diff --git a/tests/monitor/testcases/set-simple.t b/tests/monitor/testcases/set-simple.t
index ebff7cbd..8ca4f324 100644
--- a/tests/monitor/testcases/set-simple.t
+++ b/tests/monitor/testcases/set-simple.t
@@ -2,15 +2,21 @@
 I add table ip t
 I add set ip t portrange { type inet_service; flags interval; }
 O -
+J {"add": {"table": {"family": "ip", "name": "t", "handle": 0}}}
+J {"add": {"set": {"family": "ip", "name": "portrange", "table": "t", "type": "inet_service", "handle": 0, "flags": ["interval"]}}}
 
 # adding some ranges
 I add element ip t portrange { 1-10 }
 O -
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [1, 10]}]}}}}
 I add element ip t portrange { 1024-65535 }
 O -
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [1024, 65535]}]}}}}
 I add element ip t portrange { 20-30, 40-50 }
 O add element ip t portrange { 20-30 }
 O add element ip t portrange { 40-50 }
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [20, 30]}]}}}}
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [40, 50]}]}}}}
 
 # test flushing -> elements are removed in reverse
 I flush set ip t portrange
@@ -18,25 +24,38 @@ O delete element ip t portrange { 1024-65535 }
 O delete element ip t portrange { 40-50 }
 O delete element ip t portrange { 20-30 }
 O delete element ip t portrange { 1-10 }
+J {"delete": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [1024, 65535]}]}}}}
+J {"delete": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [40, 50]}]}}}}
+J {"delete": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [20, 30]}]}}}}
+J {"delete": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [1, 10]}]}}}}
 
 # make sure lower scope boundary works
 I add element ip t portrange { 0-10 }
 O -
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [0, 10]}]}}}}
 
 # make sure half open before other element works
 I add element ip t portrange { 1024-65535 }
 I add element ip t portrange { 100-200 }
 O -
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [1024, 65535]}]}}}}
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [100, 200]}]}}}}
 
 # make sure deletion of elements works
 I delete element ip t portrange { 0-10 }
 O -
+J {"delete": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [0, 10]}]}}}}
 I delete element ip t portrange { 100-200 }
 I delete element ip t portrange { 1024-65535 }
 O -
+J {"delete": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [100, 200]}]}}}}
+J {"delete": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [1024, 65535]}]}}}}
 
 # make sure mixed add/delete works
 I add element ip t portrange { 10-20 }
 I add element ip t portrange { 1024-65535 }
 I delete element ip t portrange { 10-20 }
 O -
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [10, 20]}]}}}}
+J {"add": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [1024, 65535]}]}}}}
+J {"delete": {"element": {"family": "ip", "table": "t", "name": "portrange", "elem": {"set": [{"range": [10, 20]}]}}}}
diff --git a/tests/monitor/testcases/simple.t b/tests/monitor/testcases/simple.t
index e4dc073e..78d8f8b0 100644
--- a/tests/monitor/testcases/simple.t
+++ b/tests/monitor/testcases/simple.t
@@ -2,19 +2,27 @@
 I add table ip t
 I add chain ip t c
 O -
+J {"add": {"table": {"family": "ip", "name": "t", "handle": 0}}}
+J {"add": {"chain": {"family": "ip", "table": "t", "name": "c", "handle": 0}}}
 
 I add rule ip t c accept
 O -
+J {"add": {"rule": {"family": "ip", "table": "t", "chain": "c", "handle": 0, "expr": [{"accept": null}]}}}
 
 I add rule ip t c tcp dport { 22, 80, 443 } accept
 O -
+J {"add": {"rule": {"family": "ip", "table": "t", "chain": "c", "handle": 0, "expr": [{"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": {"set": [22, 80, 443]}}}, {"accept": null}]}}}
 
 I insert rule ip t c counter accept
 O add rule ip t c counter packets 0 bytes 0 accept
+J {"add": {"rule": {"family": "ip", "table": "t", "chain": "c", "handle": 0, "expr": [{"counter": {"packets": 0, "bytes": 0}}, {"accept": null}]}}}
 
 I replace rule ip t c handle 2 accept comment "foo bar"
 O delete rule ip t c handle 2
 O add rule ip t c accept comment "foo bar"
+J {"delete": {"rule": {"family": "ip", "table": "t", "chain": "c", "handle": 0, "expr": [{"accept": null}]}}}
+J {"add": {"rule": {"family": "ip", "table": "t", "chain": "c", "handle": 0, "comment": "foo bar", "expr": [{"accept": null}]}}}
 
 I add counter ip t cnt
 O add counter ip t cnt { packets 0 bytes 0 }
+J {"add": {"counter": {"family": "ip", "name": "cnt", "table": "t", "handle": 0, "packets": 0, "bytes": 0}}}