tests/py: Support testing JSON input and output as well

This extends nft-test.py by optional JSON testing capabilities,
activated via '-j'/'--enable-json' parameter).

JSON testing happens for all rules which are supposed to work: After a
rule has been added and the existing tests (payload, ruleset listing
output) have been performed, basically the same test is done again using
a recorded JSON equivalent and (if necessary) a recorded listing output.

The code tries to ease new test case creation overhead by
auto-generating JSON equivalent input via listing the (non-JSON) rule in
JSON format. Also, differing netlink debug and listing output are stored
in *.got files to assist in analyzing/fixing failing test cases.

Signed-off-by: Phil Sutter <phil@nwl.cc>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>

2 files changed, 1025 insertions, 0 deletions

diff --git a/tests/py/arp/arp.t.json b/tests/py/arp/arp.t.json
new file mode 100644
index 00000000..005fcf64
--- /dev/null
+++ b/tests/py/arp/arp.t.json
@@ -0,0 +1,877 @@
+# arp htype 1
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "right": 1
+        }
+    }
+]
+
+# arp htype != 1
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": 1
+        }
+    }
+]
+
+# arp htype 22
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "right": 22
+        }
+    }
+]
+
+# arp htype != 233
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": 233
+        }
+    }
+]
+
+# arp htype 33-45
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "range": [ 33, 45 ]
+            }
+        }
+    }
+]
+
+# arp htype != 33-45
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "range": [ 33, 45 ]
+            }
+        }
+    }
+]
+
+# arp htype { 33, 55, 67, 88}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "set": [
+                    33,
+                    55,
+                    67,
+                    88
+                ]
+            }
+        }
+    }
+]
+
+# arp htype != { 33, 55, 67, 88}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "set": [
+                    33,
+                    55,
+                    67,
+                    88
+                ]
+            }
+        }
+    }
+]
+
+# arp htype { 33-55}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "set": [
+                    { "range": [ 33, 55 ] }
+                ]
+            }
+        }
+    }
+]
+
+# arp htype != { 33-55}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "set": [
+                    { "range": [ 33, 55 ] }
+                ]
+            }
+        }
+    }
+]
+
+# arp ptype 0x0800
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "ptype",
+                    "name": "arp"
+                }
+            },
+            "right": "0x0800"
+        }
+    }
+]
+
+# arp hlen 22
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "hlen",
+                    "name": "arp"
+                }
+            },
+            "right": 22
+        }
+    }
+]
+
+# arp hlen != 233
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "hlen",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": 233
+        }
+    }
+]
+
+# arp hlen 33-45
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "hlen",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "range": [ 33, 45 ]
+            }
+        }
+    }
+]
+
+# arp hlen != 33-45
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "hlen",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "range": [ 33, 45 ]
+            }
+        }
+    }
+]
+
+# arp hlen { 33, 55, 67, 88}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "hlen",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "set": [
+                    33,
+                    55,
+                    67,
+                    88
+                ]
+            }
+        }
+    }
+]
+
+# arp hlen != { 33, 55, 67, 88}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "hlen",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "set": [
+                    33,
+                    55,
+                    67,
+                    88
+                ]
+            }
+        }
+    }
+]
+
+# arp hlen { 33-55}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "hlen",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "set": [
+                    { "range": [ 33, 55 ] }
+                ]
+            }
+        }
+    }
+]
+
+# arp hlen != { 33-55}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "hlen",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "set": [
+                    { "range": [ 33, 55 ] }
+                ]
+            }
+        }
+    }
+]
+
+# arp plen 22
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "plen",
+                    "name": "arp"
+                }
+            },
+            "right": 22
+        }
+    }
+]
+
+# arp plen != 233
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "plen",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": 233
+        }
+    }
+]
+
+# arp plen 33-45
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "plen",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "range": [ 33, 45 ]
+            }
+        }
+    }
+]
+
+# arp plen != 33-45
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "plen",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "range": [ 33, 45 ]
+            }
+        }
+    }
+]
+
+# arp plen { 33, 55, 67, 88}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "plen",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "set": [
+                    33,
+                    55,
+                    67,
+                    88
+                ]
+            }
+        }
+    }
+]
+
+# arp plen != { 33, 55, 67, 88}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "plen",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "set": [
+                    33,
+                    55,
+                    67,
+                    88
+                ]
+            }
+        }
+    }
+]
+
+# arp plen { 33-55}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "plen",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "set": [
+                    { "range": [ 33, 55 ] }
+                ]
+            }
+        }
+    }
+]
+
+# arp plen != {33-55}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "plen",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "set": [
+                    { "range": [ 33, 55 ] }
+                ]
+            }
+        }
+    }
+]
+
+# arp operation {nak, inreply, inrequest, rreply, rrequest, reply, request}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "set": [
+                    "nak",
+                    "inreply",
+                    "inrequest",
+                    "rreply",
+                    "rrequest",
+                    "reply",
+                    "request"
+                ]
+            }
+        }
+    }
+]
+
+# arp operation != {nak, inreply, inrequest, rreply, rrequest, reply, request}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "set": [
+                    "nak",
+                    "inreply",
+                    "inrequest",
+                    "rreply",
+                    "rrequest",
+                    "reply",
+                    "request"
+                ]
+            }
+        }
+    }
+]
+
+# arp operation request
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "right": "request"
+        }
+    }
+]
+
+# arp operation reply
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "right": "reply"
+        }
+    }
+]
+
+# arp operation rrequest
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "right": "rrequest"
+        }
+    }
+]
+
+# arp operation rreply
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "right": "rreply"
+        }
+    }
+]
+
+# arp operation inrequest
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "right": "inrequest"
+        }
+    }
+]
+
+# arp operation inreply
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "right": "inreply"
+        }
+    }
+]
+
+# arp operation nak
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "right": "nak"
+        }
+    }
+]
+
+# arp operation reply
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "right": "reply"
+        }
+    }
+]
+
+# arp operation != request
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": "request"
+        }
+    }
+]
+
+# arp operation != reply
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": "reply"
+        }
+    }
+]
+
+# arp operation != rrequest
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": "rrequest"
+        }
+    }
+]
+
+# arp operation != rreply
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": "rreply"
+        }
+    }
+]
+
+# arp operation != inrequest
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": "inrequest"
+        }
+    }
+]
+
+# arp operation != inreply
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": "inreply"
+        }
+    }
+]
+
+# arp operation != nak
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": "nak"
+        }
+    }
+]
+
+# arp operation != reply
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": "reply"
+        }
+    }
+]
+
+# meta iifname "invalid" arp ptype 0x0800 arp htype 1 arp hlen 6 arp plen 4 @nh,192,32 0xc0a88f10 @nh,144,48 set 0x112233445566
+[
+    {
+        "match": {
+            "left": {
+                "meta": "iifname"
+            },
+            "right": "invalid"
+        }
+    },
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "ptype",
+                    "name": "arp"
+                }
+            },
+            "right": "0x0800"
+        }
+    },
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "right": 1
+        }
+    },
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "hlen",
+                    "name": "arp"
+                }
+            },
+            "right": 6
+        }
+    },
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "plen",
+                    "name": "arp"
+                }
+            },
+            "right": 4
+        }
+    },
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "base": "nh",
+                    "len": 32,
+                    "name": "raw",
+                    "offset": 192
+                }
+            },
+            "right": "0xc0a88f10"
+        }
+    },
+    {
+        "mangle": {
+            "left": {
+                "payload": {
+                    "base": "nh",
+                    "len": 48,
+                    "name": "raw",
+                    "offset": 144
+                }
+            },
+            "right": "0x112233445566"
+        }
+    }
+]
+
diff --git a/tests/py/arp/arp.t.json.output b/tests/py/arp/arp.t.json.output
new file mode 100644
index 00000000..0120025b
--- /dev/null
+++ b/tests/py/arp/arp.t.json.output
@@ -0,0 +1,148 @@
+# arp ptype 0x0800
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "ptype",
+                    "name": "arp"
+                }
+            },
+            "right": "ip"
+        }
+    }
+]
+
+# arp operation {nak, inreply, inrequest, rreply, rrequest, reply, request}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "right": {
+                "set": [
+                    "request",
+                    "reply",
+                    "rrequest",
+                    "rreply",
+                    "inrequest",
+                    "inreply",
+                    "nak"
+                ]
+            }
+        }
+    }
+]
+
+# arp operation != {nak, inreply, inrequest, rreply, rrequest, reply, request}
+[
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "operation",
+                    "name": "arp"
+                }
+            },
+            "op": "!=",
+            "right": {
+                "set": [
+                    "request",
+                    "reply",
+                    "rrequest",
+                    "rreply",
+                    "inrequest",
+                    "inreply",
+                    "nak"
+                ]
+            }
+        }
+    }
+]
+
+# meta iifname "invalid" arp ptype 0x0800 arp htype 1 arp hlen 6 arp plen 4 @nh,192,32 0xc0a88f10 @nh,144,48 set 0x112233445566
+[
+    {
+        "match": {
+            "left": {
+                "meta": "iifname"
+            },
+            "right": "invalid"
+        }
+    },
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "htype",
+                    "name": "arp"
+                }
+            },
+            "right": 1
+        }
+    },
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "ptype",
+                    "name": "arp"
+                }
+            },
+            "right": "ip"
+        }
+    },
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "hlen",
+                    "name": "arp"
+                }
+            },
+            "right": 6
+        }
+    },
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "field": "plen",
+                    "name": "arp"
+                }
+            },
+            "right": 4
+        }
+    },
+    {
+        "match": {
+            "left": {
+                "payload": {
+                    "base": "nh",
+                    "len": 32,
+                    "name": "raw",
+                    "offset": 192
+                }
+            },
+            "right": 3232272144
+        }
+    },
+    {
+        "mangle": {
+            "left": {
+                "payload": {
+                    "base": "nh",
+                    "len": 48,
+                    "name": "raw",
+                    "offset": 144
+                }
+            },
+            "right": 18838586676582
+        }
+    }
+]
+