add query_size field. handle user-specified dig output better - especially when querying dnssec

docs/parsers/dig.md

@@ -85,6 +85,7 @@ Schema:
             "data":             string
           }
         ],
+        "query_size":           integer,
         "query_time":           integer,   # in msec
         "server":               string,
         "when":                 string,

jc/parsers/dig.py

@@ -82,6 +82,7 @@ Schema:
             "data":             string
           }
         ],
+        "query_size":           integer,
         "query_time":           integer,   # in msec
         "server":               string,
         "when":                 string,
@@ -301,7 +302,7 @@ def _process(proc_data):
         List of Dictionaries. Structured data to conform to the schema.
     """
     for entry in proc_data:
-        int_list = ['id', 'query_num', 'answer_num', 'authority_num', 'additional_num', 'rcvd']
+        int_list = ['id', 'query_num', 'answer_num', 'authority_num', 'additional_num', 'rcvd', 'query_size']
         for key in int_list:
             if key in entry:
                 try:
@@ -483,6 +484,28 @@ def _parse_axfr(axfr):
             'data': axfr_data}
 
 
+def _parse_footer(footer):
+    # footer consists of 4 lines
+    # footer line 1
+    if footer.startswith(';; Query time:'):
+        return {'query_time': footer.split(':')[1].lstrip()}
+
+    # footer line 2
+    if footer.startswith(';; SERVER:'):
+        return {'server': footer.split(':', maxsplit=1)[1].lstrip()}
+
+    # footer line 3
+    if footer.startswith(';; WHEN:'):
+        return {'when': footer.split(':', maxsplit=1)[1].lstrip()}
+
+    # footer line 4 (last line)
+    if footer.startswith(';; MSG SIZE  rcvd:'):
+        return {'rcvd': footer.split(':')[1].lstrip()}
+
+    elif footer.startswith(';; XFR size:'):
+        return {'size': footer.split(':')[1].lstrip()}
+
+
 def parse(data, raw=False, quiet=False):
     """
     Main text parsing function
@@ -506,7 +529,7 @@ def parse(data, raw=False, quiet=False):
     # remove blank lines
     cleandata = list(filter(None, cleandata))
 
-    # section can be: header, flags, question, authority, answer, xfr, additional, opt_pseudosection, footer
+    # section can be: header, flags, question, authority, answer, axfr, additional, opt_pseudosection, footer
     section = ''
     output_entry = {}
 
@@ -514,6 +537,9 @@ def parse(data, raw=False, quiet=False):
         for line in cleandata:
 
             # identify sections
+            if line.startswith(';; Got answer:'):
+                section = ''
+                continue
 
             if line.startswith('; <<>> ') and ' axfr ' in line.lower():
                 section = 'axfr'
@@ -522,6 +548,8 @@ def parse(data, raw=False, quiet=False):
 
             if line.startswith(';; ->>HEADER<<-'):
                 section = 'header'
+                if output_entry:
+                    raw_output.append(output_entry)
                 output_entry = {}
                 output_entry.update(_parse_header(line))
                 continue
@@ -554,8 +582,17 @@ def parse(data, raw=False, quiet=False):
                 additional_list = []
                 continue
 
+            if line.startswith(';; Query time:'):
+                section = 'footer'
+                output_entry.update(_parse_footer(line))
+                continue
+
             # parse sections
 
+            if line.startswith(';; QUERY SIZE:'):
+                output_entry.update({'query_size': line.split(': ', maxsplit=1)[1]})
+                continue
+
             if not line.startswith(';') and section == 'axfr':
                 axfr_list.append(_parse_axfr(line))
                 output_entry.update({'axfr': axfr_list})
@@ -586,37 +623,12 @@ def parse(data, raw=False, quiet=False):
                 output_entry.update({'additional': additional_list})
                 continue
 
-            # footer consists of 4 lines
+            if section == 'footer':
-            # footer line 1
+                output_entry.update(_parse_footer(line))
-            if line.startswith(';; Query time:'):
-                section = 'footer'
-                output_entry.update({'query_time': line.split(':')[1].lstrip()})
                 continue
 
-            # footer line 2
+        if output_entry:
-            if line.startswith(';; SERVER:'):
+            raw_output.append(output_entry)
-                output_entry.update({'server': line.split(':', maxsplit=1)[1].lstrip()})
-                continue
-
-            # footer line 3
-            if line.startswith(';; WHEN:'):
-                output_entry.update({'when': line.split(':', maxsplit=1)[1].lstrip()})
-                continue
-
-            # footer line 4 (last line)
-            if line.startswith(';; MSG SIZE  rcvd:'):
-                section = ''
-                output_entry.update({'rcvd': line.split(':')[1].lstrip()})
-
-                if output_entry:
-                    raw_output.append(output_entry)
-
-            elif line.startswith(';; XFR size:'):
-                section = ''
-                output_entry.update({'size': line.split(':')[1].lstrip()})
-
-                if output_entry:
-                    raw_output.append(output_entry)
 
         raw_output = list(filter(None, raw_output))
 

tests/fixtures/generic/dig-edns3.json

@@ -1 +1 @@
-[{"id":37727,"opcode":"QUERY","status":"NOERROR","flags":["qr","rd","ra","ad"],"query_num":1,"answer_num":3,"authority_num":0,"additional_num":1,"opt_pseudosection":{"edns":{"version":0,"flags":["do"],"udp":512}},"question":{"name":"metebalci.com.","class":"IN","type":"A"},"answer":[{"name":"metebalci.com.","class":"IN","type":"A","ttl":299,"data":"151.101.1.195"},{"name":"metebalci.com.","class":"IN","type":"A","ttl":299,"data":"151.101.65.195"},{"name":"metebalci.com.","class":"IN","type":"RRSIG","ttl":299,"data":"A 8 2 300 20181227144044 20181205144044 59764 metebalci.com. z6FupNLEU/8OcB3rNMkVqVaan05Xu89T8hV6+IC7LGjWPtrD+TlNJd8D cGeq8xJLR8b1Q+gBK0QSxpGvk89GaCTjNtMGHLBBdgpyV4syFv2BNzK7 iAJhA8QJ6i5xVFJdzMSsn3WvQvN1W71sirt8+56r1nQ47aVkBSLJoZKP lgw="}],"query_time":41,"server":"8.8.8.8#53(8.8.8.8)","when":"Fri Dec 07 14:09:43 CET 2018","rcvd":247,"when_epoch":1544220583,"when_epoch_utc":null}]
+[{"id":37727,"opcode":"QUERY","status":"NOERROR","flags":["rd","ad"],"query_num":1,"answer_num":0,"authority_num":0,"additional_num":1,"opt_pseudosection":{"edns":{"version":0,"flags":["do"],"udp":4096},"cookie":"a263795b817be1b1"},"question":{"name":"metebalci.com.","class":"IN","type":"A"},"query_size":54},{"id":37727,"opcode":"QUERY","status":"NOERROR","flags":["qr","rd","ra","ad"],"query_num":1,"answer_num":3,"authority_num":0,"additional_num":1,"opt_pseudosection":{"edns":{"version":0,"flags":["do"],"udp":512}},"question":{"name":"metebalci.com.","class":"IN","type":"A"},"answer":[{"name":"metebalci.com.","class":"IN","type":"A","ttl":299,"data":"151.101.1.195"},{"name":"metebalci.com.","class":"IN","type":"A","ttl":299,"data":"151.101.65.195"},{"name":"metebalci.com.","class":"IN","type":"RRSIG","ttl":299,"data":"A 8 2 300 20181227144044 20181205144044 59764 metebalci.com. z6FupNLEU/8OcB3rNMkVqVaan05Xu89T8hV6+IC7LGjWPtrD+TlNJd8D cGeq8xJLR8b1Q+gBK0QSxpGvk89GaCTjNtMGHLBBdgpyV4syFv2BNzK7 iAJhA8QJ6i5xVFJdzMSsn3WvQvN1W71sirt8+56r1nQ47aVkBSLJoZKP lgw="}],"query_time":41,"server":"8.8.8.8#53(8.8.8.8)","when":"Fri Dec 07 14:09:43 CET 2018","rcvd":247,"when_epoch":1544220583,"when_epoch_utc":null}]