diff --git a/docs/parsers/dig.md b/docs/parsers/dig.md index ceb6a960..a7d4943c 100644 --- a/docs/parsers/dig.md +++ b/docs/parsers/dig.md @@ -85,6 +85,7 @@ Schema: "data": string } ], + "query_size": integer, "query_time": integer, # in msec "server": string, "when": string, diff --git a/jc/parsers/dig.py b/jc/parsers/dig.py index 7325fa36..8bbb621c 100644 --- a/jc/parsers/dig.py +++ b/jc/parsers/dig.py @@ -82,6 +82,7 @@ Schema: "data": string } ], + "query_size": integer, "query_time": integer, # in msec "server": string, "when": string, @@ -301,7 +302,7 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ for entry in proc_data: - int_list = ['id', 'query_num', 'answer_num', 'authority_num', 'additional_num', 'rcvd'] + int_list = ['id', 'query_num', 'answer_num', 'authority_num', 'additional_num', 'rcvd', 'query_size'] for key in int_list: if key in entry: try: @@ -483,6 +484,28 @@ def _parse_axfr(axfr): 'data': axfr_data} +def _parse_footer(footer): + # footer consists of 4 lines + # footer line 1 + if footer.startswith(';; Query time:'): + return {'query_time': footer.split(':')[1].lstrip()} + + # footer line 2 + if footer.startswith(';; SERVER:'): + return {'server': footer.split(':', maxsplit=1)[1].lstrip()} + + # footer line 3 + if footer.startswith(';; WHEN:'): + return {'when': footer.split(':', maxsplit=1)[1].lstrip()} + + # footer line 4 (last line) + if footer.startswith(';; MSG SIZE rcvd:'): + return {'rcvd': footer.split(':')[1].lstrip()} + + elif footer.startswith(';; XFR size:'): + return {'size': footer.split(':')[1].lstrip()} + + def parse(data, raw=False, quiet=False): """ Main text parsing function @@ -506,7 +529,7 @@ def parse(data, raw=False, quiet=False): # remove blank lines cleandata = list(filter(None, cleandata)) - # section can be: header, flags, question, authority, answer, xfr, additional, opt_pseudosection, footer + # section can be: header, flags, question, authority, answer, axfr, additional, opt_pseudosection, footer section = '' output_entry = {} @@ -514,6 +537,9 @@ def parse(data, raw=False, quiet=False): for line in cleandata: # identify sections + if line.startswith(';; Got answer:'): + section = '' + continue if line.startswith('; <<>> ') and ' axfr ' in line.lower(): section = 'axfr' @@ -522,6 +548,8 @@ def parse(data, raw=False, quiet=False): if line.startswith(';; ->>HEADER<<-'): section = 'header' + if output_entry: + raw_output.append(output_entry) output_entry = {} output_entry.update(_parse_header(line)) continue @@ -554,8 +582,17 @@ def parse(data, raw=False, quiet=False): additional_list = [] continue + if line.startswith(';; Query time:'): + section = 'footer' + output_entry.update(_parse_footer(line)) + continue + # parse sections + if line.startswith(';; QUERY SIZE:'): + output_entry.update({'query_size': line.split(': ', maxsplit=1)[1]}) + continue + if not line.startswith(';') and section == 'axfr': axfr_list.append(_parse_axfr(line)) output_entry.update({'axfr': axfr_list}) @@ -586,37 +623,12 @@ def parse(data, raw=False, quiet=False): output_entry.update({'additional': additional_list}) continue - # footer consists of 4 lines - # footer line 1 - if line.startswith(';; Query time:'): - section = 'footer' - output_entry.update({'query_time': line.split(':')[1].lstrip()}) + if section == 'footer': + output_entry.update(_parse_footer(line)) continue - # footer line 2 - if line.startswith(';; SERVER:'): - output_entry.update({'server': line.split(':', maxsplit=1)[1].lstrip()}) - continue - - # footer line 3 - if line.startswith(';; WHEN:'): - output_entry.update({'when': line.split(':', maxsplit=1)[1].lstrip()}) - continue - - # footer line 4 (last line) - if line.startswith(';; MSG SIZE rcvd:'): - section = '' - output_entry.update({'rcvd': line.split(':')[1].lstrip()}) - - if output_entry: - raw_output.append(output_entry) - - elif line.startswith(';; XFR size:'): - section = '' - output_entry.update({'size': line.split(':')[1].lstrip()}) - - if output_entry: - raw_output.append(output_entry) + if output_entry: + raw_output.append(output_entry) raw_output = list(filter(None, raw_output)) diff --git a/tests/fixtures/generic/dig-edns3.json b/tests/fixtures/generic/dig-edns3.json index a863d25f..c662b326 100644 --- a/tests/fixtures/generic/dig-edns3.json +++ b/tests/fixtures/generic/dig-edns3.json @@ -1 +1 @@ -[{"id":37727,"opcode":"QUERY","status":"NOERROR","flags":["qr","rd","ra","ad"],"query_num":1,"answer_num":3,"authority_num":0,"additional_num":1,"opt_pseudosection":{"edns":{"version":0,"flags":["do"],"udp":512}},"question":{"name":"metebalci.com.","class":"IN","type":"A"},"answer":[{"name":"metebalci.com.","class":"IN","type":"A","ttl":299,"data":"151.101.1.195"},{"name":"metebalci.com.","class":"IN","type":"A","ttl":299,"data":"151.101.65.195"},{"name":"metebalci.com.","class":"IN","type":"RRSIG","ttl":299,"data":"A 8 2 300 20181227144044 20181205144044 59764 metebalci.com. z6FupNLEU/8OcB3rNMkVqVaan05Xu89T8hV6+IC7LGjWPtrD+TlNJd8D cGeq8xJLR8b1Q+gBK0QSxpGvk89GaCTjNtMGHLBBdgpyV4syFv2BNzK7 iAJhA8QJ6i5xVFJdzMSsn3WvQvN1W71sirt8+56r1nQ47aVkBSLJoZKP lgw="}],"query_time":41,"server":"8.8.8.8#53(8.8.8.8)","when":"Fri Dec 07 14:09:43 CET 2018","rcvd":247,"when_epoch":1544220583,"when_epoch_utc":null}] +[{"id":37727,"opcode":"QUERY","status":"NOERROR","flags":["rd","ad"],"query_num":1,"answer_num":0,"authority_num":0,"additional_num":1,"opt_pseudosection":{"edns":{"version":0,"flags":["do"],"udp":4096},"cookie":"a263795b817be1b1"},"question":{"name":"metebalci.com.","class":"IN","type":"A"},"query_size":54},{"id":37727,"opcode":"QUERY","status":"NOERROR","flags":["qr","rd","ra","ad"],"query_num":1,"answer_num":3,"authority_num":0,"additional_num":1,"opt_pseudosection":{"edns":{"version":0,"flags":["do"],"udp":512}},"question":{"name":"metebalci.com.","class":"IN","type":"A"},"answer":[{"name":"metebalci.com.","class":"IN","type":"A","ttl":299,"data":"151.101.1.195"},{"name":"metebalci.com.","class":"IN","type":"A","ttl":299,"data":"151.101.65.195"},{"name":"metebalci.com.","class":"IN","type":"RRSIG","ttl":299,"data":"A 8 2 300 20181227144044 20181205144044 59764 metebalci.com. z6FupNLEU/8OcB3rNMkVqVaan05Xu89T8hV6+IC7LGjWPtrD+TlNJd8D cGeq8xJLR8b1Q+gBK0QSxpGvk89GaCTjNtMGHLBBdgpyV4syFv2BNzK7 iAJhA8QJ6i5xVFJdzMSsn3WvQvN1W71sirt8+56r1nQ47aVkBSLJoZKP lgw="}],"query_time":41,"server":"8.8.8.8#53(8.8.8.8)","when":"Fri Dec 07 14:09:43 CET 2018","rcvd":247,"when_epoch":1544220583,"when_epoch_utc":null}]