From a19c12096a8e8de02a6ff761cc9c13ec249d416e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 19 Sep 2021 13:18:23 -0700 Subject: [PATCH] initial working parser for both linux and bsd --- jc/parsers/ping_s.py | 508 +++++++++++++++++++++++++++++++------------ 1 file changed, 369 insertions(+), 139 deletions(-) diff --git a/jc/parsers/ping_s.py b/jc/parsers/ping_s.py index 3d275343..545f6126 100644 --- a/jc/parsers/ping_s.py +++ b/jc/parsers/ping_s.py @@ -56,6 +56,7 @@ Examples: ... """ import string +import ipaddress import jc.utils from jc.utils import stream_success, stream_error @@ -102,6 +103,346 @@ def _process(proc_data): return proc_data +class state: + os_detected = None + linux = None + bsd = None + ipv4 = None + hostname = None + destination_ip = None + sent_bytes = None + pattern = None + footer = False + packets_transmitted = None + packets_received = None + packet_loss_percent = None + time_ms = None + duplicates = None + ping_error = None + + +def _ipv6_in(line): + line_list = line.replace('(', ' ').replace(')', ' ').replace(',', ' ').replace('%', ' ').split() + ipv6 = False + for item in line_list: + try: + _ = ipaddress.IPv6Address(item) + ipv6 = True + except Exception: + pass + return ipv6 + + +def _error_type(line): + # from https://github.com/dgibson/iputils/blob/master/ping.c + # https://android.googlesource.com/platform/external/ping/+/8fc3c91cf9e7f87bc20b9e6d3ea2982d87b70d9a/ping.c + # https://opensource.apple.com/source/network_cmds/network_cmds-328/ping.tproj/ping.c + type_map = { + 'Destination Net Unreachable': 'destination_net_unreachable', + 'Destination Host Unreachable': 'destination_host_unreachable', + 'Destination Protocol Unreachable': 'destination_protocol_unreachable', + 'Destination Port Unreachable': 'destination_port_unreachable', + 'Frag needed and DF set': 'frag_needed_and_df_set', + 'Source Route Failed': 'source_route_failed', + 'Destination Net Unknown': 'destination_net_unknown', + 'Destination Host Unknown': 'destination_host_unknown', + 'Source Host Isolated': 'source_host_isolated', + 'Destination Net Prohibited': 'destination_net_prohibited', + 'Destination Host Prohibited': 'destination_host_prohibited', + 'Destination Net Unreachable for Type of Service': 'destination_net_unreachable_for_type_of_service', + 'Destination Host Unreachable for Type of Service': 'destination_host_unreachable_for_type_of_service', + 'Packet filtered': 'packet_filtered', + 'Precedence Violation': 'precedence_violation', + 'Precedence Cutoff': 'precedence_cutoff', + 'Dest Unreachable, Bad Code': 'dest_unreachable_bad_code', + 'Redirect Network': 'redirect_network', + 'Redirect Host': 'redirect_host', + 'Redirect Type of Service and Network': 'redirect_type_of_service_and_network', + 'Redirect, Bad Code': 'redirect_bad_code', + 'Time to live exceeded': 'time_to_live_exceeded', + 'Frag reassembly time exceeded': 'frag_reassembly_time_exceeded', + 'Time exceeded, Bad Code': 'time_exceeded_bad_code' + } + + for err_type, code in type_map.items(): + if err_type in line: + return code + + +def _bsd_parse(line, s): + output_line = {} + + + if line.startswith('PING '): + s.destination_ip = line.split()[2].lstrip('(').rstrip(':').rstrip(')') + s.sent_bytes = line.split()[3] + return None + + if line.startswith('PING6('): + line = line.replace('(', ' ').replace(')', ' ').replace('=', ' ') + s.source_ip = line.split()[4] + s.destination_ip = line.split()[6] + s.sent_bytes = line.split()[1] + return None + + if line.startswith('---'): + s.footer = True + return None + + if s.footer: + if 'packets transmitted' in line: + if ' duplicates,' in line: + s.packets_transmitted = line.split()[0] + s.packets_received = line.split()[3] + s.packet_loss_percent = line.split()[8].rstrip('%') + s.duplicates = line.split()[6].lstrip('+') + return None + + else: + s.packets_transmitted = line.split()[0] + s.packets_received = line.split()[3] + s.packet_loss_percent = line.split()[6].rstrip('%') + s.duplicates = '0' + return None + + else: + split_line = line.split(' = ')[1] + split_line = split_line.split('/') + + output_line = { + 'type': 'summary', + 'destination_ip': s.destination_ip or None, + 'sent_bytes': s.sent_bytes or None, + 'pattern': s.pattern or None, + 'packets_transmitted': s.packets_transmitted or None, + 'packets_received': s.packets_received or None, + 'packet_loss_percent': s.packet_loss_percent or None, + 'duplicates': s.duplicates or None, + 'round_trip_ms_min': split_line[0], + 'round_trip_ms_avg': split_line[1], + 'round_trip_ms_max': split_line[2], + 'round_trip_ms_stddev': split_line[3].replace(' ms', '') + } + + return output_line + + # ping response lines + else: + # ipv4 lines + if not _ipv6_in(line): + + # request timeout + if line.startswith('Request timeout for '): + output_line = { + 'type': 'timeout', + 'destination_ip': s.destination_ip or None, + 'sent_bytes': s.sent_bytes or None, + 'pattern': s.pattern or None, + 'icmp_seq': line.split()[4] + } + + return output_line + + # catch error responses + err = _error_type(line) + if err: + output_line = { + 'type': err + } + + try: + output_line['bytes'] = line.split()[0] + output_line['response_ip'] = line.split()[4].strip(':').strip('(').strip(')') + except Exception: + pass + + return output_line + + # normal response + elif ' bytes from ' in line: + line = line.replace(':', ' ').replace('=', ' ') + + output_line = { + 'type': 'reply', + 'destination_ip': s.destination_ip or None, + 'sent_bytes': s.sent_bytes or None, + 'pattern': s.pattern or None, + 'response_bytes': line.split()[0], + 'response_ip': line.split()[3], + 'icmp_seq': line.split()[5], + 'ttl': line.split()[7], + 'time_ms': line.split()[9] + } + + return output_line + + # ipv6 lines + elif ' bytes from ' in line: + line = line.replace(',', ' ').replace('=', ' ') + + output_line = { + 'type': 'reply', + 'destination_ip': s.destination_ip or None, + 'sent_bytes': s.sent_bytes or None, + 'pattern': s.pattern or None, + 'bytes': line.split()[0], + 'response_ip': line.split()[3], + 'icmp_seq': line.split()[5], + 'ttl': line.split()[7], + 'time_ms': line.split()[9] + } + + return output_line + + +def _linux_parse(line, s): + """ + Linux ping line parsing function. + + Parameters: + + line: (string) line of text data to parse + s: (state object) global state + + Returns: + + Dictionary. Raw structured data. + """ + output_line = {} + + if line.startswith('PING '): + s.ipv4 = True if 'bytes of data' in line else False + + if s.ipv4 and line[5] not in string.digits: + s.hostname = True + elif s.ipv4 and line[5] in string.digits: + s.hostname = False + elif not s.ipv4 and ' (' in line: + s.hostname = True + else: + s.hostname = False + + if s.ipv4 and not s.hostname: + dst_ip, dta_byts = (2, 3) + elif s.ipv4 and s.hostname: + dst_ip, dta_byts = (2, 3) + elif not s.ipv4 and not s.hostname: + dst_ip, dta_byts = (2, 3) + else: + dst_ip, dta_byts = (3, 4) + + line = line.replace('(', ' ').replace(')', ' ') + s.destination_ip = line.split()[dst_ip].lstrip('(').rstrip(')') + s.sent_bytes = line.split()[dta_byts] + + return None + + if line.startswith('---'): + s.footer = True + return None + + if s.footer: + if 'packets transmitted' in line: + if ' duplicates,' in line: + s.packets_transmitted = line.split()[0] + s.packets_received = line.split()[3] + s.packet_loss_percent = line.split()[7].rstrip('%') + s.duplicates = line.split()[5].lstrip('+') + s.time_ms = line.split()[11].replace('ms', '') + + return None + + else: + s.packets_transmitted = line.split()[0] + s.packets_received = line.split()[3] + s.packet_loss_percent = line.split()[5].rstrip('%') + s.duplicates = '0' + s.time_ms = line.split()[9].replace('ms', '') + + return None + + else: + split_line = line.split(' = ')[1] + split_line = split_line.split('/') + output_line = { + 'type': 'summary', + 'destination_ip': s.destination_ip or None, + 'sent_bytes': s.sent_bytes or None, + 'pattern': s.pattern or None, + 'packets_transmitted': s.packets_transmitted or None, + 'packets_received': s.packets_received or None, + 'packet_loss_percent': s.packet_loss_percent or None, + 'duplicates': s.duplicates or None, + 'time_ms': s.time_ms or None, + 'round_trip_ms_min': split_line[0], + 'round_trip_ms_avg': split_line[1], + 'round_trip_ms_max': split_line[2], + 'round_trip_ms_stddev': split_line[3].split()[0] + } + + return output_line + + # ping response lines + else: + # request timeout + if 'no answer yet for icmp_seq=' in line: + timestamp = False + isequence = 5 + + # if timestamp option is specified, then shift icmp sequence field right by one + if line[0] == '[': + timestamp = True + isequence = 6 + + output_line = { + 'type': 'timeout', + 'destination_ip': s.destination_ip or None, + 'sent_bytes': s.sent_bytes or None, + 'pattern': s.pattern or None, + 'timestamp': line.split()[0].lstrip('[').rstrip(']') if timestamp else None, + 'icmp_seq': line.replace('=', ' ').split()[isequence] + } + + return output_line + + # normal responses + elif ' bytes from ' in line: + + line = line.replace('(', ' ').replace(')', ' ').replace('=', ' ') + + # positions of items depend on whether ipv4/ipv6 and/or ip/hostname is used + if s.ipv4 and not s.hostname: + bts, rip, iseq, t2l, tms = (0, 3, 5, 7, 9) + elif s.ipv4 and s.hostname: + bts, rip, iseq, t2l, tms = (0, 4, 7, 9, 11) + elif not s.ipv4 and not s.hostname: + bts, rip, iseq, t2l, tms = (0, 3, 5, 7, 9) + elif not s.ipv4 and s.hostname: + bts, rip, iseq, t2l, tms = (0, 4, 7, 9, 11) + + # if timestamp option is specified, then shift everything right by one + timestamp = False + if line[0] == '[': + timestamp = True + bts, rip, iseq, t2l, tms = (bts + 1, rip + 1, iseq + 1, t2l + 1, tms + 1) + + output_line = { + 'type': 'reply', + 'destination_ip': s.destination_ip or None, + 'sent_bytes': s.sent_bytes or None, + 'pattern': s.pattern or None, + 'timestamp': line.split()[0].lstrip('[').rstrip(']') if timestamp else None, + 'response_bytes': line.split()[bts], + 'response_ip': line.split()[rip].rstrip(':'), + 'icmp_seq': line.split()[iseq], + 'ttl': line.split()[t2l], + 'time_ms': line.split()[tms], + 'duplicate': True if 'DUP!' in line else False + } + + return output_line + + def parse(data, raw=False, quiet=False): """ Main text parsing generator function. Produces an iterable object. @@ -116,161 +457,50 @@ def parse(data, raw=False, quiet=False): Dictionary. Raw or processed structured data. """ + s = state() + if not quiet: jc.utils.compatibility(__name__, info.compatible) - destination_ip = None - sent_bytes = None - pattern = None - footer = False - packets_transmitted = None - packets_received = None - packet_loss_percent = None - time_ms = None - duplicates = None - for line in data: - try: - output_line = {} + output_line = {} + + try: # check for PATTERN if line.startswith('PATTERN: '): - pattern = line.strip().split(': ')[1] + s.pattern = line.strip().split(': ')[1] continue - if line.startswith('PING '): - ipv4 = True if 'bytes of data' in line else False + # detect Linux vs. BSD ping + if not s.os_detected and line.strip().endswith('bytes of data.'): + s.os_detected = True + s.linux = True - if ipv4 and line[5] not in string.digits: - hostname = True - elif ipv4 and line[5] in string.digits: - hostname = False - elif not ipv4 and ' (' in line: - hostname = True - else: - hostname = False + if not s.os_detected and '-->' in line: + s.os_detected = True + s.bsd = True - if ipv4 and not hostname: - dst_ip, dta_byts = (2, 3) - elif ipv4 and hostname: - dst_ip, dta_byts = (2, 3) - elif not ipv4 and not hostname: - dst_ip, dta_byts = (2, 3) - else: - dst_ip, dta_byts = (3, 4) + if not s.os_detected and _ipv6_in(line) and line.strip().endswith('data bytes'): + s.os_detected = True + s.linux = True - line = line.replace('(', ' ').replace(')', ' ') - destination_ip = line.split()[dst_ip].lstrip('(').rstrip(')') - sent_bytes = line.split()[dta_byts] + if not s.os_detected: + s.os_detected = True + s.bsd = True - continue + # parse the data + if s.os_detected and s.linux: + output_line = _linux_parse(line, s) - if line.startswith('---'): - footer = True - continue + if s.os_detected and s.bsd: + output_line = _bsd_parse(line, s) - if footer: - if 'packets transmitted' in line: - if ' duplicates,' in line: - packets_transmitted = line.split()[0] - packets_received = line.split()[3] - packet_loss_percent = line.split()[7].rstrip('%') - duplicates = line.split()[5].lstrip('+') - time_ms = line.split()[11].replace('ms', '') - - continue - - else: - packets_transmitted = line.split()[0] - packets_received = line.split()[3] - packet_loss_percent = line.split()[5].rstrip('%') - duplicates = '0' - time_ms = line.split()[9].replace('ms', '') - - continue - - else: - split_line = line.split(' = ')[1] - split_line = split_line.split('/') - output_line = { - 'type': 'summary', - 'destination_ip': destination_ip or None, - 'sent_bytes': sent_bytes or None, - 'pattern': pattern or None, - 'packets_transmitted': packets_transmitted or None, - 'packets_received': packets_received or None, - 'packet_loss_percent': packet_loss_percent or None, - 'duplicates': duplicates or None, - 'time_ms': time_ms or None, - 'round_trip_ms_min': split_line[0], - 'round_trip_ms_avg': split_line[1], - 'round_trip_ms_max': split_line[2], - 'round_trip_ms_stddev': split_line[3].split()[0] - } - - yield stream_success(output_line, quiet) if raw else stream_success(_process(output_line), quiet) - continue - - # ping response lines + # yield the output line if it has data + if output_line: + yield stream_success(output_line, quiet) if raw else stream_success(_process(output_line), quiet) else: - # request timeout - if 'no answer yet for icmp_seq=' in line: - timestamp = False - isequence = 5 + continue - # if timestamp option is specified, then shift icmp sequence field right by one - if line[0] == '[': - timestamp = True - isequence = 6 - - output_line = { - 'type': 'timeout', - 'destination_ip': destination_ip or None, - 'sent_bytes': sent_bytes or None, - 'pattern': pattern or None, - 'timestamp': line.split()[0].lstrip('[').rstrip(']') if timestamp else None, - 'icmp_seq': line.replace('=', ' ').split()[isequence] - } - - yield stream_success(output_line, quiet) if raw else stream_success(_process(output_line), quiet) - continue - - # normal responses - elif ' bytes from ' in line: - - line = line.replace('(', ' ').replace(')', ' ').replace('=', ' ') - - # positions of items depend on whether ipv4/ipv6 and/or ip/hostname is used - if ipv4 and not hostname: - bts, rip, iseq, t2l, tms = (0, 3, 5, 7, 9) - elif ipv4 and hostname: - bts, rip, iseq, t2l, tms = (0, 4, 7, 9, 11) - elif not ipv4 and not hostname: - bts, rip, iseq, t2l, tms = (0, 3, 5, 7, 9) - elif not ipv4 and hostname: - bts, rip, iseq, t2l, tms = (0, 4, 7, 9, 11) - - # if timestamp option is specified, then shift everything right by one - timestamp = False - if line[0] == '[': - timestamp = True - bts, rip, iseq, t2l, tms = (bts + 1, rip + 1, iseq + 1, t2l + 1, tms + 1) - - output_line = { - 'type': 'reply', - 'destination_ip': destination_ip or None, - 'sent_bytes': sent_bytes or None, - 'pattern': pattern or None, - 'timestamp': line.split()[0].lstrip('[').rstrip(']') if timestamp else None, - 'response_bytes': line.split()[bts], - 'response_ip': line.split()[rip].rstrip(':'), - 'icmp_seq': line.split()[iseq], - 'ttl': line.split()[t2l], - 'time_ms': line.split()[tms], - 'duplicate': True if 'DUP!' in line else False - } - - yield stream_success(output_line, quiet) if raw else stream_success(_process(output_line), quiet) - except Exception as e: yield stream_error(e, quiet, line)