diff --git a/docs/parsers/openvpn.md b/docs/parsers/openvpn.md new file mode 100644 index 00000000..a031bb52 --- /dev/null +++ b/docs/parsers/openvpn.md @@ -0,0 +1,178 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.openvpn + +jc - JSON Convert openvpn-status.log file parser + +The `*_epoch` calculated timestamp fields are naive. (i.e. based on +the local time of the system the parser is run on) + +Usage (cli): + + $ cat openvpn-status.log | jc --openvpn + +Usage (module): + + import jc + result = jc.parse('openvpn', openvpn_status_log_file_output) + +Schema: + + { + "clients": [ + { + "common_name": string, + "real_address": string, + "real_address_prefix": integer, # [0] + "real_address_port": integer, # [0] + "bytes_received": integer, + "bytes_sent": integer, + "connected_since": string, + "connected_since_epoch": integer, + "updated": string, + "updated_epoch": integer, + } + ], + "routing_table": [ + { + "virtual_address": string, + "virtual_address_prefix": integer, # [0] + "virtual_address_port": integer, # [0] + "common_name": string, + "real_address": string, + "real_address_prefix": integer, # [0] + "real_address_port": integer, # [0] + "last_reference": string, + "last_reference_epoch": integer, + } + ], + "global_stats": { + "max_bcast_mcast_queue_len": integer + } + } + + [0] null/None if not found + +Examples: + + $ cat openvpn-status.log | jc --openvpn -p + { + "clients": [ + { + "common_name": "foo@example.com", + "real_address": "10.10.10.10", + "bytes_received": 334948, + "bytes_sent": 1973012, + "connected_since": "Thu Jun 18 04:23:03 2015", + "updated": "Thu Jun 18 08:12:15 2015", + "real_address_prefix": null, + "real_address_port": 49502, + "connected_since_epoch": 1434626583, + "updated_epoch": 1434640335 + }, + { + "common_name": "foo@example.com", + "real_address": "10.10.10.10", + "bytes_received": 334948, + "bytes_sent": 1973012, + "connected_since": "Thu Jun 18 04:23:03 2015", + "updated": "Thu Jun 18 08:12:15 2015", + "real_address_prefix": null, + "real_address_port": 49503, + "connected_since_epoch": 1434626583, + "updated_epoch": 1434640335 + } + ], + "routing_table": [ + { + "virtual_address": "192.168.255.118", + "common_name": "baz@example.com", + "real_address": "10.10.10.10", + "last_reference": "Thu Jun 18 08:12:09 2015", + "virtual_address_prefix": null, + "virtual_address_port": null, + "real_address_prefix": null, + "real_address_port": 63414, + "last_reference_epoch": 1434640329 + }, + { + "virtual_address": "10.200.0.0", + "common_name": "baz@example.com", + "real_address": "10.10.10.10", + "last_reference": "Thu Jun 18 08:12:09 2015", + "virtual_address_prefix": 16, + "virtual_address_port": null, + "real_address_prefix": null, + "real_address_port": 63414, + "last_reference_epoch": 1434640329 + } + ], + "global_stats": { + "max_bcast_mcast_queue_len": 0 + } + } + + $ cat openvpn-status.log | jc --openvpn -p -r + { + "clients": [ + { + "common_name": "foo@example.com", + "real_address": "10.10.10.10:49502", + "bytes_received": "334948", + "bytes_sent": "1973012", + "connected_since": "Thu Jun 18 04:23:03 2015", + "updated": "Thu Jun 18 08:12:15 2015" + }, + { + "common_name": "foo@example.com", + "real_address": "10.10.10.10:49503", + "bytes_received": "334948", + "bytes_sent": "1973012", + "connected_since": "Thu Jun 18 04:23:03 2015", + "updated": "Thu Jun 18 08:12:15 2015" + } + ], + "routing_table": [ + { + "virtual_address": "192.168.255.118", + "common_name": "baz@example.com", + "real_address": "10.10.10.10:63414", + "last_reference": "Thu Jun 18 08:12:09 2015" + }, + { + "virtual_address": "10.200.0.0/16", + "common_name": "baz@example.com", + "real_address": "10.10.10.10:63414", + "last_reference": "Thu Jun 18 08:12:09 2015" + } + ], + "global_stats": { + "max_bcast_mcast_queue_len": "0" + } + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> JSONDictType +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/openvpn.py b/jc/parsers/openvpn.py index 56257ae9..e3c5ab32 100644 --- a/jc/parsers/openvpn.py +++ b/jc/parsers/openvpn.py @@ -3,9 +3,6 @@ The `*_epoch` calculated timestamp fields are naive. (i.e. based on the local time of the system the parser is run on) -The `*_epoch_utc` calculated timestamp fields are timezone-aware and -is only available if the timestamp has a UTC timezone. - Usage (cli): $ cat openvpn-status.log | jc --openvpn @@ -22,24 +19,27 @@ Schema: { "common_name": string, "real_address": string, + "real_address_prefix": integer, # [0] + "real_address_port": integer, # [0] "bytes_received": integer, "bytes_sent": integer, "connected_since": string, - "updated": string, "connected_since_epoch": integer, - "connected_since_epoch_utc": integer, + "updated": string, "updated_epoch": integer, - "updated_epoch_utc": integer } ], "routing_table": [ { "virtual_address": string, + "virtual_address_prefix": integer, # [0] + "virtual_address_port": integer, # [0] "common_name": string, "real_address": string, + "real_address_prefix": integer, # [0] + "real_address_port": integer, # [0] "last_reference": string, "last_reference_epoch": integer, - "last_reference_epoch_utc": integer } ], "global_stats": { @@ -47,6 +47,8 @@ Schema: } } + [0] null/None if not found + Examples: $ cat openvpn-status.log | jc --openvpn -p @@ -54,45 +56,51 @@ Examples: "clients": [ { "common_name": "foo@example.com", - "real_address": "10.10.10.10:49502", + "real_address": "10.10.10.10", "bytes_received": 334948, "bytes_sent": 1973012, "connected_since": "Thu Jun 18 04:23:03 2015", "updated": "Thu Jun 18 08:12:15 2015", + "real_address_prefix": null, + "real_address_port": 49502, "connected_since_epoch": 1434626583, - "connected_since_epoch_utc": null, - "updated_epoch": 1434640335, - "updated_epoch_utc": null + "updated_epoch": 1434640335 }, { "common_name": "foo@example.com", - "real_address": "10.10.10.10:49503", + "real_address": "10.10.10.10", "bytes_received": 334948, "bytes_sent": 1973012, "connected_since": "Thu Jun 18 04:23:03 2015", "updated": "Thu Jun 18 08:12:15 2015", + "real_address_prefix": null, + "real_address_port": 49503, "connected_since_epoch": 1434626583, - "connected_since_epoch_utc": null, - "updated_epoch": 1434640335, - "updated_epoch_utc": null + "updated_epoch": 1434640335 } ], "routing_table": [ { "virtual_address": "192.168.255.118", "common_name": "baz@example.com", - "real_address": "10.10.10.10:63414", + "real_address": "10.10.10.10", "last_reference": "Thu Jun 18 08:12:09 2015", - "last_reference_epoch": 1434640329, - "last_reference_epoch_utc": null + "virtual_address_prefix": null, + "virtual_address_port": null, + "real_address_prefix": null, + "real_address_port": 63414, + "last_reference_epoch": 1434640329 }, { - "virtual_address": "10.200.0.0/16", + "virtual_address": "10.200.0.0", "common_name": "baz@example.com", - "real_address": "10.10.10.10:63414", + "real_address": "10.10.10.10", "last_reference": "Thu Jun 18 08:12:09 2015", - "last_reference_epoch": 1434640329, - "last_reference_epoch_utc": null + "virtual_address_prefix": 16, + "virtual_address_port": null, + "real_address_prefix": null, + "real_address_port": 63414, + "last_reference_epoch": 1434640329 } ], "global_stats": { @@ -139,7 +147,9 @@ Examples: } } """ -from typing import List, Dict +import re +import ipaddress +from typing import List, Dict, Tuple from jc.jc_types import JSONDictType import jc.utils @@ -156,6 +166,34 @@ class info(): __version__ = info.version +def _split_addr(addr_str: str) -> Tuple: + """Check the type of address (v4, v6, mac) and split out the address, + prefix, and port. Values are None if they don't exist.""" + address = possible_addr = prefix = port = possible_port = None + + try: + address, prefix = addr_str.rsplit('/', maxsplit=1) + except Exception: + address = addr_str + + # is this a mac address? then stop + if re.match(r'(?:\S\S\:){5}\S\S', address): + return address, prefix, port + + # is it an ipv4 with port or just ipv6? + if ':' in address: + try: + possible_addr, possible_port = address.rsplit(':', maxsplit=1) + _ = ipaddress.IPv4Address(possible_addr) + address = possible_addr + port = possible_port + # assume it was an IPv6 address + except Exception: + pass + + return address, prefix, port + + def _process(proc_data: JSONDictType) -> JSONDictType: """ Final processing to conform to the schema. @@ -170,6 +208,7 @@ def _process(proc_data: JSONDictType) -> JSONDictType: """ int_list = {'bytes_received', 'bytes_sent', 'max_bcast_mcast_queue_len'} date_fields = {'connected_since', 'updated', 'last_reference'} + addr_fields = {'real_address', 'virtual_address'} if 'clients' in proc_data: for item in proc_data['clients']: @@ -180,7 +219,12 @@ def _process(proc_data: JSONDictType) -> JSONDictType: if k in date_fields: dt = jc.utils.timestamp(item[k], format_hint=(1000,)) item[k + '_epoch'] = dt.naive - item[k + '_epoch_utc'] = dt.utc + + if k in addr_fields: + addr, prefix, port = _split_addr(v) + item[k] = addr + item[k + '_prefix'] = jc.utils.convert_to_int(prefix) + item[k + '_port'] = jc.utils.convert_to_int(port) if 'routing_table' in proc_data: for item in proc_data['routing_table']: @@ -188,7 +232,12 @@ def _process(proc_data: JSONDictType) -> JSONDictType: if k in date_fields: dt = jc.utils.timestamp(item[k], format_hint=(1000,)) item[k + '_epoch'] = dt.naive - item[k + '_epoch_utc'] = dt.utc + + if k in addr_fields: + addr, prefix, port = _split_addr(v) + item[k] = addr + item[k + '_prefix'] = jc.utils.convert_to_int(prefix) + item[k + '_port'] = jc.utils.convert_to_int(port) if 'global_stats' in proc_data: for k, v in proc_data['global_stats'].items(): diff --git a/man/jc.1 b/man/jc.1 index 62958be8..fe77cd44 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-12-13 1.22.3 "JSON Convert" +.TH jc 1 2022-12-14 1.22.3 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS diff --git a/tests/fixtures/generic/openvpn-status.json b/tests/fixtures/generic/openvpn-status.json index 361566eb..5f3dd5fd 100644 --- a/tests/fixtures/generic/openvpn-status.json +++ b/tests/fixtures/generic/openvpn-status.json @@ -1 +1 @@ -{"clients":[{"common_name":"foo@example.com","real_address":"10.10.10.10:49502","bytes_received":334948,"bytes_sent":1973012,"connected_since":"Thu Jun 18 04:23:03 2015","updated":"Thu Jun 18 08:12:15 2015","connected_since_epoch":1434626583,"connected_since_epoch_utc":null,"updated_epoch":1434640335,"updated_epoch_utc":null},{"common_name":"foo@example.com","real_address":"10.10.10.10:49503","bytes_received":334948,"bytes_sent":1973012,"connected_since":"Thu Jun 18 04:23:03 2015","updated":"Thu Jun 18 08:12:15 2015","connected_since_epoch":1434626583,"connected_since_epoch_utc":null,"updated_epoch":1434640335,"updated_epoch_utc":null},{"common_name":"bar@example.com","real_address":"10.10.10.10:64169","bytes_received":1817262,"bytes_sent":28981224,"connected_since":"Thu Jun 18 04:08:39 2015","updated":"Thu Jun 18 08:12:15 2015","connected_since_epoch":1434625719,"connected_since_epoch_utc":null,"updated_epoch":1434640335,"updated_epoch_utc":null},{"common_name":"baz@example.com","real_address":"10.10.10.10:63414","bytes_received":111183,"bytes_sent":1202203,"connected_since":"Thu Jun 18 07:57:25 2015","updated":"Thu Jun 18 08:12:15 2015","connected_since_epoch":1434639445,"connected_since_epoch_utc":null,"updated_epoch":1434640335,"updated_epoch_utc":null},{"common_name":"tap@example.com","real_address":"10.0.0.100:55712","bytes_received":0,"bytes_sent":0,"connected_since":"Thu Oct 19 20:14:19 2017","updated":"Thu Jun 18 08:12:15 2015","connected_since_epoch":1508469259,"connected_since_epoch_utc":null,"updated_epoch":1434640335,"updated_epoch_utc":null},{"common_name":"baz@example.com","real_address":"10.10.10.10","bytes_received":111183,"bytes_sent":1202203,"connected_since":"Thu Jun 18 07:57:25 2015","updated":"Thu Jun 18 08:12:15 2015","connected_since_epoch":1434639445,"connected_since_epoch_utc":null,"updated_epoch":1434640335,"updated_epoch_utc":null}],"routing_table":[{"virtual_address":"192.168.255.118","common_name":"baz@example.com","real_address":"10.10.10.10:63414","last_reference":"Thu Jun 18 08:12:09 2015","last_reference_epoch":1434640329,"last_reference_epoch_utc":null},{"virtual_address":"10.200.0.0/16","common_name":"baz@example.com","real_address":"10.10.10.10:63414","last_reference":"Thu Jun 18 08:12:09 2015","last_reference_epoch":1434640329,"last_reference_epoch_utc":null},{"virtual_address":"2001:db8::1000/124","common_name":"baz@example.com","real_address":"10.10.10.10:63414","last_reference":"Thu Jun 18 08:12:09 2015","last_reference_epoch":1434640329,"last_reference_epoch_utc":null},{"virtual_address":"192.168.255.134","common_name":"foo@example.com","real_address":"10.10.10.10:49502","last_reference":"Thu Jun 18 08:12:09 2015","last_reference_epoch":1434640329,"last_reference_epoch_utc":null},{"virtual_address":"192.168.255.135","common_name":"foo@example.com","real_address":"10.10.10.10:49503","last_reference":"Thu Jun 18 08:12:09 2015","last_reference_epoch":1434640329,"last_reference_epoch_utc":null},{"virtual_address":"192.168.255.126","common_name":"bar@example.com","real_address":"10.10.10.10:64169","last_reference":"Thu Jun 18 08:11:55 2015","last_reference_epoch":1434640315,"last_reference_epoch_utc":null},{"virtual_address":"22:1d:63:bf:62:38","common_name":"tap@example.com","real_address":"10.0.0.100:55712","last_reference":"Thu Oct 19 20:14:19 2017","last_reference_epoch":1508469259,"last_reference_epoch_utc":null},{"virtual_address":"192.168.255.126","common_name":"bar@example.com","real_address":"10.10.10.10:64169","last_reference":"Thu Jun 18 08:11:55 2015","last_reference_epoch":1434640315,"last_reference_epoch_utc":null},{"virtual_address":"192.168.255.16","common_name":"bar@example.com","real_address":"10.10.10.10:64169","last_reference":"Thu Jun 18 08:11:55 2015","last_reference_epoch":1434640315,"last_reference_epoch_utc":null},{"virtual_address":"192.168.255.1","common_name":"bar@example.com","real_address":"10.10.10.10:64169","last_reference":"Thu Jun 18 08:11:55 2015","last_reference_epoch":1434640315,"last_reference_epoch_utc":null},{"virtual_address":"192.168.255.1","common_name":"baz@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:11:55 2015","last_reference_epoch":1434640315,"last_reference_epoch_utc":null}],"global_stats":{"max_bcast_mcast_queue_len":0}} +{"clients":[{"common_name":"foo@example.com","real_address":"10.10.10.10","bytes_received":334948,"bytes_sent":1973012,"connected_since":"Thu Jun 18 04:23:03 2015","updated":"Thu Jun 18 08:12:15 2015","real_address_prefix":null,"real_address_port":49502,"connected_since_epoch":1434626583,"updated_epoch":1434640335},{"common_name":"foo@example.com","real_address":"10.10.10.10","bytes_received":334948,"bytes_sent":1973012,"connected_since":"Thu Jun 18 04:23:03 2015","updated":"Thu Jun 18 08:12:15 2015","real_address_prefix":null,"real_address_port":49503,"connected_since_epoch":1434626583,"updated_epoch":1434640335},{"common_name":"bar@example.com","real_address":"10.10.10.10","bytes_received":1817262,"bytes_sent":28981224,"connected_since":"Thu Jun 18 04:08:39 2015","updated":"Thu Jun 18 08:12:15 2015","real_address_prefix":null,"real_address_port":64169,"connected_since_epoch":1434625719,"updated_epoch":1434640335},{"common_name":"baz@example.com","real_address":"10.10.10.10","bytes_received":111183,"bytes_sent":1202203,"connected_since":"Thu Jun 18 07:57:25 2015","updated":"Thu Jun 18 08:12:15 2015","real_address_prefix":null,"real_address_port":63414,"connected_since_epoch":1434639445,"updated_epoch":1434640335},{"common_name":"tap@example.com","real_address":"10.0.0.100","bytes_received":0,"bytes_sent":0,"connected_since":"Thu Oct 19 20:14:19 2017","updated":"Thu Jun 18 08:12:15 2015","real_address_prefix":null,"real_address_port":55712,"connected_since_epoch":1508469259,"updated_epoch":1434640335},{"common_name":"baz@example.com","real_address":"10.10.10.10","bytes_received":111183,"bytes_sent":1202203,"connected_since":"Thu Jun 18 07:57:25 2015","updated":"Thu Jun 18 08:12:15 2015","real_address_prefix":null,"real_address_port":null,"connected_since_epoch":1434639445,"updated_epoch":1434640335}],"routing_table":[{"virtual_address":"192.168.255.118","common_name":"baz@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:12:09 2015","virtual_address_prefix":null,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":63414,"last_reference_epoch":1434640329},{"virtual_address":"10.200.0.0","common_name":"baz@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:12:09 2015","virtual_address_prefix":16,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":63414,"last_reference_epoch":1434640329},{"virtual_address":"2001:db8::1000","common_name":"baz@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:12:09 2015","virtual_address_prefix":124,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":63414,"last_reference_epoch":1434640329},{"virtual_address":"192.168.255.134","common_name":"foo@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:12:09 2015","virtual_address_prefix":null,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":49502,"last_reference_epoch":1434640329},{"virtual_address":"192.168.255.135","common_name":"foo@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:12:09 2015","virtual_address_prefix":null,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":49503,"last_reference_epoch":1434640329},{"virtual_address":"192.168.255.126","common_name":"bar@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:11:55 2015","virtual_address_prefix":null,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":64169,"last_reference_epoch":1434640315},{"virtual_address":"22:1d:63:bf:62:38","common_name":"tap@example.com","real_address":"10.0.0.100","last_reference":"Thu Oct 19 20:14:19 2017","virtual_address_prefix":null,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":55712,"last_reference_epoch":1508469259},{"virtual_address":"192.168.255.126","common_name":"bar@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:11:55 2015","virtual_address_prefix":null,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":64169,"last_reference_epoch":1434640315},{"virtual_address":"192.168.255.16","common_name":"bar@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:11:55 2015","virtual_address_prefix":null,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":64169,"last_reference_epoch":1434640315},{"virtual_address":"192.168.255.1","common_name":"bar@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:11:55 2015","virtual_address_prefix":null,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":64169,"last_reference_epoch":1434640315},{"virtual_address":"192.168.255.1","common_name":"baz@example.com","real_address":"10.10.10.10","last_reference":"Thu Jun 18 08:11:55 2015","virtual_address_prefix":null,"virtual_address_port":null,"real_address_prefix":null,"real_address_port":null,"last_reference_epoch":1434640315}],"global_stats":{"max_bcast_mcast_queue_len":0}}