diff --git a/README.md b/README.md index aa97f2d7..b3672b63 100755 --- a/README.md +++ b/README.md @@ -3,6 +3,21 @@ JSON CLI output utility `jc` is used to JSONify the output of many standard linux cli tools for easier parsing in scripts. Parsers for `ls`, `ifconfig`, and `netstat` are currently included and more can be added via modules. +This allows further command line processing of output with tools like `jq` simply by piping commands: + +``` +$ ls -l /usr/bin | jc --ls | jq .[] | jq 'select(.bytes > 50000000)' +{ + "filename": "emacs", + "flags": "-r-xr-xr-x", + "links": 1, + "owner": "root", + "group": "wheel", + "bytes": 117164432, + "date": "May 3 22:26" +} +``` + ## Installation ``` $ pip3 install jc @@ -15,10 +30,13 @@ The first argument is required and identifies the command that is piping output - `--ls` enables the `ls` parser - `--ifconfig` enables the `ifconfig` parser - `--netstat` enables the `netstat` parser +- `--ps` enables the `ps` parser +- `--route` enables the `route` parser The second `-p` argument is optional and specifies whether to pretty format the JSON output. ## Examples +### ls ``` $ ls -l /bin | jc --ls -p [ @@ -52,6 +70,7 @@ $ ls -l /bin | jc --ls -p ... ] ``` +### ifconfig ``` $ ifconfig | jc --ifconfig -p [ @@ -135,6 +154,7 @@ $ ifconfig | jc --ifconfig -p } ] ``` +### netstat ``` $ netstat -p | jc --netstat -p { @@ -261,6 +281,103 @@ $ netstat -lp | jc --netstat -p } } ``` +### ps +``` +$ ps -ef | jc --ps -p +[ + { + "UID": "root", + "PID": "1", + "PPID": "0", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:05", + "CMD": "/lib/systemd/systemd --system --deserialize 35" + }, + { + "UID": "root", + "PID": "2", + "PPID": "0", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[kthreadd]" + }, + { + "UID": "root", + "PID": "4", + "PPID": "2", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[kworker/0:0H]" + }, + { + "UID": "root", + "PID": "6", + "PPID": "2", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[mm_percpu_wq]" + }, + ... +] +``` +### route +``` +$ route -n | jc --route -p +[ + { + "Destination": "0.0.0.0", + "Gateway": "192.168.71.2", + "Genmask": "0.0.0.0", + "Flags": "UG", + "Metric": "100", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + }, + { + "Destination": "172.17.0.0", + "Gateway": "0.0.0.0", + "Genmask": "255.255.0.0", + "Flags": "U", + "Metric": "0", + "Ref": "0", + "Use": "0", + "Iface": "docker0" + }, + { + "Destination": "192.168.71.0", + "Gateway": "0.0.0.0", + "Genmask": "255.255.255.0", + "Flags": "U", + "Metric": "0", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + }, + { + "Destination": "192.168.71.2", + "Gateway": "0.0.0.0", + "Genmask": "255.255.255.255", + "Flags": "UH", + "Metric": "100", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + } +] +``` +## Contributions +Feel free to add/improve code or parsers! - +## Acknowledgments +- `ifconfig-parser` module from https://github.com/KnightWhoSayNi/ifconfig-parser +- Parsing code from Conor Heine at https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 diff --git a/changelog.txt b/changelog.txt new file mode 100644 index 00000000..17addc74 --- /dev/null +++ b/changelog.txt @@ -0,0 +1,10 @@ +jc changelog + +20191018 v0.5.5 +- Fix netstat -p parsing for Ubuntu +- Add ps parser +- Add route parser +- ls parser fixes + +20191017 v0.2.0 +- ifconfig, ls, and netstat support diff --git a/jc/__init__.py b/jc/__init__.py index fb8c1668..d584b3c6 100644 --- a/jc/__init__.py +++ b/jc/__init__.py @@ -1,7 +1,5 @@ """JC - JSON CLI output utility -v0.1 - * kellyjonbrazil@gmail.com This module serializes standard unix command line output to structured JSON diff --git a/jc/jc.py b/jc/jc.py index 58551973..182fac45 100755 --- a/jc/jc.py +++ b/jc/jc.py @@ -9,15 +9,19 @@ import json import jc.parsers.ifconfig import jc.parsers.ls import jc.parsers.netstat +import jc.parsers.ps +import jc.parsers.route + def main(): pretty = False data = sys.stdin.read() if len(sys.argv) < 2: - print(f'\nError: jc\n Must specify parser. (e.g. --ls, --netstat, --ifconfig, etc.)') + print('Error: jc') + print(' Must specify parser. (e.g. --ls, --netstat, --ifconfig, etc.)') print(' Use -p to pretty print') - print(f'\nExample: ls -al | jc --ls -p\n') + print('Example: ls -al | jc --ls -p\n') exit() arg = sys.argv[1] @@ -32,6 +36,10 @@ def main(): result = jc.parsers.ls.parse(data) elif arg == '--netstat': result = jc.parsers.netstat.parse(data) + elif arg == '--ps': + result = jc.parsers.ps.parse(data) + elif arg == '--route': + result = jc.parsers.route.parse(data) # output resulting dictionary as json if pretty: @@ -39,5 +47,6 @@ def main(): else: print(json.dumps(result)) + if __name__ == '__main__': main() diff --git a/jc/parsers/ifconfig.py b/jc/parsers/ifconfig.py index 4bbac6fb..e716ba05 100644 --- a/jc/parsers/ifconfig.py +++ b/jc/parsers/ifconfig.py @@ -13,9 +13,10 @@ $ ifconfig | jc --ifconfig -p from collections import namedtuple from ifconfigparser import IfconfigParser + def parse(data): output = [] - + parsed = IfconfigParser(console_output=data) interfaces = parsed.get_interfaces() @@ -24,7 +25,5 @@ def parse(data): d = interfaces[iface]._asdict() dct = dict(d) output.append(dct) - + return output - - diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index 78b275bf..24a8d841 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -85,46 +85,48 @@ $ $ ls -l /usr/bin | jc --ls | jq .[] | jq 'select(.bytes > 50000000)' """ import re + def parse(data): output = [] - - cleandata = data.splitlines() + + linedata = data.splitlines() # Delete first line if it starts with 'total' - if cleandata[0].find('total') == 0: - cleandata.pop(0) + if linedata: + if linedata[0].find('total') == 0: + linedata.pop(0) - # Delete last line if it is blank - if cleandata[-1] == '': - cleandata.pop(-1) + # Clear any blank lines + cleandata = list(filter(None, linedata)) - # Check if -l was used to parse extra data - if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', cleandata[0]): - for entry in cleandata: - output_line = {} + if cleandata: + # Check if -l was used to parse extra data + if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', cleandata[0]): + for entry in cleandata: + output_line = {} - parsed_line = entry.split() + parsed_line = entry.split(maxsplit=8) - # split filenames and links - filename_field = ' '.join(parsed_line[8:]).split(' -> ') + # split filenames and links + filename_field = parsed_line[8].split(' -> ') - # create list of dictionaries - output_line['filename'] = filename_field[0] + # create list of dictionaries + output_line['filename'] = filename_field[0] - if len(filename_field) > 1: - output_line['link_to'] = filename_field[1] + if len(filename_field) > 1: + output_line['link_to'] = filename_field[1] - output_line['flags'] = parsed_line[0] - output_line['links'] = int(parsed_line[1]) - output_line['owner'] = parsed_line[2] - output_line['group'] = parsed_line[3] - output_line['bytes'] = int(parsed_line[4]) - output_line['date'] = ' '.join(parsed_line[5:8]) - output.append(output_line) - else: - for entry in cleandata: - output_line = {} - output_line['filename'] = entry - output.append(output_line) + output_line['flags'] = parsed_line[0] + output_line['links'] = int(parsed_line[1]) + output_line['owner'] = parsed_line[2] + output_line['group'] = parsed_line[3] + output_line['bytes'] = int(parsed_line[4]) + output_line['date'] = ' '.join(parsed_line[5:8]) + output.append(output_line) + else: + for entry in cleandata: + output_line = {} + output_line['filename'] = entry + output.append(output_line) return output diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index 9291fc84..4061add0 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -138,6 +138,7 @@ import string output = {} + class state(): section = '' session = '' @@ -147,12 +148,13 @@ class state(): client_tcp_ip6 = [] client_udp_ip4 = [] client_udp_ip6 = [] - + server_tcp_ip4 = [] server_tcp_ip6 = [] server_udp_ip4 = [] server_udp_ip6 = [] + def parse_line(entry): parsed_line = entry.split() output_line = {} @@ -164,40 +166,42 @@ def parse_line(entry): if len(parsed_line) > 5: - if parsed_line[5][0] not in string.digits: + if parsed_line[5][0] not in string.digits and parsed_line[5][0] != '-': output_line['state'] = parsed_line[5] - - if len(parsed_line) > 6: + + if len(parsed_line) > 6 and parsed_line[6][0] in string.digits: output_line['pid'] = int(parsed_line[6].split('/')[0]) output_line['program_name'] = parsed_line[6].split('/')[1] else: - output_line['pid'] = int(parsed_line[5].split('/')[0]) - output_line['program_name'] = parsed_line[5].split('/')[1] + if parsed_line[5][0] in string.digits: + output_line['pid'] = int(parsed_line[5].split('/')[0]) + output_line['program_name'] = parsed_line[5].split('/')[1] output_line['receive_q'] = int(parsed_line[1]) output_line['send_q'] = int(parsed_line[2]) return output_line + def parse(data): cleandata = data.splitlines() for line in cleandata: if line.find('Active Internet connections (w/o servers)') == 0: - state.section = "client" + state.section = 'client' continue if line.find('Active Internet connections (only servers)') == 0: - state.section = "server" + state.section = 'server' continue - + if line.find('Proto') == 0: continue if line.find('Active UNIX') == 0: break - + if state.section == 'client': if line.find('tcp') == 0: state.session = 'tcp' @@ -225,6 +229,7 @@ def parse(data): else: state.network = 'ipv4' + # client section if state.section == 'client' and state.session == 'tcp' and state.network == 'ipv4': state.client_tcp_ip4.append(parse_line(line)) @@ -237,7 +242,7 @@ def parse(data): if state.section == 'client' and state.session == 'udp' and state.network == 'ipv6': state.client_udp_ip6.append(parse_line(line)) - + # server section if state.section == 'server' and state.session == 'tcp' and state.network == 'ipv4': state.server_tcp_ip4.append(parse_line(line)) @@ -254,6 +259,7 @@ def parse(data): state.network = '' # build dictionary + # client section if state.client_tcp_ip4: if 'client' not in output: output['client'] = {} @@ -281,8 +287,8 @@ def parse(data): if 'udp' not in output['client']: output['client']['udp'] = {} output['client']['udp']['ipv6'] = state.client_udp_ip6 - - + + # server section if state.server_tcp_ip4: if 'server' not in output: output['server'] = {} @@ -303,7 +309,7 @@ def parse(data): if 'udp' not in output['server']: output['server']['udp'] = {} output['server']['udp']['ipv4'] = state.server_udp_ip4 - + if state.server_udp_ip6: if 'server' not in output: output['server'] = {} @@ -311,4 +317,4 @@ def parse(data): output['server']['udp'] = {} output['server']['udp']['ipv6'] = state.server_udp_ip6 - return output \ No newline at end of file + return output diff --git a/jc/parsers/ps.py b/jc/parsers/ps.py new file mode 100644 index 00000000..14fc45f0 --- /dev/null +++ b/jc/parsers/ps.py @@ -0,0 +1,67 @@ +"""jc - JSON CLI output utility ps Parser + +Usage: + specify --ps as the first argument if the piped input is coming from ps + + ps options supported: + - ef + - axu + +Example: + +$ ps -ef | jc --ps -p +[ + { + "UID": "root", + "PID": "1", + "PPID": "0", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:05", + "CMD": "/lib/systemd/systemd --system --deserialize 35" + }, + { + "UID": "root", + "PID": "2", + "PPID": "0", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[kthreadd]" + }, + { + "UID": "root", + "PID": "4", + "PPID": "2", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[kworker/0:0H]" + }, + { + "UID": "root", + "PID": "6", + "PPID": "2", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[mm_percpu_wq]" + }, + ... +] +""" + + +def parse(data): + + # code adapted from Conor Heine at: + # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 + + cleandata = data.splitlines() + headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) + return [dict(zip(headers, r)) for r in raw_data] diff --git a/jc/parsers/route.py b/jc/parsers/route.py new file mode 100644 index 00000000..46c8f2dd --- /dev/null +++ b/jc/parsers/route.py @@ -0,0 +1,63 @@ +"""jc - JSON CLI output utility route Parser + +Usage: + specify --route as the first argument if the piped input is coming from route + + +Example: + +$ route -n | jc --route -p +[ + { + "Destination": "0.0.0.0", + "Gateway": "192.168.71.2", + "Genmask": "0.0.0.0", + "Flags": "UG", + "Metric": "100", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + }, + { + "Destination": "172.17.0.0", + "Gateway": "0.0.0.0", + "Genmask": "255.255.0.0", + "Flags": "U", + "Metric": "0", + "Ref": "0", + "Use": "0", + "Iface": "docker0" + }, + { + "Destination": "192.168.71.0", + "Gateway": "0.0.0.0", + "Genmask": "255.255.255.0", + "Flags": "U", + "Metric": "0", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + }, + { + "Destination": "192.168.71.2", + "Gateway": "0.0.0.0", + "Genmask": "255.255.255.255", + "Flags": "UH", + "Metric": "100", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + } +] +""" + + +def parse(data): + + # code adapted from Conor Heine at: + # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 + + cleandata = data.splitlines()[1:] + headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) + return [dict(zip(headers, r)) for r in raw_data] diff --git a/setup.py b/setup.py index f7ff05c7..27ffc49e 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='0.2.0', + version='0.5.5', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='This tool serializes the output of popular command line tools to structured JSON output.',