diff --git a/README.md b/README.md index 754c40e8..88f3d542 100755 --- a/README.md +++ b/README.md @@ -18,58 +18,38 @@ $ ls -l /usr/bin | jc --ls | jq .[] | jq 'select(.bytes > 50000000)' } ``` +The `jc` parsers can also be used as python modules by referencing them via: +``` +import jc.parsers.[parser] + +data = 'data to parse' +jc.parsers.[parser].parse(data) +``` +In this case the output will be a python dictionary instead of JSON. + ## Installation ``` $ pip3 install jc ``` ## Usage +``` +jc [parser] [options] +``` + `jc` accepts piped input from `STDIN` and outputs a JSON representation of the previous command's output to `STDOUT`. The JSON output can be compact or pretty formatted. -The first argument is required and identifies the command that is piping output to `jc` input. For example: -- `--ls` enables the `ls` parser +Parsers: - `--ifconfig` enables the `ifconfig` parser +- `--ls` enables the `ls` parser - `--netstat` enables the `netstat` parser - `--ps` enables the `ps` parser - `--route` enables the `route` parser -The second `-p` argument is optional and specifies whether to pretty format the JSON output. +Options: +- `-p` specifies whether to pretty format the JSON output ## Examples -### ls -``` -$ ls -l /bin | jc --ls -p -[ - { - "filename": "bash", - "flags": "-r-xr-xr-x", - "links": 1, - "owner": "root", - "group": "wheel", - "bytes": 618416, - "date": "May 3 22:26" - }, - { - "filename": "cat", - "flags": "-rwxr-xr-x", - "links": 1, - "owner": "root", - "group": "wheel", - "bytes": 23648, - "date": "May 3 22:26" - }, - { - "filename": "chmod", - "flags": "-rwxr-xr-x", - "links": 1, - "owner": "root", - "group": "wheel", - "bytes": 30016, - "date": "May 3 22:26" - }, - ... -] -``` ### ifconfig ``` $ ifconfig | jc --ifconfig -p @@ -154,132 +134,152 @@ $ ifconfig | jc --ifconfig -p } ] ``` +### ls +``` +$ ls -l /bin | jc --ls -p +[ + { + "filename": "bash", + "flags": "-r-xr-xr-x", + "links": 1, + "owner": "root", + "group": "wheel", + "bytes": 618416, + "date": "May 3 22:26" + }, + { + "filename": "cat", + "flags": "-rwxr-xr-x", + "links": 1, + "owner": "root", + "group": "wheel", + "bytes": 23648, + "date": "May 3 22:26" + }, + { + "filename": "chmod", + "flags": "-rwxr-xr-x", + "links": 1, + "owner": "root", + "group": "wheel", + "bytes": 30016, + "date": "May 3 22:26" + }, + ... +] +``` ### netstat ``` $ netstat -p | jc --netstat -p -{ - "client": { - "tcp": { - "ipv4": [ - { - "local_address": "localhost.localdo", - "local_port": "34480", - "foreign_address": "lb-192-30-255-113", - "foreign_port": "https", - "state": "ESTABLISHED", - "pid": 53550, - "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 - }, - { - "local_address": "localhost.localdo", - "local_port": "34478", - "foreign_address": "lb-192-30-255-113", - "foreign_port": "https", - "state": "ESTABLISHED", - "pid": 53550, - "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 - } - ] - } +[ + { + "session_protocol": "tcp", + "network_protocol": "ipv4", + "local_address": "localhost.localdo", + "local_port": "34480", + "foreign_address": "lb-192-30-255-113", + "foreign_port": "https", + "state": "ESTABLISHED", + "pid": 53550, + "program_name": "git-remote-ht", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "tcp", + "network_protocol": "ipv4", + "local_address": "localhost.localdo", + "local_port": "34478", + "foreign_address": "lb-192-30-255-113", + "foreign_port": "https", + "state": "ESTABLISHED", + "pid": 53550, + "program_name": "git-remote-ht", + "receive_q": 0, + "send_q": 0 } -} +] ``` ``` -$ netstat -lp | jc --netstat -p -{ - "server": { - "tcp": { - "ipv4": [ - { - "local_address": "localhost", - "local_port": "smtp", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "state": "LISTEN", - "pid": 1594, - "program_name": "master", - "receive_q": 0, - "send_q": 0 - }, - { - "local_address": "0.0.0.0", - "local_port": "ssh", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "state": "LISTEN", - "pid": 21918, - "program_name": "sshd", - "receive_q": 0, - "send_q": 0 - } - ], - "ipv6": [ - { - "local_address": "localhost", - "local_port": "smtp", - "foreign_address": "[::]", - "foreign_port": "*", - "state": "LISTEN", - "pid": 1594, - "program_name": "master", - "receive_q": 0, - "send_q": 0 - }, - { - "local_address": "[::]", - "local_port": "ssh", - "foreign_address": "[::]", - "foreign_port": "*", - "state": "LISTEN", - "pid": 21918, - "program_name": "sshd", - "receive_q": 0, - "send_q": 0 - } - ] - }, - "udp": { - "ipv4": [ - { - "local_address": "0.0.0.0", - "local_port": "bootpc", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "pid": 13903, - "program_name": "dhclient", - "receive_q": 0, - "send_q": 0 - }, - { - "local_address": "localhost", - "local_port": "323", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "pid": 30926, - "program_name": "chronyd", - "receive_q": 0, - "send_q": 0 - } - ], - "ipv6": [ - { - "local_address": "localhost", - "local_port": "323", - "foreign_address": "[::]", - "foreign_port": "*", - "pid": 30926, - "program_name": "chronyd", - "receive_q": 0, - "send_q": 0 - } - ] - } +$ netstat -lpn | jc --netstat -p +[ + { + "session_protocol": "tcp", + "network_protocol": "ipv4", + "local_address": "127.0.0.1", + "local_port": "42351", + "foreign_address": "0.0.0.0", + "foreign_port": "*", + "state": "LISTEN", + "pid": 1112, + "program_name": "containerd", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "tcp", + "network_protocol": "ipv4", + "local_address": "127.0.0.53", + "local_port": "53", + "foreign_address": "0.0.0.0", + "foreign_port": "*", + "state": "LISTEN", + "pid": 885, + "program_name": "systemd-resolve", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "tcp", + "network_protocol": "ipv4", + "local_address": "0.0.0.0", + "local_port": "22", + "foreign_address": "0.0.0.0", + "foreign_port": "*", + "state": "LISTEN", + "pid": 1127, + "program_name": "sshd", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "tcp", + "network_protocol": "ipv6", + "local_address": "::", + "local_port": "22", + "foreign_address": "::", + "foreign_port": "*", + "state": "LISTEN", + "pid": 1127, + "program_name": "sshd", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "udp", + "network_protocol": "ipv4", + "local_address": "127.0.0.53", + "local_port": "53", + "foreign_address": "0.0.0.0", + "foreign_port": "*", + "pid": 885, + "program_name": "systemd-resolve", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "udp", + "network_protocol": "ipv4", + "local_address": "192.168.71.131", + "local_port": "68", + "foreign_address": "0.0.0.0", + "foreign_port": "*", + "pid": 867, + "program_name": "systemd-network", + "receive_q": 0, + "send_q": 0 } -} +] ``` ### ps ``` diff --git a/changelog.txt b/changelog.txt index 17addc74..fe8a040e 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,9 @@ jc changelog +20191021 v0.6.1 +- Flatten netstat parser output +- Clean up argument parsing + 20191018 v0.5.5 - Fix netstat -p parsing for Ubuntu - Add ps parser diff --git a/jc/jc.py b/jc/jc.py index 182fac45..03d2d083 100755 --- a/jc/jc.py +++ b/jc/jc.py @@ -14,33 +14,42 @@ import jc.parsers.route def main(): - pretty = False data = sys.stdin.read() + pretty = False - if len(sys.argv) < 2: - print('Error: jc') - print(' Must specify parser. (e.g. --ls, --netstat, --ifconfig, etc.)') - print(' Use -p to pretty print') - print('Example: ls -al | jc --ls -p\n') - exit() + if '-p' in sys.argv: + pretty = True - arg = sys.argv[1] - - if len(sys.argv) > 2: - if sys.argv[2] == '-p': - pretty = True - - if arg == '--ifconfig': + if '--ifconfig' in sys.argv: result = jc.parsers.ifconfig.parse(data) - elif arg == '--ls': + + elif '--ls' in sys.argv: result = jc.parsers.ls.parse(data) - elif arg == '--netstat': + + elif '--netstat' in sys.argv: result = jc.parsers.netstat.parse(data) - elif arg == '--ps': + + elif '--ps' in sys.argv: result = jc.parsers.ps.parse(data) - elif arg == '--route': + + elif '--route' in sys.argv: result = jc.parsers.route.parse(data) + else: + print('jc: missing arguments', file=sys.stderr) + print('Usage: jc [parser] [options]\n', file=sys.stderr) + print('Parsers:', file=sys.stderr) + print(' --ifconfig iconfig parser', file=sys.stderr) + print(' --ls ls parser', file=sys.stderr) + print(' --netstat netstat parser', file=sys.stderr) + print(' --ps ps parser', file=sys.stderr) + print(' --route route parser\n', file=sys.stderr) + print('Options:', file=sys.stderr) + print(' -p pretty print output\n', file=sys.stderr) + print('Example:', file=sys.stderr) + print(' ls -al | jc -p --ls\n', file=sys.stderr) + exit() + # output resulting dictionary as json if pretty: print(json.dumps(result, indent=2)) diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index 4061add0..de5a460c 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -11,154 +11,145 @@ Limitations: Examples: $ netstat -p | jc --netstat -p -{ - "client": { - "tcp": { - "ipv4": [ - { - "local_address": "localhost.localdo", - "local_port": "34480", - "foreign_address": "lb-192-30-255-113", - "foreign_port": "https", - "state": "ESTABLISHED", - "pid": 53550, - "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 - }, - { - "local_address": "localhost.localdo", - "local_port": "34478", - "foreign_address": "lb-192-30-255-113", - "foreign_port": "https", - "state": "ESTABLISHED", - "pid": 53550, - "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 - } - ] - } +[ + { + "session_protocol": "tcp", + "network_protocol": "ipv4", + "local_address": "localhost.localdo", + "local_port": "34480", + "foreign_address": "lb-192-30-255-113", + "foreign_port": "https", + "state": "ESTABLISHED", + "pid": 53550, + "program_name": "git-remote-ht", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "tcp", + "network_protocol": "ipv4", + "local_address": "localhost.localdo", + "local_port": "34478", + "foreign_address": "lb-192-30-255-113", + "foreign_port": "https", + "state": "ESTABLISHED", + "pid": 53550, + "program_name": "git-remote-ht", + "receive_q": 0, + "send_q": 0 } -} +] -$ netstat -lp | jc --netstat -p -{ - "server": { - "tcp": { - "ipv4": [ - { - "local_address": "localhost", - "local_port": "smtp", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "state": "LISTEN", - "pid": 1594, - "program_name": "master", - "receive_q": 0, - "send_q": 0 - }, - { - "local_address": "0.0.0.0", - "local_port": "ssh", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "state": "LISTEN", - "pid": 21918, - "program_name": "sshd", - "receive_q": 0, - "send_q": 0 - } - ], - "ipv6": [ - { - "local_address": "localhost", - "local_port": "smtp", - "foreign_address": "[::]", - "foreign_port": "*", - "state": "LISTEN", - "pid": 1594, - "program_name": "master", - "receive_q": 0, - "send_q": 0 - }, - { - "local_address": "[::]", - "local_port": "ssh", - "foreign_address": "[::]", - "foreign_port": "*", - "state": "LISTEN", - "pid": 21918, - "program_name": "sshd", - "receive_q": 0, - "send_q": 0 - } - ] - }, - "udp": { - "ipv4": [ - { - "local_address": "0.0.0.0", - "local_port": "bootpc", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "pid": 13903, - "program_name": "dhclient", - "receive_q": 0, - "send_q": 0 - }, - { - "local_address": "localhost", - "local_port": "323", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "pid": 30926, - "program_name": "chronyd", - "receive_q": 0, - "send_q": 0 - } - ], - "ipv6": [ - { - "local_address": "localhost", - "local_port": "323", - "foreign_address": "[::]", - "foreign_port": "*", - "pid": 30926, - "program_name": "chronyd", - "receive_q": 0, - "send_q": 0 - } - ] - } +$ netstat -lpn | jc --netstat -p +[ + { + "session_protocol": "tcp", + "network_protocol": "ipv4", + "local_address": "127.0.0.1", + "local_port": "42351", + "foreign_address": "0.0.0.0", + "foreign_port": "*", + "state": "LISTEN", + "pid": 1112, + "program_name": "containerd", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "tcp", + "network_protocol": "ipv4", + "local_address": "127.0.0.53", + "local_port": "53", + "foreign_address": "0.0.0.0", + "foreign_port": "*", + "state": "LISTEN", + "pid": 885, + "program_name": "systemd-resolve", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "tcp", + "network_protocol": "ipv4", + "local_address": "0.0.0.0", + "local_port": "22", + "foreign_address": "0.0.0.0", + "foreign_port": "*", + "state": "LISTEN", + "pid": 1127, + "program_name": "sshd", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "tcp", + "network_protocol": "ipv6", + "local_address": "::", + "local_port": "22", + "foreign_address": "::", + "foreign_port": "*", + "state": "LISTEN", + "pid": 1127, + "program_name": "sshd", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "udp", + "network_protocol": "ipv4", + "local_address": "127.0.0.53", + "local_port": "53", + "foreign_address": "0.0.0.0", + "foreign_port": "*", + "pid": 885, + "program_name": "systemd-resolve", + "receive_q": 0, + "send_q": 0 + }, + { + "session_protocol": "udp", + "network_protocol": "ipv4", + "local_address": "192.168.71.131", + "local_port": "68", + "foreign_address": "0.0.0.0", + "foreign_port": "*", + "pid": 867, + "program_name": "systemd-network", + "receive_q": 0, + "send_q": 0 } -} +] """ import string -output = {} - - -class state(): - section = '' - session = '' - network = '' - - client_tcp_ip4 = [] - client_tcp_ip6 = [] - client_udp_ip4 = [] - client_udp_ip6 = [] - - server_tcp_ip4 = [] - server_tcp_ip6 = [] - server_udp_ip4 = [] - server_udp_ip6 = [] +output = [] def parse_line(entry): - parsed_line = entry.split() output_line = {} + if entry.find('tcp') == 0: + output_line['session_protocol'] = 'tcp' + + if entry.find('p6') == 2: + output_line['network_protocol'] = 'ipv6' + + else: + output_line['network_protocol'] = 'ipv4' + + elif entry.find('udp') == 0: + output_line['session_protocol'] = 'udp' + + if entry.find('p6') == 2: + output_line['network_protocol'] = 'ipv6' + + else: + output_line['network_protocol'] = 'ipv4' + else: + return + + parsed_line = entry.split() + output_line['local_address'] = parsed_line[3].rsplit(':', 1)[0] output_line['local_port'] = parsed_line[3].rsplit(':', 1)[-1] output_line['foreign_address'] = parsed_line[4].rsplit(':', 1)[0] @@ -189,11 +180,9 @@ def parse(data): for line in cleandata: if line.find('Active Internet connections (w/o servers)') == 0: - state.section = 'client' continue if line.find('Active Internet connections (only servers)') == 0: - state.section = 'server' continue if line.find('Proto') == 0: @@ -202,119 +191,7 @@ def parse(data): if line.find('Active UNIX') == 0: break - if state.section == 'client': - if line.find('tcp') == 0: - state.session = 'tcp' - if line.find('p6') == 2: - state.network = 'ipv6' - else: - state.network = 'ipv4' - elif line.find('udp') == 0: - state.session = 'udp' - if line.find('p6') == 2: - state.network = 'ipv6' - else: - state.network = 'ipv4' - elif state.section == 'server': - if line.find('tcp') == 0: - state.session = 'tcp' - if line.find('p6') == 2: - state.network = 'ipv6' - else: - state.network = 'ipv4' - elif line.find('udp') == 0: - state.session = 'udp' - if line.find('p6') == 2: - state.network = 'ipv6' - else: - state.network = 'ipv4' + output.append(parse_line(line)) - # client section - if state.section == 'client' and state.session == 'tcp' and state.network == 'ipv4': - state.client_tcp_ip4.append(parse_line(line)) - - if state.section == 'client' and state.session == 'tcp' and state.network == 'ipv6': - state.client_tcp_ip6.append(parse_line(line)) - - if state.section == 'client' and state.session == 'udp' and state.network == 'ipv4': - state.client_udp_ip4.append(parse_line(line)) - - if state.section == 'client' and state.session == 'udp' and state.network == 'ipv6': - state.client_udp_ip6.append(parse_line(line)) - - # server section - if state.section == 'server' and state.session == 'tcp' and state.network == 'ipv4': - state.server_tcp_ip4.append(parse_line(line)) - - if state.section == 'server' and state.session == 'tcp' and state.network == 'ipv6': - state.server_tcp_ip6.append(parse_line(line)) - - if state.section == 'server' and state.session == 'udp' and state.network == 'ipv4': - state.server_udp_ip4.append(parse_line(line)) - - if state.section == 'server' and state.session == 'udp' and state.network == 'ipv6': - state.server_udp_ip6.append(parse_line(line)) - - state.session = '' - state.network = '' - - # build dictionary - # client section - if state.client_tcp_ip4: - if 'client' not in output: - output['client'] = {} - if 'tcp' not in output['client']: - output['client']['tcp'] = {} - output['client']['tcp']['ipv4'] = state.client_tcp_ip4 - - if state.client_tcp_ip6: - if 'client' not in output: - output['client'] = {} - if 'tcp' not in output['client']: - output['client']['tcp'] = {} - output['client']['tcp']['ipv6'] = state.client_tcp_ip6 - - if state.client_udp_ip4: - if 'client' not in output: - output['client'] = {} - if 'udp' not in output['client']: - output['client']['udp'] = {} - output['client']['udp']['ipv4'] = state.client_udp_ip4 - - if state.client_udp_ip6: - if 'client' not in output: - output['client'] = {} - if 'udp' not in output['client']: - output['client']['udp'] = {} - output['client']['udp']['ipv6'] = state.client_udp_ip6 - - # server section - if state.server_tcp_ip4: - if 'server' not in output: - output['server'] = {} - if 'tcp' not in output['server']: - output['server']['tcp'] = {} - output['server']['tcp']['ipv4'] = state.server_tcp_ip4 - - if state.server_tcp_ip6: - if 'server' not in output: - output['server'] = {} - if 'tcp' not in output['server']: - output['server']['tcp'] = {} - output['server']['tcp']['ipv6'] = state.server_tcp_ip6 - - if state.server_udp_ip4: - if 'server' not in output: - output['server'] = {} - if 'udp' not in output['server']: - output['server']['udp'] = {} - output['server']['udp']['ipv4'] = state.server_udp_ip4 - - if state.server_udp_ip6: - if 'server' not in output: - output['server'] = {} - if 'udp' not in output['server']: - output['server']['udp'] = {} - output['server']['udp']['ipv6'] = state.server_udp_ip6 - - return output + clean_output = list(filter(None, output)) + return clean_output diff --git a/setup.py b/setup.py index 27ffc49e..298f721a 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='0.5.5', + version='0.6.2', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='This tool serializes the output of popular command line tools to structured JSON output.',