From 0bec67c29ce5d965271440da41a0c4b4c2a1f0fb Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 17 Oct 2019 15:39:12 -0700 Subject: [PATCH 01/21] double quotes to single quotes --- jc/parsers/netstat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index 9291fc84..ed8c0ea7 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -185,11 +185,11 @@ def parse(data): for line in cleandata: if line.find('Active Internet connections (w/o servers)') == 0: - state.section = "client" + state.section = 'client' continue if line.find('Active Internet connections (only servers)') == 0: - state.section = "server" + state.section = 'server' continue if line.find('Proto') == 0: From c2b013150ec1f1b1ab7d416f1aa8a62165a5bb95 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 17 Oct 2019 15:51:39 -0700 Subject: [PATCH 02/21] add changelog --- changelog.txt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changelog.txt diff --git a/changelog.txt b/changelog.txt new file mode 100644 index 00000000..bf0620ac --- /dev/null +++ b/changelog.txt @@ -0,0 +1,9 @@ +jc changelog + +2019xxxx v0.5.0 +- Add ps parser +- Add route parser +- Change some ifconfig fields to integers + +20191017 v0.2.0 +- ifconfig, ls, and netstat support From 177f948f9740e3e33449b04b9136a346c9a2c265 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 17 Oct 2019 16:11:55 -0700 Subject: [PATCH 03/21] version update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f7ff05c7..6cdabcd5 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='0.2.0', + version='0.5.0', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='This tool serializes the output of popular command line tools to structured JSON output.', From 02d97394dd76a8782e6e7d3f8a85e1aed05e796b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 17 Oct 2019 17:28:14 -0700 Subject: [PATCH 04/21] changelog update --- changelog.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.txt b/changelog.txt index bf0620ac..1fd9174c 100644 --- a/changelog.txt +++ b/changelog.txt @@ -4,6 +4,7 @@ jc changelog - Add ps parser - Add route parser - Change some ifconfig fields to integers +- Fix netstat -p parsing for Ubuntu 20191017 v0.2.0 - ifconfig, ls, and netstat support From 084048987c7afc6516f69e68d37d73b2402dc7f2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 17 Oct 2019 17:59:46 -0700 Subject: [PATCH 05/21] changelog update --- changelog.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.txt b/changelog.txt index 1fd9174c..9d7b4fb1 100644 --- a/changelog.txt +++ b/changelog.txt @@ -5,6 +5,8 @@ jc changelog - Add route parser - Change some ifconfig fields to integers - Fix netstat -p parsing for Ubuntu +- Use maxsplit option in split in ls.py line 109... otherwise filenames with multiple spaces + between words can be incorrectly represented with the .join operation 20191017 v0.2.0 - ifconfig, ls, and netstat support From 756c2bc9acd60111281b76085a40778e51ca06cc Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 09:57:10 -0700 Subject: [PATCH 06/21] add acknowledgments --- README.md | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aa97f2d7..46918e5b 100755 --- a/README.md +++ b/README.md @@ -3,6 +3,21 @@ JSON CLI output utility `jc` is used to JSONify the output of many standard linux cli tools for easier parsing in scripts. Parsers for `ls`, `ifconfig`, and `netstat` are currently included and more can be added via modules. +This allows further command line processing of output with tools like `jq` simply by piping commands: + +``` +$ ls -l /usr/bin | jc --ls | jq .[] | jq 'select(.bytes > 50000000)' +{ + "filename": "emacs", + "flags": "-r-xr-xr-x", + "links": 1, + "owner": "root", + "group": "wheel", + "bytes": 117164432, + "date": "May 3 22:26" +} +``` + ## Installation ``` $ pip3 install jc @@ -19,6 +34,7 @@ The first argument is required and identifies the command that is piping output The second `-p` argument is optional and specifies whether to pretty format the JSON output. ## Examples +### ls ``` $ ls -l /bin | jc --ls -p [ @@ -52,6 +68,7 @@ $ ls -l /bin | jc --ls -p ... ] ``` +### ifconfig ``` $ ifconfig | jc --ifconfig -p [ @@ -135,6 +152,7 @@ $ ifconfig | jc --ifconfig -p } ] ``` +### netstat ``` $ netstat -p | jc --netstat -p { @@ -261,6 +279,5 @@ $ netstat -lp | jc --netstat -p } } ``` - - - +## Acknowledgments +- `ifconfig-parser` module from https://github.com/KnightWhoSayNi/ifconfig-parser From cec73d61310591e4890605f61dc5c1bef5808ec4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 09:57:22 -0700 Subject: [PATCH 07/21] linting --- jc/jc.py | 7 +++++-- jc/parsers/ifconfig.py | 7 +++---- jc/parsers/ls.py | 3 ++- jc/parsers/netstat.py | 23 ++++++++++++++--------- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/jc/jc.py b/jc/jc.py index 58551973..cc401823 100755 --- a/jc/jc.py +++ b/jc/jc.py @@ -10,14 +10,16 @@ import jc.parsers.ifconfig import jc.parsers.ls import jc.parsers.netstat + def main(): pretty = False data = sys.stdin.read() if len(sys.argv) < 2: - print(f'\nError: jc\n Must specify parser. (e.g. --ls, --netstat, --ifconfig, etc.)') + print('Error: jc') + print(' Must specify parser. (e.g. --ls, --netstat, --ifconfig, etc.)') print(' Use -p to pretty print') - print(f'\nExample: ls -al | jc --ls -p\n') + print('Example: ls -al | jc --ls -p\n') exit() arg = sys.argv[1] @@ -39,5 +41,6 @@ def main(): else: print(json.dumps(result)) + if __name__ == '__main__': main() diff --git a/jc/parsers/ifconfig.py b/jc/parsers/ifconfig.py index 4bbac6fb..e716ba05 100644 --- a/jc/parsers/ifconfig.py +++ b/jc/parsers/ifconfig.py @@ -13,9 +13,10 @@ $ ifconfig | jc --ifconfig -p from collections import namedtuple from ifconfigparser import IfconfigParser + def parse(data): output = [] - + parsed = IfconfigParser(console_output=data) interfaces = parsed.get_interfaces() @@ -24,7 +25,5 @@ def parse(data): d = interfaces[iface]._asdict() dct = dict(d) output.append(dct) - + return output - - diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index 78b275bf..4c2a411b 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -85,9 +85,10 @@ $ $ ls -l /usr/bin | jc --ls | jq .[] | jq 'select(.bytes > 50000000)' """ import re + def parse(data): output = [] - + cleandata = data.splitlines() # Delete first line if it starts with 'total' diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index ed8c0ea7..de277e12 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -138,6 +138,7 @@ import string output = {} + class state(): section = '' session = '' @@ -147,12 +148,13 @@ class state(): client_tcp_ip6 = [] client_udp_ip4 = [] client_udp_ip6 = [] - + server_tcp_ip4 = [] server_tcp_ip6 = [] server_udp_ip4 = [] server_udp_ip6 = [] + def parse_line(entry): parsed_line = entry.split() output_line = {} @@ -166,7 +168,7 @@ def parse_line(entry): if parsed_line[5][0] not in string.digits: output_line['state'] = parsed_line[5] - + if len(parsed_line) > 6: output_line['pid'] = int(parsed_line[6].split('/')[0]) output_line['program_name'] = parsed_line[6].split('/')[1] @@ -179,6 +181,7 @@ def parse_line(entry): return output_line + def parse(data): cleandata = data.splitlines() @@ -191,13 +194,13 @@ def parse(data): if line.find('Active Internet connections (only servers)') == 0: state.section = 'server' continue - + if line.find('Proto') == 0: continue if line.find('Active UNIX') == 0: break - + if state.section == 'client': if line.find('tcp') == 0: state.session = 'tcp' @@ -225,6 +228,7 @@ def parse(data): else: state.network = 'ipv4' + # client section if state.section == 'client' and state.session == 'tcp' and state.network == 'ipv4': state.client_tcp_ip4.append(parse_line(line)) @@ -237,7 +241,7 @@ def parse(data): if state.section == 'client' and state.session == 'udp' and state.network == 'ipv6': state.client_udp_ip6.append(parse_line(line)) - + # server section if state.section == 'server' and state.session == 'tcp' and state.network == 'ipv4': state.server_tcp_ip4.append(parse_line(line)) @@ -254,6 +258,7 @@ def parse(data): state.network = '' # build dictionary + # client section if state.client_tcp_ip4: if 'client' not in output: output['client'] = {} @@ -281,8 +286,8 @@ def parse(data): if 'udp' not in output['client']: output['client']['udp'] = {} output['client']['udp']['ipv6'] = state.client_udp_ip6 - - + + # server section if state.server_tcp_ip4: if 'server' not in output: output['server'] = {} @@ -303,7 +308,7 @@ def parse(data): if 'udp' not in output['server']: output['server']['udp'] = {} output['server']['udp']['ipv4'] = state.server_udp_ip4 - + if state.server_udp_ip6: if 'server' not in output: output['server'] = {} @@ -311,4 +316,4 @@ def parse(data): output['server']['udp'] = {} output['server']['udp']['ipv6'] = state.server_udp_ip6 - return output \ No newline at end of file + return output From c04895407f203aa0a3a080d5242faf8f6889ae41 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 12:57:02 -0700 Subject: [PATCH 08/21] ubuntu fixes --- changelog.txt | 1 + jc/parsers/netstat.py | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/changelog.txt b/changelog.txt index 9d7b4fb1..9a435c34 100644 --- a/changelog.txt +++ b/changelog.txt @@ -7,6 +7,7 @@ jc changelog - Fix netstat -p parsing for Ubuntu - Use maxsplit option in split in ls.py line 109... otherwise filenames with multiple spaces between words can be incorrectly represented with the .join operation +- Use list(filter(None, cleandata)) or list comprehension to clean any blank entries in ls.py line 98 20191017 v0.2.0 - ifconfig, ls, and netstat support diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index de277e12..4061add0 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -166,15 +166,16 @@ def parse_line(entry): if len(parsed_line) > 5: - if parsed_line[5][0] not in string.digits: + if parsed_line[5][0] not in string.digits and parsed_line[5][0] != '-': output_line['state'] = parsed_line[5] - if len(parsed_line) > 6: + if len(parsed_line) > 6 and parsed_line[6][0] in string.digits: output_line['pid'] = int(parsed_line[6].split('/')[0]) output_line['program_name'] = parsed_line[6].split('/')[1] else: - output_line['pid'] = int(parsed_line[5].split('/')[0]) - output_line['program_name'] = parsed_line[5].split('/')[1] + if parsed_line[5][0] in string.digits: + output_line['pid'] = int(parsed_line[5].split('/')[0]) + output_line['program_name'] = parsed_line[5].split('/')[1] output_line['receive_q'] = int(parsed_line[1]) output_line['send_q'] = int(parsed_line[2]) From 8ec1bec31711d2ef487da3e4a29a842d98d77626 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 13:19:39 -0700 Subject: [PATCH 09/21] add ps parser --- changelog.txt | 10 +++++----- jc/__init__.py | 2 -- jc/jc.py | 3 +++ jc/parsers/ps.py | 21 +++++++++++++++++++++ 4 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 jc/parsers/ps.py diff --git a/changelog.txt b/changelog.txt index 9a435c34..97004a7b 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,13 +1,13 @@ jc changelog 2019xxxx v0.5.0 -- Add ps parser -- Add route parser -- Change some ifconfig fields to integers - Fix netstat -p parsing for Ubuntu -- Use maxsplit option in split in ls.py line 109... otherwise filenames with multiple spaces ++ Add ps parser ++ Add route parser ++ Change some ifconfig fields to integers ++ Use maxsplit option in split in ls.py line 109... otherwise filenames with multiple spaces between words can be incorrectly represented with the .join operation -- Use list(filter(None, cleandata)) or list comprehension to clean any blank entries in ls.py line 98 ++ Use list(filter(None, cleandata)) or list comprehension to clean any blank entries in ls.py line 98 20191017 v0.2.0 - ifconfig, ls, and netstat support diff --git a/jc/__init__.py b/jc/__init__.py index fb8c1668..d584b3c6 100644 --- a/jc/__init__.py +++ b/jc/__init__.py @@ -1,7 +1,5 @@ """JC - JSON CLI output utility -v0.1 - * kellyjonbrazil@gmail.com This module serializes standard unix command line output to structured JSON diff --git a/jc/jc.py b/jc/jc.py index cc401823..5f981b7e 100755 --- a/jc/jc.py +++ b/jc/jc.py @@ -9,6 +9,7 @@ import json import jc.parsers.ifconfig import jc.parsers.ls import jc.parsers.netstat +import jc.parsers.ps def main(): @@ -34,6 +35,8 @@ def main(): result = jc.parsers.ls.parse(data) elif arg == '--netstat': result = jc.parsers.netstat.parse(data) + elif arg == '--ps': + result = jc.parsers.ps.parse(data) # output resulting dictionary as json if pretty: diff --git a/jc/parsers/ps.py b/jc/parsers/ps.py new file mode 100644 index 00000000..c3c27bfa --- /dev/null +++ b/jc/parsers/ps.py @@ -0,0 +1,21 @@ +"""jc - JSON CLI output utility ps Parser + +Usage: + specify --ps as the first argument if the piped input is coming from ps + + ps options supported: + - ef + - axu + +Examples: + +""" + + +def parse(data): + + cleandata = data.splitlines() + + headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) + return [dict(zip(headers, r)) for r in raw_data] From 7571139f7980f6e74c29d46971881e82ab0f04bc Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 13:20:38 -0700 Subject: [PATCH 10/21] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6cdabcd5..b389bf5b 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='0.5.0', + version='0.5.1', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='This tool serializes the output of popular command line tools to structured JSON output.', From 4f552e370e656cc1882a7dae231475a016f8aa14 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 13:26:24 -0700 Subject: [PATCH 11/21] acknowledgment --- jc/parsers/ps.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jc/parsers/ps.py b/jc/parsers/ps.py index c3c27bfa..f494d452 100644 --- a/jc/parsers/ps.py +++ b/jc/parsers/ps.py @@ -14,8 +14,10 @@ Examples: def parse(data): - cleandata = data.splitlines() + # code adapted from Conor Heine at: + # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 + cleandata = data.splitlines() headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) return [dict(zip(headers, r)) for r in raw_data] From 367ab54f94bf7f456377ec9d20f8daf99dbdc1ed Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 13:34:28 -0700 Subject: [PATCH 12/21] formatting --- README.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ changelog.txt | 2 +- jc/parsers/ps.py | 46 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 46918e5b..8edaee88 100755 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ The first argument is required and identifies the command that is piping output - `--ls` enables the `ls` parser - `--ifconfig` enables the `ifconfig` parser - `--netstat` enables the `netstat` parser +- `--ps` enables the `ps` parser The second `-p` argument is optional and specifies whether to pretty format the JSON output. @@ -279,5 +280,57 @@ $ netstat -lp | jc --netstat -p } } ``` +### ps +``` +$ ps -ef | jc --ps -p +[ + { + "UID": "root", + "PID": "1", + "PPID": "0", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:05", + "CMD": "/lib/systemd/systemd --system --deserialize 35" + }, + { + "UID": "root", + "PID": "2", + "PPID": "0", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[kthreadd]" + }, + { + "UID": "root", + "PID": "4", + "PPID": "2", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[kworker/0:0H]" + }, + { + "UID": "root", + "PID": "6", + "PPID": "2", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[mm_percpu_wq]" + }, + ... +] +``` + +## Contributions +Feel free to add/improve code or parsers! + ## Acknowledgments - `ifconfig-parser` module from https://github.com/KnightWhoSayNi/ifconfig-parser +- Parsing code from Conor Heine at https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 diff --git a/changelog.txt b/changelog.txt index 97004a7b..10356c01 100644 --- a/changelog.txt +++ b/changelog.txt @@ -2,7 +2,7 @@ jc changelog 2019xxxx v0.5.0 - Fix netstat -p parsing for Ubuntu -+ Add ps parser +- Add ps parser + Add route parser + Change some ifconfig fields to integers + Use maxsplit option in split in ls.py line 109... otherwise filenames with multiple spaces diff --git a/jc/parsers/ps.py b/jc/parsers/ps.py index f494d452..14fc45f0 100644 --- a/jc/parsers/ps.py +++ b/jc/parsers/ps.py @@ -7,8 +7,52 @@ Usage: - ef - axu -Examples: +Example: +$ ps -ef | jc --ps -p +[ + { + "UID": "root", + "PID": "1", + "PPID": "0", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:05", + "CMD": "/lib/systemd/systemd --system --deserialize 35" + }, + { + "UID": "root", + "PID": "2", + "PPID": "0", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[kthreadd]" + }, + { + "UID": "root", + "PID": "4", + "PPID": "2", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[kworker/0:0H]" + }, + { + "UID": "root", + "PID": "6", + "PPID": "2", + "C": "0", + "STIME": "13:58", + "TTY": "?", + "TIME": "00:00:00", + "CMD": "[mm_percpu_wq]" + }, + ... +] """ From 43ed09ce5ba53261dfd7ef50a2e187cb524c416c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 13:38:11 -0700 Subject: [PATCH 13/21] add route parser --- jc/jc.py | 3 +++ jc/parsers/route.py | 21 +++++++++++++++++++++ setup.py | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 jc/parsers/route.py diff --git a/jc/jc.py b/jc/jc.py index 5f981b7e..182fac45 100755 --- a/jc/jc.py +++ b/jc/jc.py @@ -10,6 +10,7 @@ import jc.parsers.ifconfig import jc.parsers.ls import jc.parsers.netstat import jc.parsers.ps +import jc.parsers.route def main(): @@ -37,6 +38,8 @@ def main(): result = jc.parsers.netstat.parse(data) elif arg == '--ps': result = jc.parsers.ps.parse(data) + elif arg == '--route': + result = jc.parsers.route.parse(data) # output resulting dictionary as json if pretty: diff --git a/jc/parsers/route.py b/jc/parsers/route.py new file mode 100644 index 00000000..ca37fbe2 --- /dev/null +++ b/jc/parsers/route.py @@ -0,0 +1,21 @@ +"""jc - JSON CLI output utility route Parser + +Usage: + specify --route as the first argument if the piped input is coming from route + + +Example: + + +""" + + +def parse(data): + + # code adapted from Conor Heine at: + # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 + + cleandata = data.splitlines() + headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) + return [dict(zip(headers, r)) for r in raw_data] diff --git a/setup.py b/setup.py index b389bf5b..530a3522 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='0.5.1', + version='0.5.2', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='This tool serializes the output of popular command line tools to structured JSON output.', From 2eba30422b44f648c7356a44cbff7416c6648f19 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 13:43:49 -0700 Subject: [PATCH 14/21] skip first line --- jc/parsers/route.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/route.py b/jc/parsers/route.py index ca37fbe2..9911a738 100644 --- a/jc/parsers/route.py +++ b/jc/parsers/route.py @@ -15,7 +15,7 @@ def parse(data): # code adapted from Conor Heine at: # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 - cleandata = data.splitlines() + cleandata = data.splitlines()[1:] headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) return [dict(zip(headers, r)) for r in raw_data] From 51a1d3e6f48cf7084267be4092f0dd3f778dcef3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 13:51:36 -0700 Subject: [PATCH 15/21] version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 530a3522..a649a3f2 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='0.5.2', + version='0.5.3', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='This tool serializes the output of popular command line tools to structured JSON output.', From 77b74c550795f96b808e48a9b0bfd1a8af9866f9 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 13:59:30 -0700 Subject: [PATCH 16/21] add route parser --- changelog.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.txt b/changelog.txt index 10356c01..b37f2e71 100644 --- a/changelog.txt +++ b/changelog.txt @@ -3,7 +3,7 @@ jc changelog 2019xxxx v0.5.0 - Fix netstat -p parsing for Ubuntu - Add ps parser -+ Add route parser +- Add route parser + Change some ifconfig fields to integers + Use maxsplit option in split in ls.py line 109... otherwise filenames with multiple spaces between words can be incorrectly represented with the .join operation From 4d93b38fe47329e3175025878e87f7c963061f98 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 14:18:34 -0700 Subject: [PATCH 17/21] add route documentation --- README.md | 47 +++++++++++++++++++++++++++++++++++++++++++++ jc/parsers/route.py | 44 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8edaee88..b3672b63 100755 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ The first argument is required and identifies the command that is piping output - `--ifconfig` enables the `ifconfig` parser - `--netstat` enables the `netstat` parser - `--ps` enables the `ps` parser +- `--route` enables the `route` parser The second `-p` argument is optional and specifies whether to pretty format the JSON output. @@ -327,6 +328,52 @@ $ ps -ef | jc --ps -p ... ] ``` +### route +``` +$ route -n | jc --route -p +[ + { + "Destination": "0.0.0.0", + "Gateway": "192.168.71.2", + "Genmask": "0.0.0.0", + "Flags": "UG", + "Metric": "100", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + }, + { + "Destination": "172.17.0.0", + "Gateway": "0.0.0.0", + "Genmask": "255.255.0.0", + "Flags": "U", + "Metric": "0", + "Ref": "0", + "Use": "0", + "Iface": "docker0" + }, + { + "Destination": "192.168.71.0", + "Gateway": "0.0.0.0", + "Genmask": "255.255.255.0", + "Flags": "U", + "Metric": "0", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + }, + { + "Destination": "192.168.71.2", + "Gateway": "0.0.0.0", + "Genmask": "255.255.255.255", + "Flags": "UH", + "Metric": "100", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + } +] +``` ## Contributions Feel free to add/improve code or parsers! diff --git a/jc/parsers/route.py b/jc/parsers/route.py index 9911a738..46c8f2dd 100644 --- a/jc/parsers/route.py +++ b/jc/parsers/route.py @@ -6,7 +6,49 @@ Usage: Example: - +$ route -n | jc --route -p +[ + { + "Destination": "0.0.0.0", + "Gateway": "192.168.71.2", + "Genmask": "0.0.0.0", + "Flags": "UG", + "Metric": "100", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + }, + { + "Destination": "172.17.0.0", + "Gateway": "0.0.0.0", + "Genmask": "255.255.0.0", + "Flags": "U", + "Metric": "0", + "Ref": "0", + "Use": "0", + "Iface": "docker0" + }, + { + "Destination": "192.168.71.0", + "Gateway": "0.0.0.0", + "Genmask": "255.255.255.0", + "Flags": "U", + "Metric": "0", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + }, + { + "Destination": "192.168.71.2", + "Gateway": "0.0.0.0", + "Genmask": "255.255.255.255", + "Flags": "UH", + "Metric": "100", + "Ref": "0", + "Use": "0", + "Iface": "ens33" + } +] """ From a9294f32a00709737d0b82ff434a311e75ad0bed Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 18:40:56 -0700 Subject: [PATCH 18/21] ls fixes --- changelog.txt | 7 ++---- jc/parsers/ls.py | 58 ++++++++++++++++++++++++------------------------ setup.py | 2 +- 3 files changed, 32 insertions(+), 35 deletions(-) diff --git a/changelog.txt b/changelog.txt index b37f2e71..d929c5e5 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,13 +1,10 @@ jc changelog -2019xxxx v0.5.0 +20191018 v0.5.4 - Fix netstat -p parsing for Ubuntu - Add ps parser - Add route parser -+ Change some ifconfig fields to integers -+ Use maxsplit option in split in ls.py line 109... otherwise filenames with multiple spaces - between words can be incorrectly represented with the .join operation -+ Use list(filter(None, cleandata)) or list comprehension to clean any blank entries in ls.py line 98 +- ls parser fixes 20191017 v0.2.0 - ifconfig, ls, and netstat support diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index 4c2a411b..ad985a2f 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -89,43 +89,43 @@ import re def parse(data): output = [] - cleandata = data.splitlines() + linedata = data.splitlines() # Delete first line if it starts with 'total' - if cleandata[0].find('total') == 0: - cleandata.pop(0) + if linedata[0].find('total') == 0: + linedata.pop(0) - # Delete last line if it is blank - if cleandata[-1] == '': - cleandata.pop(-1) + # Clear any blank lines + cleandata = list(filter(None, linedata)) - # Check if -l was used to parse extra data - if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', cleandata[0]): - for entry in cleandata: - output_line = {} + if cleandata: + # Check if -l was used to parse extra data + if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', cleandata[0]): + for entry in cleandata: + output_line = {} - parsed_line = entry.split() + parsed_line = entry.split(maxsplit=8) - # split filenames and links - filename_field = ' '.join(parsed_line[8:]).split(' -> ') + # split filenames and links + filename_field = parsed_line[8].split(' -> ') - # create list of dictionaries - output_line['filename'] = filename_field[0] + # create list of dictionaries + output_line['filename'] = filename_field[0] - if len(filename_field) > 1: - output_line['link_to'] = filename_field[1] + if len(filename_field) > 1: + output_line['link_to'] = filename_field[1] - output_line['flags'] = parsed_line[0] - output_line['links'] = int(parsed_line[1]) - output_line['owner'] = parsed_line[2] - output_line['group'] = parsed_line[3] - output_line['bytes'] = int(parsed_line[4]) - output_line['date'] = ' '.join(parsed_line[5:8]) - output.append(output_line) - else: - for entry in cleandata: - output_line = {} - output_line['filename'] = entry - output.append(output_line) + output_line['flags'] = parsed_line[0] + output_line['links'] = int(parsed_line[1]) + output_line['owner'] = parsed_line[2] + output_line['group'] = parsed_line[3] + output_line['bytes'] = int(parsed_line[4]) + output_line['date'] = ' '.join(parsed_line[5:8]) + output.append(output_line) + else: + for entry in cleandata: + output_line = {} + output_line['filename'] = entry + output.append(output_line) return output diff --git a/setup.py b/setup.py index a649a3f2..301394f4 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='0.5.3', + version='0.5.4', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='This tool serializes the output of popular command line tools to structured JSON output.', From d6f94c53a4df84fb75554605477e49536a809dc2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 18:46:25 -0700 Subject: [PATCH 19/21] fix blank output case in ls --- jc/parsers/ls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index ad985a2f..24a8d841 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -92,8 +92,9 @@ def parse(data): linedata = data.splitlines() # Delete first line if it starts with 'total' - if linedata[0].find('total') == 0: - linedata.pop(0) + if linedata: + if linedata[0].find('total') == 0: + linedata.pop(0) # Clear any blank lines cleandata = list(filter(None, linedata)) From 9181d242aaed5219bb35fde01d672be88949cceb Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 18:47:20 -0700 Subject: [PATCH 20/21] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 301394f4..27ffc49e 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='0.5.4', + version='0.5.5', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='This tool serializes the output of popular command line tools to structured JSON output.', From e416c77bed1267254da972b0f95b7ff1d43fccef Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Oct 2019 18:56:26 -0700 Subject: [PATCH 21/21] version bump --- changelog.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.txt b/changelog.txt index d929c5e5..17addc74 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,6 +1,6 @@ jc changelog -20191018 v0.5.4 +20191018 v0.5.5 - Fix netstat -p parsing for Ubuntu - Add ps parser - Add route parser