python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-19 00:17:51 +02:00
Code Issues Releases Activity

Merge pull request #1 from kellyjonbrazil/dev

Browse Source 
Dev v0.5.5
This commit is contained in:
Kelly Brazil
2019-10-18 18:57:32 -07:00
committed by GitHub
parent 8073d15fe1 e416c77bed
commit 0040febbf0
10 changed files with 326 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
README.md
View File

@ -3,6 +3,21 @@ JSON CLI output utility
`jc` is used to JSONify the output of many standard linux cli tools for easier parsing in scripts. Parsers for `ls`, `ifconfig`, and `netstat` are currently included and more can be added via modules. `jc` is used to JSONify the output of many standard linux cli tools for easier parsing in scripts. Parsers for `ls`, `ifconfig`, and `netstat` are currently included and more can be added via modules.
This allows further command line processing of output with tools like `jq` simply by piping commands:
```
$ ls -l /usr/bin | jc --ls | jq .[] | jq 'select(.bytes > 50000000)'
{
"filename": "emacs",
"flags": "-r-xr-xr-x",
"links": 1,
"owner": "root",
"group": "wheel",
"bytes": 117164432,
"date": "May 3 22:26"
}
```
## Installation ## Installation
``` ```
$ pip3 install jc $ pip3 install jc
@ -15,10 +30,13 @@ The first argument is required and identifies the command that is piping output
- `--ls` enables the `ls` parser - `--ls` enables the `ls` parser
- `--ifconfig` enables the `ifconfig` parser - `--ifconfig` enables the `ifconfig` parser
- `--netstat` enables the `netstat` parser - `--netstat` enables the `netstat` parser
- `--ps` enables the `ps` parser
- `--route` enables the `route` parser
The second `-p` argument is optional and specifies whether to pretty format the JSON output. The second `-p` argument is optional and specifies whether to pretty format the JSON output.
## Examples ## Examples
### ls
``` ```
$ ls -l /bin | jc --ls -p $ ls -l /bin | jc --ls -p
[ [
@ -52,6 +70,7 @@ $ ls -l /bin | jc --ls -p
... ...
] ]
``` ```
### ifconfig
``` ```
$ ifconfig | jc --ifconfig -p $ ifconfig | jc --ifconfig -p
[ [
@ -135,6 +154,7 @@ $ ifconfig | jc --ifconfig -p
} }
] ]
``` ```
### netstat
``` ```
$ netstat -p | jc --netstat -p $ netstat -p | jc --netstat -p
{ {
@ -261,6 +281,103 @@ $ netstat -lp | jc --netstat -p
} }
} }
``` ```
### ps
```
$ ps -ef | jc --ps -p
[
{
"UID": "root",
"PID": "1",
"PPID": "0",
"C": "0",
"STIME": "13:58",
"TTY": "?",
"TIME": "00:00:05",
"CMD": "/lib/systemd/systemd --system --deserialize 35"
},
{
"UID": "root",
"PID": "2",
"PPID": "0",
"C": "0",
"STIME": "13:58",
"TTY": "?",
"TIME": "00:00:00",
"CMD": "[kthreadd]"
},
{
"UID": "root",
"PID": "4",
"PPID": "2",
"C": "0",
"STIME": "13:58",
"TTY": "?",
"TIME": "00:00:00",
"CMD": "[kworker/0:0H]"
},
{
"UID": "root",
"PID": "6",
"PPID": "2",
"C": "0",
"STIME": "13:58",
"TTY": "?",
"TIME": "00:00:00",
"CMD": "[mm_percpu_wq]"
},
...
]
```
### route
```
$ route -n | jc --route -p
[
{
"Destination": "0.0.0.0",
"Gateway": "192.168.71.2",
"Genmask": "0.0.0.0",
"Flags": "UG",
"Metric": "100",
"Ref": "0",
"Use": "0",
"Iface": "ens33"
},
{
"Destination": "172.17.0.0",
"Gateway": "0.0.0.0",
"Genmask": "255.255.0.0",
"Flags": "U",
"Metric": "0",
"Ref": "0",
"Use": "0",
"Iface": "docker0"
},
{
"Destination": "192.168.71.0",
"Gateway": "0.0.0.0",
"Genmask": "255.255.255.0",
"Flags": "U",
"Metric": "0",
"Ref": "0",
"Use": "0",
"Iface": "ens33"
},
{
"Destination": "192.168.71.2",
"Gateway": "0.0.0.0",
"Genmask": "255.255.255.255",
"Flags": "UH",
"Metric": "100",
"Ref": "0",
"Use": "0",
"Iface": "ens33"
}
]
```
## Contributions
Feel free to add/improve code or parsers!
## Acknowledgments
- `ifconfig-parser` module from https://github.com/KnightWhoSayNi/ifconfig-parser
- Parsing code from Conor Heine at https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501

10
changelog.txt Normal file
View File

@ -0,0 +1,10 @@
jc changelog
20191018 v0.5.5
- Fix netstat -p parsing for Ubuntu
- Add ps parser
- Add route parser
- ls parser fixes
20191017 v0.2.0
- ifconfig, ls, and netstat support

2
jc/__init__.py
View File

@ -1,7 +1,5 @@
"""JC - JSON CLI output utility """JC - JSON CLI output utility
v0.1
* kellyjonbrazil@gmail.com * kellyjonbrazil@gmail.com
This module serializes standard unix command line output to structured JSON This module serializes standard unix command line output to structured JSON

13
jc/jc.py
View File

@ -9,15 +9,19 @@ import json
import jc.parsers.ifconfig import jc.parsers.ifconfig
import jc.parsers.ls import jc.parsers.ls
import jc.parsers.netstat import jc.parsers.netstat
import jc.parsers.ps
import jc.parsers.route
def main(): def main():
pretty = False pretty = False
data = sys.stdin.read() data = sys.stdin.read()
if len(sys.argv) < 2: if len(sys.argv) < 2:
print(f'\nError: jc\n Must specify parser. (e.g. --ls, --netstat, --ifconfig, etc.)') print('Error: jc')
print(' Must specify parser. (e.g. --ls, --netstat, --ifconfig, etc.)')
print(' Use -p to pretty print') print(' Use -p to pretty print')
print(f'\nExample: ls -al | jc --ls -p\n') print('Example: ls -al | jc --ls -p\n')
exit() exit()
arg = sys.argv[1] arg = sys.argv[1]
@ -32,6 +36,10 @@ def main():
result = jc.parsers.ls.parse(data) result = jc.parsers.ls.parse(data)
elif arg == '--netstat': elif arg == '--netstat':
result = jc.parsers.netstat.parse(data) result = jc.parsers.netstat.parse(data)
elif arg == '--ps':
result = jc.parsers.ps.parse(data)
elif arg == '--route':
result = jc.parsers.route.parse(data)
# output resulting dictionary as json # output resulting dictionary as json
if pretty: if pretty:
@ -39,5 +47,6 @@ def main():
else: else:
print(json.dumps(result)) print(json.dumps(result))
if __name__ == '__main__': if __name__ == '__main__':
main() main()

7
jc/parsers/ifconfig.py
View File

@ -13,9 +13,10 @@ $ ifconfig | jc --ifconfig -p
from collections import namedtuple from collections import namedtuple
from ifconfigparser import IfconfigParser from ifconfigparser import IfconfigParser
def parse(data): def parse(data):
output = [] output = []
parsed = IfconfigParser(console_output=data) parsed = IfconfigParser(console_output=data)
interfaces = parsed.get_interfaces() interfaces = parsed.get_interfaces()
@ -24,7 +25,5 @@ def parse(data):
d = interfaces[iface]._asdict() d = interfaces[iface]._asdict()
dct = dict(d) dct = dict(d)
output.append(dct) output.append(dct)
return output return output

62
jc/parsers/ls.py
View File

@ -85,46 +85,48 @@ $ $ ls -l /usr/bin | jc --ls | jq .[] | jq 'select(.bytes > 50000000)'
""" """
import re import re
def parse(data): def parse(data):
output = [] output = []
cleandata = data.splitlines() linedata = data.splitlines()
# Delete first line if it starts with 'total' # Delete first line if it starts with 'total'
if cleandata[0].find('total') == 0: if linedata:
cleandata.pop(0) if linedata[0].find('total') == 0:
linedata.pop(0)
# Delete last line if it is blank # Clear any blank lines
if cleandata[-1] == '': cleandata = list(filter(None, linedata))
cleandata.pop(-1)
# Check if -l was used to parse extra data if cleandata:
if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', cleandata[0]): # Check if -l was used to parse extra data
for entry in cleandata: if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', cleandata[0]):
output_line = {} for entry in cleandata:
output_line = {}
parsed_line = entry.split() parsed_line = entry.split(maxsplit=8)
# split filenames and links # split filenames and links
filename_field = ' '.join(parsed_line[8:]).split(' -> ') filename_field = parsed_line[8].split(' -> ')
# create list of dictionaries # create list of dictionaries
output_line['filename'] = filename_field[0] output_line['filename'] = filename_field[0]
if len(filename_field) > 1: if len(filename_field) > 1:
output_line['link_to'] = filename_field[1] output_line['link_to'] = filename_field[1]
output_line['flags'] = parsed_line[0] output_line['flags'] = parsed_line[0]
output_line['links'] = int(parsed_line[1]) output_line['links'] = int(parsed_line[1])
output_line['owner'] = parsed_line[2] output_line['owner'] = parsed_line[2]
output_line['group'] = parsed_line[3] output_line['group'] = parsed_line[3]
output_line['bytes'] = int(parsed_line[4]) output_line['bytes'] = int(parsed_line[4])
output_line['date'] = ' '.join(parsed_line[5:8]) output_line['date'] = ' '.join(parsed_line[5:8])
output.append(output_line) output.append(output_line)
else: else:
for entry in cleandata: for entry in cleandata:
output_line = {} output_line = {}
output_line['filename'] = entry output_line['filename'] = entry
output.append(output_line) output.append(output_line)
return output return output

36
jc/parsers/netstat.py
View File

@ -138,6 +138,7 @@ import string
output = {} output = {}
class state(): class state():
section = '' section = ''
session = '' session = ''
@ -147,12 +148,13 @@ class state():
client_tcp_ip6 = [] client_tcp_ip6 = []
client_udp_ip4 = [] client_udp_ip4 = []
client_udp_ip6 = [] client_udp_ip6 = []
server_tcp_ip4 = [] server_tcp_ip4 = []
server_tcp_ip6 = [] server_tcp_ip6 = []
server_udp_ip4 = [] server_udp_ip4 = []
server_udp_ip6 = [] server_udp_ip6 = []
def parse_line(entry): def parse_line(entry):
parsed_line = entry.split() parsed_line = entry.split()
output_line = {} output_line = {}
@ -164,40 +166,42 @@ def parse_line(entry):
if len(parsed_line) > 5: if len(parsed_line) > 5:
if parsed_line[5][0] not in string.digits: if parsed_line[5][0] not in string.digits and parsed_line[5][0] != '-':
output_line['state'] = parsed_line[5] output_line['state'] = parsed_line[5]
if len(parsed_line) > 6: if len(parsed_line) > 6 and parsed_line[6][0] in string.digits:
output_line['pid'] = int(parsed_line[6].split('/')[0]) output_line['pid'] = int(parsed_line[6].split('/')[0])
output_line['program_name'] = parsed_line[6].split('/')[1] output_line['program_name'] = parsed_line[6].split('/')[1]
else: else:
output_line['pid'] = int(parsed_line[5].split('/')[0]) if parsed_line[5][0] in string.digits:
output_line['program_name'] = parsed_line[5].split('/')[1] output_line['pid'] = int(parsed_line[5].split('/')[0])
output_line['program_name'] = parsed_line[5].split('/')[1]
output_line['receive_q'] = int(parsed_line[1]) output_line['receive_q'] = int(parsed_line[1])
output_line['send_q'] = int(parsed_line[2]) output_line['send_q'] = int(parsed_line[2])
return output_line return output_line
def parse(data): def parse(data):
cleandata = data.splitlines() cleandata = data.splitlines()
for line in cleandata: for line in cleandata:
if line.find('Active Internet connections (w/o servers)') == 0: if line.find('Active Internet connections (w/o servers)') == 0:
state.section = "client" state.section = 'client'
continue continue
if line.find('Active Internet connections (only servers)') == 0: if line.find('Active Internet connections (only servers)') == 0:
state.section = "server" state.section = 'server'
continue continue
if line.find('Proto') == 0: if line.find('Proto') == 0:
continue continue
if line.find('Active UNIX') == 0: if line.find('Active UNIX') == 0:
break break
if state.section == 'client': if state.section == 'client':
if line.find('tcp') == 0: if line.find('tcp') == 0:
state.session = 'tcp' state.session = 'tcp'
@ -225,6 +229,7 @@ def parse(data):
else: else:
state.network = 'ipv4' state.network = 'ipv4'
# client section
if state.section == 'client' and state.session == 'tcp' and state.network == 'ipv4': if state.section == 'client' and state.session == 'tcp' and state.network == 'ipv4':
state.client_tcp_ip4.append(parse_line(line)) state.client_tcp_ip4.append(parse_line(line))
@ -237,7 +242,7 @@ def parse(data):
if state.section == 'client' and state.session == 'udp' and state.network == 'ipv6': if state.section == 'client' and state.session == 'udp' and state.network == 'ipv6':
state.client_udp_ip6.append(parse_line(line)) state.client_udp_ip6.append(parse_line(line))
# server section
if state.section == 'server' and state.session == 'tcp' and state.network == 'ipv4': if state.section == 'server' and state.session == 'tcp' and state.network == 'ipv4':
state.server_tcp_ip4.append(parse_line(line)) state.server_tcp_ip4.append(parse_line(line))
@ -254,6 +259,7 @@ def parse(data):
state.network = '' state.network = ''
# build dictionary # build dictionary
# client section
if state.client_tcp_ip4: if state.client_tcp_ip4:
if 'client' not in output: if 'client' not in output:
output['client'] = {} output['client'] = {}
@ -281,8 +287,8 @@ def parse(data):
if 'udp' not in output['client']: if 'udp' not in output['client']:
output['client']['udp'] = {} output['client']['udp'] = {}
output['client']['udp']['ipv6'] = state.client_udp_ip6 output['client']['udp']['ipv6'] = state.client_udp_ip6
# server section
if state.server_tcp_ip4: if state.server_tcp_ip4:
if 'server' not in output: if 'server' not in output:
output['server'] = {} output['server'] = {}
@ -303,7 +309,7 @@ def parse(data):
if 'udp' not in output['server']: if 'udp' not in output['server']:
output['server']['udp'] = {} output['server']['udp'] = {}
output['server']['udp']['ipv4'] = state.server_udp_ip4 output['server']['udp']['ipv4'] = state.server_udp_ip4
if state.server_udp_ip6: if state.server_udp_ip6:
if 'server' not in output: if 'server' not in output:
output['server'] = {} output['server'] = {}
@ -311,4 +317,4 @@ def parse(data):
output['server']['udp'] = {} output['server']['udp'] = {}
output['server']['udp']['ipv6'] = state.server_udp_ip6 output['server']['udp']['ipv6'] = state.server_udp_ip6
return output return output

67
jc/parsers/ps.py Normal file
View File

@ -0,0 +1,67 @@
"""jc - JSON CLI output utility ps Parser
Usage:
specify --ps as the first argument if the piped input is coming from ps
ps options supported:
- ef
- axu
Example:
$ ps -ef | jc --ps -p
[
{
"UID": "root",
"PID": "1",
"PPID": "0",
"C": "0",
"STIME": "13:58",
"TTY": "?",
"TIME": "00:00:05",
"CMD": "/lib/systemd/systemd --system --deserialize 35"
},
{
"UID": "root",
"PID": "2",
"PPID": "0",
"C": "0",
"STIME": "13:58",
"TTY": "?",
"TIME": "00:00:00",
"CMD": "[kthreadd]"
},
{
"UID": "root",
"PID": "4",
"PPID": "2",
"C": "0",
"STIME": "13:58",
"TTY": "?",
"TIME": "00:00:00",
"CMD": "[kworker/0:0H]"
},
{
"UID": "root",
"PID": "6",
"PPID": "2",
"C": "0",
"STIME": "13:58",
"TTY": "?",
"TIME": "00:00:00",
"CMD": "[mm_percpu_wq]"
},
...
]
"""
def parse(data):
# code adapted from Conor Heine at:
# https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501
cleandata = data.splitlines()
headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h]
raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:])
return [dict(zip(headers, r)) for r in raw_data]

63
jc/parsers/route.py Normal file
View File

@ -0,0 +1,63 @@
"""jc - JSON CLI output utility route Parser
Usage:
specify --route as the first argument if the piped input is coming from route
Example:
$ route -n | jc --route -p
[
{
"Destination": "0.0.0.0",
"Gateway": "192.168.71.2",
"Genmask": "0.0.0.0",
"Flags": "UG",
"Metric": "100",
"Ref": "0",
"Use": "0",
"Iface": "ens33"
},
{
"Destination": "172.17.0.0",
"Gateway": "0.0.0.0",
"Genmask": "255.255.0.0",
"Flags": "U",
"Metric": "0",
"Ref": "0",
"Use": "0",
"Iface": "docker0"
},
{
"Destination": "192.168.71.0",
"Gateway": "0.0.0.0",
"Genmask": "255.255.255.0",
"Flags": "U",
"Metric": "0",
"Ref": "0",
"Use": "0",
"Iface": "ens33"
},
{
"Destination": "192.168.71.2",
"Gateway": "0.0.0.0",
"Genmask": "255.255.255.255",
"Flags": "UH",
"Metric": "100",
"Ref": "0",
"Use": "0",
"Iface": "ens33"
}
]
"""
def parse(data):
# code adapted from Conor Heine at:
# https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501
cleandata = data.splitlines()[1:]
headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h]
raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:])
return [dict(zip(headers, r)) for r in raw_data]

2
setup.py
View File

@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
setuptools.setup( setuptools.setup(
name='jc', name='jc',
version='0.2.0', version='0.5.5',
author='Kelly Brazil', author='Kelly Brazil',
author_email='kellyjonbrazil@gmail.com', author_email='kellyjonbrazil@gmail.com',
description='This tool serializes the output of popular command line tools to structured JSON output.', description='This tool serializes the output of popular command line tools to structured JSON output.',