Merge pull request #1 from kellyjonbrazil/dev

Dev v0.5.5

README.md

@@ -3,6 +3,21 @@ JSON CLI output utility
 
 `jc` is used to JSONify the output of many standard linux cli tools for easier parsing in scripts. Parsers for `ls`, `ifconfig`, and `netstat` are currently included and more can be added via modules.
 
+This allows further command line processing of output with tools like `jq` simply by piping commands:
+
+```
+$ ls -l /usr/bin | jc --ls | jq .[] | jq 'select(.bytes > 50000000)'
+{
+  "filename": "emacs",
+  "flags": "-r-xr-xr-x",
+  "links": 1,
+  "owner": "root",
+  "group": "wheel",
+  "bytes": 117164432,
+  "date": "May 3 22:26"
+}
+```
+
 ## Installation
 ```
 $ pip3 install jc
@@ -15,10 +30,13 @@ The first argument is required and identifies the command that is piping output
 - `--ls` enables the `ls` parser
 - `--ifconfig` enables the `ifconfig` parser
 - `--netstat` enables the `netstat` parser
+- `--ps` enables the `ps` parser
+- `--route` enables the `route` parser
 
 The second `-p` argument is optional and specifies whether to pretty format the JSON output.
 
 ## Examples
+### ls
 ```
 $ ls -l /bin | jc --ls -p
 [
@@ -52,6 +70,7 @@ $ ls -l /bin | jc --ls -p
   ...
 ]
 ```
+### ifconfig
 ```
 $ ifconfig | jc --ifconfig -p
 [
@@ -135,6 +154,7 @@ $ ifconfig | jc --ifconfig -p
   }
 ]
 ```
+### netstat
 ```
 $ netstat -p | jc --netstat -p
 {
@@ -261,6 +281,103 @@ $ netstat -lp | jc --netstat -p
   }
 }
 ```
+### ps
+```
+$ ps -ef | jc --ps -p
+[
+  {
+    "UID": "root",
+    "PID": "1",
+    "PPID": "0",
+    "C": "0",
+    "STIME": "13:58",
+    "TTY": "?",
+    "TIME": "00:00:05",
+    "CMD": "/lib/systemd/systemd --system --deserialize 35"
+  },
+  {
+    "UID": "root",
+    "PID": "2",
+    "PPID": "0",
+    "C": "0",
+    "STIME": "13:58",
+    "TTY": "?",
+    "TIME": "00:00:00",
+    "CMD": "[kthreadd]"
+  },
+  {
+    "UID": "root",
+    "PID": "4",
+    "PPID": "2",
+    "C": "0",
+    "STIME": "13:58",
+    "TTY": "?",
+    "TIME": "00:00:00",
+    "CMD": "[kworker/0:0H]"
+  },
+  {
+    "UID": "root",
+    "PID": "6",
+    "PPID": "2",
+    "C": "0",
+    "STIME": "13:58",
+    "TTY": "?",
+    "TIME": "00:00:00",
+    "CMD": "[mm_percpu_wq]"
+  },
+  ...
+]
+```
+### route
+```
+$ route -n | jc --route -p
+[
+  {
+    "Destination": "0.0.0.0",
+    "Gateway": "192.168.71.2",
+    "Genmask": "0.0.0.0",
+    "Flags": "UG",
+    "Metric": "100",
+    "Ref": "0",
+    "Use": "0",
+    "Iface": "ens33"
+  },
+  {
+    "Destination": "172.17.0.0",
+    "Gateway": "0.0.0.0",
+    "Genmask": "255.255.0.0",
+    "Flags": "U",
+    "Metric": "0",
+    "Ref": "0",
+    "Use": "0",
+    "Iface": "docker0"
+  },
+  {
+    "Destination": "192.168.71.0",
+    "Gateway": "0.0.0.0",
+    "Genmask": "255.255.255.0",
+    "Flags": "U",
+    "Metric": "0",
+    "Ref": "0",
+    "Use": "0",
+    "Iface": "ens33"
+  },
+  {
+    "Destination": "192.168.71.2",
+    "Gateway": "0.0.0.0",
+    "Genmask": "255.255.255.255",
+    "Flags": "UH",
+    "Metric": "100",
+    "Ref": "0",
+    "Use": "0",
+    "Iface": "ens33"
+  }
+]
+```
 
+## Contributions
+Feel free to add/improve code or parsers!
 
-
+## Acknowledgments
+- `ifconfig-parser` module from https://github.com/KnightWhoSayNi/ifconfig-parser
+- Parsing code from Conor Heine at https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501

changelog.txt

@@ -0,0 +1,10 @@
+jc changelog
+
+20191018 v0.5.5
+- Fix netstat -p parsing for Ubuntu
+- Add ps parser
+- Add route parser
+- ls parser fixes
+
+20191017 v0.2.0
+- ifconfig, ls, and netstat support

jc/__init__.py

@@ -1,7 +1,5 @@
 """JC - JSON CLI output utility
 
-v0.1
-
 * kellyjonbrazil@gmail.com
 
 This module serializes standard unix command line output to structured JSON

jc/jc.py

@@ -9,15 +9,19 @@ import json
 import jc.parsers.ifconfig
 import jc.parsers.ls
 import jc.parsers.netstat
+import jc.parsers.ps
+import jc.parsers.route
+
 
 def main():
     pretty = False
     data = sys.stdin.read()
 
     if len(sys.argv) < 2:
-        print(f'\nError: jc\n  Must specify parser. (e.g. --ls, --netstat, --ifconfig, etc.)')
+        print('Error: jc')
+        print('  Must specify parser. (e.g. --ls, --netstat, --ifconfig, etc.)')
         print('  Use -p to pretty print')
-        print(f'\nExample: ls -al | jc --ls -p\n')
+        print('Example: ls -al | jc --ls -p\n')
         exit()
 
     arg = sys.argv[1]
@@ -32,6 +36,10 @@ def main():
         result = jc.parsers.ls.parse(data)
     elif arg == '--netstat':
         result = jc.parsers.netstat.parse(data)
+    elif arg == '--ps':
+        result = jc.parsers.ps.parse(data)
+    elif arg == '--route':
+        result = jc.parsers.route.parse(data)
 
     # output resulting dictionary as json
     if pretty:
@@ -39,5 +47,6 @@ def main():
     else:
         print(json.dumps(result))
 
+
 if __name__ == '__main__':
     main()

jc/parsers/ifconfig.py

@@ -13,6 +13,7 @@ $ ifconfig | jc --ifconfig -p
 from collections import namedtuple
 from ifconfigparser import IfconfigParser
 
+
 def parse(data):
     output = []
 
@@ -26,5 +27,3 @@ def parse(data):
         output.append(dct)
 
     return output
-
-

jc/parsers/ls.py

@@ -85,28 +85,30 @@ $ $ ls -l /usr/bin | jc --ls | jq .[] | jq 'select(.bytes > 50000000)'
 """
 import re
 
+
 def parse(data):
     output = []
 
-    cleandata = data.splitlines()
+    linedata = data.splitlines()
 
     # Delete first line if it starts with 'total'
-    if cleandata[0].find('total') == 0:
-        cleandata.pop(0)
+    if linedata:
+        if linedata[0].find('total') == 0:
+            linedata.pop(0)
 
-    # Delete last line if it is blank
-    if cleandata[-1] == '':
-        cleandata.pop(-1)
+    # Clear any blank lines
+    cleandata = list(filter(None, linedata))
 
+    if cleandata:
         # Check if -l was used to parse extra data
         if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', cleandata[0]):
             for entry in cleandata:
                 output_line = {}
 
-            parsed_line = entry.split()
+                parsed_line = entry.split(maxsplit=8)
 
                 # split filenames and links
-            filename_field = ' '.join(parsed_line[8:]).split(' -> ')
+                filename_field = parsed_line[8].split(' -> ')
 
                 # create list of dictionaries
                 output_line['filename'] = filename_field[0]

jc/parsers/netstat.py

@@ -138,6 +138,7 @@ import string
 
 output = {}
 
+
 class state():
     section = ''
     session = ''
@@ -153,6 +154,7 @@ class state():
     server_udp_ip4 = []
     server_udp_ip6 = []
 
+
 def parse_line(entry):
     parsed_line = entry.split()
     output_line = {}
@@ -164,13 +166,14 @@ def parse_line(entry):
 
     if len(parsed_line) > 5:
 
-        if parsed_line[5][0] not in string.digits:
+        if parsed_line[5][0] not in string.digits and parsed_line[5][0] != '-':
             output_line['state'] = parsed_line[5]
 
-            if len(parsed_line) > 6:
+            if len(parsed_line) > 6 and parsed_line[6][0] in string.digits:
                 output_line['pid'] = int(parsed_line[6].split('/')[0])
                 output_line['program_name'] = parsed_line[6].split('/')[1]
         else:
+            if parsed_line[5][0] in string.digits:
                 output_line['pid'] = int(parsed_line[5].split('/')[0])
                 output_line['program_name'] = parsed_line[5].split('/')[1]
 
@@ -179,17 +182,18 @@ def parse_line(entry):
 
     return output_line
 
+
 def parse(data):
     cleandata = data.splitlines()
 
     for line in cleandata:
 
         if line.find('Active Internet connections (w/o servers)') == 0:
-            state.section = "client"
+            state.section = 'client'
             continue
 
         if line.find('Active Internet connections (only servers)') == 0:
-            state.section = "server"
+            state.section = 'server'
             continue
 
         if line.find('Proto') == 0:
@@ -225,6 +229,7 @@ def parse(data):
                 else:
                     state.network = 'ipv4'
 
+        # client section
         if state.section == 'client' and state.session == 'tcp' and state.network == 'ipv4':
             state.client_tcp_ip4.append(parse_line(line))
 
@@ -237,7 +242,7 @@ def parse(data):
         if state.section == 'client' and state.session == 'udp' and state.network == 'ipv6':
             state.client_udp_ip6.append(parse_line(line))
 
-
+        # server section
         if state.section == 'server' and state.session == 'tcp' and state.network == 'ipv4':
             state.server_tcp_ip4.append(parse_line(line))
 
@@ -254,6 +259,7 @@ def parse(data):
         state.network = ''
 
     # build dictionary
+    # client section
     if state.client_tcp_ip4:
         if 'client' not in output:
             output['client'] = {}
@@ -282,7 +288,7 @@ def parse(data):
             output['client']['udp'] = {}
         output['client']['udp']['ipv6'] = state.client_udp_ip6
 
-    
+    # server section
     if state.server_tcp_ip4:
         if 'server' not in output:
             output['server'] = {}

jc/parsers/ps.py

@@ -0,0 +1,67 @@
+"""jc - JSON CLI output utility ps Parser
+
+Usage:
+    specify --ps as the first argument if the piped input is coming from ps
+
+    ps options supported:
+    - ef
+    - axu
+
+Example:
+
+$ ps -ef | jc --ps -p
+[
+  {
+    "UID": "root",
+    "PID": "1",
+    "PPID": "0",
+    "C": "0",
+    "STIME": "13:58",
+    "TTY": "?",
+    "TIME": "00:00:05",
+    "CMD": "/lib/systemd/systemd --system --deserialize 35"
+  },
+  {
+    "UID": "root",
+    "PID": "2",
+    "PPID": "0",
+    "C": "0",
+    "STIME": "13:58",
+    "TTY": "?",
+    "TIME": "00:00:00",
+    "CMD": "[kthreadd]"
+  },
+  {
+    "UID": "root",
+    "PID": "4",
+    "PPID": "2",
+    "C": "0",
+    "STIME": "13:58",
+    "TTY": "?",
+    "TIME": "00:00:00",
+    "CMD": "[kworker/0:0H]"
+  },
+  {
+    "UID": "root",
+    "PID": "6",
+    "PPID": "2",
+    "C": "0",
+    "STIME": "13:58",
+    "TTY": "?",
+    "TIME": "00:00:00",
+    "CMD": "[mm_percpu_wq]"
+  },
+  ...
+]
+"""
+
+
+def parse(data):
+
+    # code adapted from Conor Heine at:
+    # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501
+
+    cleandata = data.splitlines()
+    headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h]
+    raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:])
+    return [dict(zip(headers, r)) for r in raw_data]

jc/parsers/route.py

@@ -0,0 +1,63 @@
+"""jc - JSON CLI output utility route Parser
+
+Usage:
+    specify --route as the first argument if the piped input is coming from route
+
+
+Example:
+
+$ route -n | jc --route -p
+[
+  {
+    "Destination": "0.0.0.0",
+    "Gateway": "192.168.71.2",
+    "Genmask": "0.0.0.0",
+    "Flags": "UG",
+    "Metric": "100",
+    "Ref": "0",
+    "Use": "0",
+    "Iface": "ens33"
+  },
+  {
+    "Destination": "172.17.0.0",
+    "Gateway": "0.0.0.0",
+    "Genmask": "255.255.0.0",
+    "Flags": "U",
+    "Metric": "0",
+    "Ref": "0",
+    "Use": "0",
+    "Iface": "docker0"
+  },
+  {
+    "Destination": "192.168.71.0",
+    "Gateway": "0.0.0.0",
+    "Genmask": "255.255.255.0",
+    "Flags": "U",
+    "Metric": "0",
+    "Ref": "0",
+    "Use": "0",
+    "Iface": "ens33"
+  },
+  {
+    "Destination": "192.168.71.2",
+    "Gateway": "0.0.0.0",
+    "Genmask": "255.255.255.255",
+    "Flags": "UH",
+    "Metric": "100",
+    "Ref": "0",
+    "Use": "0",
+    "Iface": "ens33"
+  }
+]
+"""
+
+
+def parse(data):
+
+    # code adapted from Conor Heine at:
+    # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501
+
+    cleandata = data.splitlines()[1:]
+    headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h]
+    raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:])
+    return [dict(zip(headers, r)) for r in raw_data]

setup.py

@@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
 
 setuptools.setup(
     name='jc',
-    version='0.2.0',
+    version='0.5.5',
     author='Kelly Brazil',
     author_email='kellyjonbrazil@gmail.com',
     description='This tool serializes the output of popular command line tools to structured JSON output.',