diff --git a/README.md b/README.md index 794dd804..719887b5 100755 --- a/README.md +++ b/README.md @@ -6,22 +6,22 @@ JSON CLI output utility This allows further command line processing of output with tools like `jq` simply by piping commands: ``` -$ ls -l /usr/bin | jc --ls | jq '.[] | select(.bytes > 50000000)' +$ ls -l /usr/bin | jc --ls | jq '.[] | select(.size|tonumber > 50000000)' { "filename": "emacs", "flags": "-r-xr-xr-x", "links": 1, "owner": "root", "group": "wheel", - "bytes": 117164432, + "size": "117164432", "date": "May 3 22:26" } ``` -The `jc` parsers can also be used as python modules: +The `jc` parsers can also be used as python modules. In this case the output will be a python dictionary instead of JSON: ``` >>> import jc.parsers.ls ->>> +>>> >>> data='''-rwxr-xr-x 1 root wheel 23648 May 3 22:26 cat ... -rwxr-xr-x 1 root wheel 30016 May 3 22:26 chmod ... -rwxr-xr-x 1 root wheel 29024 May 3 22:26 cp @@ -30,22 +30,12 @@ The `jc` parsers can also be used as python modules: ... -rwxr-xr-x 1 root wheel 32000 May 3 22:26 dd ... -rwxr-xr-x 1 root wheel 23392 May 3 22:26 df ... -rwxr-xr-x 1 root wheel 18128 May 3 22:26 echo''' ->>> +>>> >>> jc.parsers.ls.parse(data) -[{'filename': 'cat', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', -'bytes': 23648, 'date': 'May 3 22:26'}, {'filename': 'chmod', 'flags': '-rwxr-xr-x', 'links': 1, -'owner': 'root', 'group': 'wheel', 'bytes': 30016, 'date': 'May 3 22:26'}, {'filename': 'cp', -'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'bytes': 29024, -'date': 'May 3 22:26'}, {'filename': 'csh', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', -'group': 'wheel', 'bytes': 375824, 'date': 'May 3 22:26'}, {'filename': 'date', -'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'bytes': 28608, -'date': 'May 3 22:26'}, {'filename': 'dd', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', -'group': 'wheel', 'bytes': 32000, 'date': 'May 3 22:26'}, {'filename': 'df', 'flags': '-rwxr-xr-x', -'links': 1, 'owner': 'root', 'group': 'wheel', 'bytes': 23392, 'date': 'May 3 22:26'}, -{'filename': 'echo', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', -'bytes': 18128, 'date': 'May 3 22:26'}] +[{'filename': 'cat', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '23648', 'date': 'May 3 22:26'}, {'filename': 'chmod', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '30016', 'date': 'May 3 22:26'}, {'filename': 'cp', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '29024', 'date': 'May 3 22:26'}, {'filename': 'csh', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '375824', 'date': 'May 3 22:26'}, {'filename': 'date', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '28608', 'date': 'May 3 22:26'}, {'filename': 'dd', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '32000', 'date': 'May 3 22:26'}, {'filename': 'df', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '23392', 'date': 'May 3 22:26'}, {'filename': 'echo', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '18128', 'date': 'May 3 22:26'}] ``` -In this case the output will be a python dictionary instead of JSON. + +The goal is to keep the resulting JSON as flat and simple as possible. Also, keys have been converted to lowercase and special characters are replaced whenever possible. Numbers are kept as strings because, depending on context or the output options, numbers can sometimes turn into strings. (e.g 'human readable' options) ## Installation ``` @@ -577,69 +567,92 @@ $ sudo iptables -vnL -t filter | jc --iptables -p $ jobs -l | jc --jobs -p [ { - "job_number": 1, - "pid": 14798, + "job_number": "1", + "pid": "19510", "status": "Running", - "command": "sleep 10000 &" + "command": "sleep 1000 &" }, { - "job_number": 2, - "pid": 14799, + "job_number": "2", + "pid": "19511", "status": "Running", - "command": "sleep 10001 &" + "command": "sleep 1001 &" }, { - "job_number": 3, - "pid": 14800, - "status": "Running", - "command": "sleep 10002 &" - }, - { - "job_number": 4, - "pid": 14814, + "job_number": "3", + "pid": "19512", "history": "previous", "status": "Running", - "command": "sleep 10003 &" + "command": "sleep 1002 &" }, { - "job_number": 5, - "pid": 14815, + "job_number": "4", + "pid": "19513", "history": "current", "status": "Running", - "command": "sleep 10004 &" + "command": "sleep 1003 &" } ] ``` ### ls ``` -$ ls -l /bin | jc --ls -p +$ ls -l /usr/bin | jc --ls -p [ + { + "filename": "apropos", + "link_to": "whatis", + "flags": "lrwxrwxrwx.", + "links": "1", + "owner": "root", + "group": "root", + "size": "6", + "date": "Aug 15 10:53" + }, + { + "filename": "arch", + "flags": "-rwxr-xr-x.", + "links": "1", + "owner": "root", + "group": "root", + "size": "33080", + "date": "Aug 19 23:25" + }, + { + "filename": "awk", + "link_to": "gawk", + "flags": "lrwxrwxrwx.", + "links": "1", + "owner": "root", + "group": "root", + "size": "4", + "date": "Aug 15 10:53" + }, + { + "filename": "base64", + "flags": "-rwxr-xr-x.", + "links": "1", + "owner": "root", + "group": "root", + "size": "37360", + "date": "Aug 19 23:25" + }, + { + "filename": "basename", + "flags": "-rwxr-xr-x.", + "links": "1", + "owner": "root", + "group": "root", + "size": "29032", + "date": "Aug 19 23:25" + }, { "filename": "bash", - "flags": "-r-xr-xr-x", - "links": 1, + "flags": "-rwxr-xr-x.", + "links": "1", "owner": "root", - "group": "wheel", - "bytes": 618416, - "date": "May 3 22:26" - }, - { - "filename": "cat", - "flags": "-rwxr-xr-x", - "links": 1, - "owner": "root", - "group": "wheel", - "bytes": 23648, - "date": "May 3 22:26" - }, - { - "filename": "chmod", - "flags": "-rwxr-xr-x", - "links": 1, - "owner": "root", - "group": "wheel", - "bytes": 30016, - "date": "May 3 22:26" + "group": "root", + "size": "964600", + "date": "Aug 8 05:06" }, ... ] @@ -875,8 +888,8 @@ $ netstat -p | jc --netstat -p "state": "ESTABLISHED", "pid": 53550, "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", @@ -888,39 +901,26 @@ $ netstat -p | jc --netstat -p "state": "ESTABLISHED", "pid": 53550, "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" } ] ``` ``` -$ netstat -lpn | jc --netstat -p +$ sudo netstat -lpn | jc --netstat -p [ { "transport_protocol": "tcp", "network_protocol": "ipv4", "local_address": "127.0.0.1", - "local_port": "42351", + "local_port": "25", "foreign_address": "0.0.0.0", "foreign_port": "*", "state": "LISTEN", - "pid": 1112, - "program_name": "containerd", - "receive_q": 0, - "send_q": 0 - }, - { - "transport_protocol": "tcp", - "network_protocol": "ipv4", - "local_address": "127.0.0.53", - "local_port": "53", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "state": "LISTEN", - "pid": 885, - "program_name": "systemd-resolve", - "receive_q": 0, - "send_q": 0 + "pid": "1584", + "program_name": "master", + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", @@ -930,48 +930,37 @@ $ netstat -lpn | jc --netstat -p "foreign_address": "0.0.0.0", "foreign_port": "*", "state": "LISTEN", - "pid": 1127, + "pid": "1213", "program_name": "sshd", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", "network_protocol": "ipv6", - "local_address": "::", - "local_port": "22", + "local_address": "::1", + "local_port": "25", "foreign_address": "::", "foreign_port": "*", "state": "LISTEN", - "pid": 1127, - "program_name": "sshd", - "receive_q": 0, - "send_q": 0 + "pid": "1584", + "program_name": "master", + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "udp", "network_protocol": "ipv4", - "local_address": "127.0.0.53", - "local_port": "53", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "pid": 885, - "program_name": "systemd-resolve", - "receive_q": 0, - "send_q": 0 - }, - { - "transport_protocol": "udp", - "network_protocol": "ipv4", - "local_address": "192.168.71.131", + "local_address": "0.0.0.0", "local_port": "68", "foreign_address": "0.0.0.0", "foreign_port": "*", - "pid": 867, - "program_name": "systemd-network", - "receive_q": 0, - "send_q": 0 - } + "pid": "19177", + "program_name": "dhclient", + "receive_q": "0", + "send_q": "0" + }, + ... ] ``` ### ps diff --git a/changelog.txt b/changelog.txt index 250a6382..7348c302 100644 --- a/changelog.txt +++ b/changelog.txt @@ -7,6 +7,7 @@ jc changelog - Flatten env parser output - Remove problematic characters from key names in: df, free, history, lsblk, lsof, and w - Where possible, lowercase all keys (except cases like env where the key is the variable name) +- Remove integer values 20191023 v0.9.1 - Add jobs parser diff --git a/jc/parsers/jobs.py b/jc/parsers/jobs.py index bc94b71d..a7f8f679 100644 --- a/jc/parsers/jobs.py +++ b/jc/parsers/jobs.py @@ -95,9 +95,9 @@ def parse(data): parsed_line[0] = parsed_line[0].lstrip('[').rstrip(']') # create list of dictionaries - output_line['job_number'] = int(parsed_line[0]) + output_line['job_number'] = parsed_line[0] if pid: - output_line['pid'] = int(pid) + output_line['pid'] = pid if job_history: output_line['history'] = job_history output_line['status'] = parsed_line[1] diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index 318153aa..3e509515 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -5,81 +5,96 @@ Usage: ls options supported: - None - - l - - a + - lah Examples: -$ ls -a /usr/bin | jc --ls -p +$ ls /usr/bin | jc --ls -p [ { - "filename": "." + "filename": "apropos" }, { - "filename": ".." + "filename": "arch" }, { - "filename": "2to3-" + "filename": "awk" }, { - "filename": "2to3-2.7" - }, - { - "filename": "AssetCacheLocatorUtil" + "filename": "base64" }, ... ] -$ ls -al /usr/bin | jc --ls -p +$ ls -l /usr/bin | jc --ls -p [ { - "filename": ".", - "flags": "drwxr-xr-x", - "links": 970, + "filename": "apropos", + "link_to": "whatis", + "flags": "lrwxrwxrwx.", + "links": "1", "owner": "root", - "group": "wheel", - "bytes": 31040, - "date": "Aug 27 21:20" + "group": "root", + "size": "6", + "date": "Aug 15 10:53" }, { - "filename": "..", - "flags": "drwxr-xr-x@", - "links": 9, + "filename": "arch", + "flags": "-rwxr-xr-x.", + "links": "1", "owner": "root", - "group": "wheel", - "bytes": 288, - "date": "May 3 22:14" + "group": "root", + "size": "33080", + "date": "Aug 19 23:25" }, { - "filename": "2to3-", - "flags": "-rwxr-xr-x", - "links": 4, + "filename": "awk", + "link_to": "gawk", + "flags": "lrwxrwxrwx.", + "links": "1", "owner": "root", - "group": "wheel", - "bytes": 925, - "date": "Feb 22 2019" + "group": "root", + "size": "4", + "date": "Aug 15 10:53" }, { - "filename": "2to3-2.7", - "link_to": "../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7", - "flags": "lrwxr-xr-x", - "links": 1, + "filename": "base64", + "flags": "-rwxr-xr-x.", + "links": "1", "owner": "root", - "group": "wheel", - "bytes": 74, - "date": "May 4 02:12" + "group": "root", + "size": "37360", + "date": "Aug 19 23:25" + }, + { + "filename": "basename", + "flags": "-rwxr-xr-x.", + "links": "1", + "owner": "root", + "group": "root", + "size": "29032", + "date": "Aug 19 23:25" + }, + { + "filename": "bash", + "flags": "-rwxr-xr-x.", + "links": "1", + "owner": "root", + "group": "root", + "size": "964600", + "date": "Aug 8 05:06" }, ... ] -$ $ ls -l /usr/bin | jc --ls | jq '.[] | 'select(.bytes > 50000000)' +$ ls -l /usr/bin | jc --ls | jq '.[] | select(.size|tonumber > 50000000)' { "filename": "emacs", "flags": "-r-xr-xr-x", "links": 1, "owner": "root", "group": "wheel", - "bytes": 117164432, + "size": "117164432", "date": "May 3 22:26" } """ @@ -117,10 +132,10 @@ def parse(data): output_line['link_to'] = filename_field[1] output_line['flags'] = parsed_line[0] - output_line['links'] = int(parsed_line[1]) + output_line['links'] = parsed_line[1] output_line['owner'] = parsed_line[2] output_line['group'] = parsed_line[3] - output_line['bytes'] = int(parsed_line[4]) + output_line['size'] = parsed_line[4] output_line['date'] = ' '.join(parsed_line[5:8]) output.append(output_line) else: diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index dafc2f37..54c7e9c4 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -22,8 +22,8 @@ $ netstat -p | jc --netstat -p "state": "ESTABLISHED", "pid": 53550, "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", @@ -35,38 +35,25 @@ $ netstat -p | jc --netstat -p "state": "ESTABLISHED", "pid": 53550, "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" } ] -$ netstat -lpn | jc --netstat -p +$ sudo netstat -lpn | jc --netstat -p [ { "transport_protocol": "tcp", "network_protocol": "ipv4", "local_address": "127.0.0.1", - "local_port": "42351", + "local_port": "25", "foreign_address": "0.0.0.0", "foreign_port": "*", "state": "LISTEN", - "pid": 1112, - "program_name": "containerd", - "receive_q": 0, - "send_q": 0 - }, - { - "transport_protocol": "tcp", - "network_protocol": "ipv4", - "local_address": "127.0.0.53", - "local_port": "53", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "state": "LISTEN", - "pid": 885, - "program_name": "systemd-resolve", - "receive_q": 0, - "send_q": 0 + "pid": "1584", + "program_name": "master", + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", @@ -76,48 +63,37 @@ $ netstat -lpn | jc --netstat -p "foreign_address": "0.0.0.0", "foreign_port": "*", "state": "LISTEN", - "pid": 1127, + "pid": "1213", "program_name": "sshd", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", "network_protocol": "ipv6", - "local_address": "::", - "local_port": "22", + "local_address": "::1", + "local_port": "25", "foreign_address": "::", "foreign_port": "*", "state": "LISTEN", - "pid": 1127, - "program_name": "sshd", - "receive_q": 0, - "send_q": 0 + "pid": "1584", + "program_name": "master", + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "udp", "network_protocol": "ipv4", - "local_address": "127.0.0.53", - "local_port": "53", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "pid": 885, - "program_name": "systemd-resolve", - "receive_q": 0, - "send_q": 0 - }, - { - "transport_protocol": "udp", - "network_protocol": "ipv4", - "local_address": "192.168.71.131", + "local_address": "0.0.0.0", "local_port": "68", "foreign_address": "0.0.0.0", "foreign_port": "*", - "pid": 867, - "program_name": "systemd-network", - "receive_q": 0, - "send_q": 0 - } + "pid": "19177", + "program_name": "dhclient", + "receive_q": "0", + "send_q": "0" + }, + ... ] """ import string @@ -161,15 +137,15 @@ def parse_line(entry): output_line['state'] = parsed_line[5] if len(parsed_line) > 6 and parsed_line[6][0] in string.digits: - output_line['pid'] = int(parsed_line[6].split('/')[0]) + output_line['pid'] = parsed_line[6].split('/')[0] output_line['program_name'] = parsed_line[6].split('/')[1] else: if parsed_line[5][0] in string.digits: - output_line['pid'] = int(parsed_line[5].split('/')[0]) + output_line['pid'] = parsed_line[5].split('/')[0] output_line['program_name'] = parsed_line[5].split('/')[1] - output_line['receive_q'] = int(parsed_line[1]) - output_line['send_q'] = int(parsed_line[2]) + output_line['receive_q'] = parsed_line[1] + output_line['send_q'] = parsed_line[2] return output_line