From fa7466022bb8947c1bbf9f7b01aa4d92300a8992 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Oct 2019 15:54:31 -0700 Subject: [PATCH 01/33] fix env parser --- README.md | 37 +++++++++++-------------------------- changelog.txt | 3 +++ jc/parsers/env.py | 44 ++++++++++++++------------------------------ 3 files changed, 28 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 98d9a00f..068882ad 100755 --- a/README.md +++ b/README.md @@ -122,32 +122,17 @@ $ df | jc --df -p ### env ``` $ env | jc --env -p -[ - { - "TERM": "xterm-256color" - }, - { - "SHELL": "/bin/bash" - }, - { - "USER": "root" - }, - { - "PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" - }, - { - "PWD": "/bin" - }, - { - "LANG": "en_US.UTF-8" - }, - { - "HOME": "/root" - }, - { - "_": "/usr/bin/env" - } -] +{ + "TERM": "xterm-256color", + "SHELL": "/bin/bash", + "USER": "root", + "PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", + "PWD": "/root", + "LANG": "en_US.UTF-8", + "HOME": "/root", + "LOGNAME": "root", + "_": "/usr/bin/env" +} ``` ### ifconfig ``` diff --git a/changelog.txt b/changelog.txt index df94d9ca..c5ef7236 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,8 @@ jc changelog +2019xxxx v1.0.1 +- Fix env parser + 20191023 v0.9.1 - Add jobs parser - Add lsof parser diff --git a/jc/parsers/env.py b/jc/parsers/env.py index 2cebc027..c3164d5f 100644 --- a/jc/parsers/env.py +++ b/jc/parsers/env.py @@ -5,37 +5,23 @@ Usage: Example: $ env | jc --env -p -[ - { - "TERM": "xterm-256color" - }, - { - "SHELL": "/bin/bash" - }, - { - "USER": "root" - }, - { - "PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" - }, - { - "PWD": "/bin" - }, - { - "LANG": "en_US.UTF-8" - }, - { - "HOME": "/root" - }, - { - "_": "/usr/bin/env" - } -] +{ + "TERM": "xterm-256color", + "SHELL": "/bin/bash", + "USER": "root", + "PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", + "PWD": "/root", + "LANG": "en_US.UTF-8", + "HOME": "/root", + "LOGNAME": "root", + "_": "/usr/bin/env" +} + """ def parse(data): - output = [] + output = {} linedata = data.splitlines() @@ -45,9 +31,7 @@ def parse(data): if cleandata: for entry in cleandata: - output_line = {} parsed_line = entry.split('=', maxsplit=1) - output_line[parsed_line[0]] = parsed_line[1] - output.append(output_line) + output[parsed_line[0]] = parsed_line[1] return output From f101d881a16e662e883818749d48e96858fba853 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Oct 2019 16:06:55 -0700 Subject: [PATCH 02/33] add w parser --- README.md | 27 +++++++++++++++++++++++++++ jc/jc.py | 9 +++++++-- jc/parsers/w.py | 43 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 4 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 jc/parsers/w.py diff --git a/README.md b/README.md index 068882ad..5f8de637 100755 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ jc [parser] [options] - `--ps` enables the `ps` parser - `--route` enables the `route` parser - `--uname` enables the `uname -a` parser +- `--w` enables the `w` parser ### Options - `-p` specifies whether to pretty format the JSON output @@ -1042,6 +1043,32 @@ $ uname -a | jc --uname -p "kernel_version": "#74-Ubuntu SMP Tue Sep 17 17:06:04 UTC 2019" } ``` +### w +``` +$ w | jc --w -p +[ + { + "USER": "root", + "TTY": "ttyS0", + "FROM": "-", + "LOGIN@": "Mon20", + "IDLE": "2:27", + "JCPU": "10.61s", + "PCPU": "10.53s", + "WHAT": "-bash" + }, + { + "USER": "root", + "TTY": "pts/0", + "FROM": "192.168.71.1", + "LOGIN@": "22:58", + "IDLE": "2.00s", + "JCPU": "0.04s", + "PCPU": "0.00s", + "WHAT": "w" + } +] +``` ## Contributions Feel free to add/improve code or parsers! diff --git a/jc/jc.py b/jc/jc.py index f9de4665..3f77d3a2 100755 --- a/jc/jc.py +++ b/jc/jc.py @@ -21,10 +21,11 @@ import jc.parsers.netstat import jc.parsers.ps import jc.parsers.route import jc.parsers.uname +import jc.parsers.w def helptext(): - print('Usage: jc [parser] [options]\n', file=sys.stderr) + print('Usage: jc PARSER [OPTIONS]\n', file=sys.stderr) print('Parsers:', file=sys.stderr) print(' --df df parser', file=sys.stderr) print(' --env env parser', file=sys.stderr) @@ -40,7 +41,8 @@ def helptext(): print(' --netstat netstat parser', file=sys.stderr) print(' --ps ps parser', file=sys.stderr) print(' --route route parser', file=sys.stderr) - print(' --uname uname parser\n', file=sys.stderr) + print(' --uname uname parser', file=sys.stderr) + print(' --w w parser\n', file=sys.stderr) print('Options:', file=sys.stderr) print(' -p pretty print output\n', file=sys.stderr) print('Example:', file=sys.stderr) @@ -106,6 +108,9 @@ def main(): elif '--uname' in sys.argv: result = jc.parsers.uname.parse(data) + elif '--w' in sys.argv: + result = jc.parsers.w.parse(data) + else: print('jc: missing or incorrect arguments\n', file=sys.stderr) helptext() diff --git a/jc/parsers/w.py b/jc/parsers/w.py new file mode 100644 index 00000000..d84347d4 --- /dev/null +++ b/jc/parsers/w.py @@ -0,0 +1,43 @@ +"""jc - JSON CLI output utility w Parser + +Usage: + specify --w as the first argument if the piped input is coming from w + + +Example: + +$ w | jc --w -p +[ + { + "USER": "root", + "TTY": "ttyS0", + "FROM": "-", + "LOGIN@": "Mon20", + "IDLE": "2:27", + "JCPU": "10.61s", + "PCPU": "10.53s", + "WHAT": "-bash" + }, + { + "USER": "root", + "TTY": "pts/0", + "FROM": "192.168.71.1", + "LOGIN@": "22:58", + "IDLE": "2.00s", + "JCPU": "0.04s", + "PCPU": "0.00s", + "WHAT": "w" + } +] +""" + + +def parse(data): + + # code adapted from Conor Heine at: + # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 + + cleandata = data.splitlines()[1:] + headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) + return [dict(zip(headers, r)) for r in raw_data] diff --git a/setup.py b/setup.py index f9d24df8..8313ddf2 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='0.9.1', + version='1.0.1', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='This tool serializes the output of popular command line tools to structured JSON output.', From 41cd489c34502c42602bf64086670ba3b2b8e14b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Oct 2019 17:09:32 -0700 Subject: [PATCH 03/33] add history and uptime parsers --- README.md | 46 +++++++++++++++++++++++++++++++++++++++++++ changelog.txt | 3 +++ jc/jc.py | 10 ++++++++++ jc/parsers/history.py | 32 ++++++++++++++++++++++++++++++ jc/parsers/uptime.py | 35 ++++++++++++++++++++++++++++++++ 5 files changed, 126 insertions(+) create mode 100644 jc/parsers/history.py create mode 100644 jc/parsers/uptime.py diff --git a/README.md b/README.md index 5f8de637..2242d0e6 100755 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ jc [parser] [options] - `--df` enables the `df` parser - `--env` enables the `env` parser - `--free` enables the `free` parser +- `--history` enables the `history` parser - `--ifconfig` enables the `ifconfig` parser - `--iptables` enables the `iptables` parser - `--jobs` enables the `jobs` parser @@ -75,6 +76,7 @@ jc [parser] [options] - `--ps` enables the `ps` parser - `--route` enables the `route` parser - `--uname` enables the `uname -a` parser +- `--uptime` enables the `uptime` parser - `--w` enables the `w` parser ### Options @@ -135,6 +137,38 @@ $ env | jc --env -p "_": "/usr/bin/env" } ``` +### free +``` +$ free | jc --free -p +[ + { + "type": "Mem", + "total": "2017300", + "used": "213104", + "free": "1148452", + "shared": "1176", + "buff/cache": "655744", + "available": "1622204" + }, + { + "type": "Swap", + "total": "2097148", + "used": "0", + "free": "2097148" + } +] +``` +### history +``` +$ history | jc --history -p +{ + "118": "sleep 100", + "119": "ls /bin", + "120": "echo \"hello\"", + "121": "docker images", + ... +} +``` ### ifconfig ``` $ ifconfig | jc --ifconfig -p @@ -1043,6 +1077,18 @@ $ uname -a | jc --uname -p "kernel_version": "#74-Ubuntu SMP Tue Sep 17 17:06:04 UTC 2019" } ``` +### uptime +``` +$ uptime | jc --uptime -p +{ + "time": "16:52", + "uptime": "3 days, 4:49", + "users": "5", + "load_1m": "1.85", + "load_5m": "1.90", + "load_15m": "1.91" +} +``` ### w ``` $ w | jc --w -p diff --git a/changelog.txt b/changelog.txt index c5ef7236..727baabd 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,6 +1,9 @@ jc changelog 2019xxxx v1.0.1 +- Add w parser +- Add uptime parser +- Add free parser - Fix env parser 20191023 v0.9.1 diff --git a/jc/jc.py b/jc/jc.py index 3f77d3a2..be169920 100755 --- a/jc/jc.py +++ b/jc/jc.py @@ -9,6 +9,7 @@ import json import jc.parsers.df import jc.parsers.env import jc.parsers.free +import jc.parsers.history import jc.parsers.ifconfig import jc.parsers.iptables import jc.parsers.jobs @@ -21,6 +22,7 @@ import jc.parsers.netstat import jc.parsers.ps import jc.parsers.route import jc.parsers.uname +import jc.parsers.uptime import jc.parsers.w @@ -30,6 +32,7 @@ def helptext(): print(' --df df parser', file=sys.stderr) print(' --env env parser', file=sys.stderr) print(' --free free parser', file=sys.stderr) + print(' --history history parser', file=sys.stderr) print(' --ifconfig iconfig parser', file=sys.stderr) print(' --iptables iptables parser', file=sys.stderr) print(' --jobs jobs parser', file=sys.stderr) @@ -42,6 +45,7 @@ def helptext(): print(' --ps ps parser', file=sys.stderr) print(' --route route parser', file=sys.stderr) print(' --uname uname parser', file=sys.stderr) + print(' --uptime uptime parser', file=sys.stderr) print(' --w w parser\n', file=sys.stderr) print('Options:', file=sys.stderr) print(' -p pretty print output\n', file=sys.stderr) @@ -72,6 +76,9 @@ def main(): elif '--free' in sys.argv: result = jc.parsers.free.parse(data) + elif '--history' in sys.argv: + result = jc.parsers.history.parse(data) + elif '--ifconfig' in sys.argv: result = jc.parsers.ifconfig.parse(data) @@ -108,6 +115,9 @@ def main(): elif '--uname' in sys.argv: result = jc.parsers.uname.parse(data) + elif '--uptime' in sys.argv: + result = jc.parsers.uptime.parse(data) + elif '--w' in sys.argv: result = jc.parsers.w.parse(data) diff --git a/jc/parsers/history.py b/jc/parsers/history.py new file mode 100644 index 00000000..2c168282 --- /dev/null +++ b/jc/parsers/history.py @@ -0,0 +1,32 @@ +"""jc - JSON CLI output utility history Parser + +Usage: + specify --history as the first argument if the piped input is coming from history + +Example: + +$ history | jc --history -p +{ + "118": "sleep 100", + "119": "ls /bin", + "120": "echo \"hello\"", + "121": "docker images", + ... +} +""" + + +def parse(data): + output = {} + + linedata = data.splitlines() + + # Clear any blank lines + cleandata = list(filter(None, linedata)) + + if cleandata: + for entry in cleandata: + parsed_line = entry.split(maxsplit=1) + output[parsed_line[0]] = parsed_line[1] + + return output diff --git a/jc/parsers/uptime.py b/jc/parsers/uptime.py new file mode 100644 index 00000000..7116021a --- /dev/null +++ b/jc/parsers/uptime.py @@ -0,0 +1,35 @@ +"""jc - JSON CLI output utility uptime Parser + +Usage: + specify --uptime as the first argument if the piped input is coming from uptime + +Example: + +$ uptime | jc --uptime -p +{ + "time": "16:52", + "uptime": "3 days, 4:49", + "users": "5", + "load_1m": "1.85", + "load_5m": "1.90", + "load_15m": "1.91" +} +""" + + +def parse(data): + output = {} + + cleandata = data.splitlines() + + if cleandata: + parsed_line = cleandata[0].split() + + output['time'] = parsed_line[0] + output['uptime'] = ' '.join(parsed_line[2:5]).rstrip(',') + output['users'] = parsed_line[5] + output['load_1m'] = parsed_line[9] + output['load_5m'] = parsed_line[10] + output['load_15m'] = parsed_line[11] + + return output From 320929bf2595026d32b80b80c3e4878db9cf083f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Oct 2019 17:11:17 -0700 Subject: [PATCH 04/33] documentation update --- jc/parsers/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/env.py b/jc/parsers/env.py index c3164d5f..98ad68f8 100644 --- a/jc/parsers/env.py +++ b/jc/parsers/env.py @@ -4,6 +4,7 @@ Usage: specify --env as the first argument if the piped input is coming from env Example: + $ env | jc --env -p { "TERM": "xterm-256color", @@ -16,7 +17,6 @@ $ env | jc --env -p "LOGNAME": "root", "_": "/usr/bin/env" } - """ From 08ec21556b553cd7e64932d68e01ba8ff3b0f210 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Oct 2019 17:12:27 -0700 Subject: [PATCH 05/33] formatting --- jc/parsers/w.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jc/parsers/w.py b/jc/parsers/w.py index d84347d4..1ed1a555 100644 --- a/jc/parsers/w.py +++ b/jc/parsers/w.py @@ -3,7 +3,6 @@ Usage: specify --w as the first argument if the piped input is coming from w - Example: $ w | jc --w -p From 85bfb688862e78d5879df6973b88ba8023fa6086 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Oct 2019 17:33:42 -0700 Subject: [PATCH 06/33] history parser fixes --- README.md | 8 ++++---- jc/parsers/history.py | 17 +++++++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 2242d0e6..96a6536f 100755 --- a/README.md +++ b/README.md @@ -162,10 +162,10 @@ $ free | jc --free -p ``` $ history | jc --history -p { - "118": "sleep 100", - "119": "ls /bin", - "120": "echo \"hello\"", - "121": "docker images", + "n118": "sleep 100", + "n119": "ls /bin", + "n120": "echo \"hello\"", + "n121": "docker images", ... } ``` diff --git a/jc/parsers/history.py b/jc/parsers/history.py index 2c168282..a2b02450 100644 --- a/jc/parsers/history.py +++ b/jc/parsers/history.py @@ -7,10 +7,10 @@ Example: $ history | jc --history -p { - "118": "sleep 100", - "119": "ls /bin", - "120": "echo \"hello\"", - "121": "docker images", + "n118": "sleep 100", + "n119": "ls /bin", + "n120": "echo \"hello\"", + "n121": "docker images", ... } """ @@ -26,7 +26,12 @@ def parse(data): if cleandata: for entry in cleandata: - parsed_line = entry.split(maxsplit=1) - output[parsed_line[0]] = parsed_line[1] + try: + parsed_line = entry.split(maxsplit=1) + # prepend a alpha character n to be more json compliant + output['n' + parsed_line[0]] = parsed_line[1] + except IndexError: + # need to catch indexerror in case there is weird input from prior commands + pass return output From 42bdc058141c4a3ac6f2c8211f9c990590b1b5cf Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Oct 2019 17:41:51 -0700 Subject: [PATCH 07/33] changelog fix --- changelog.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.txt b/changelog.txt index 727baabd..fae1a996 100644 --- a/changelog.txt +++ b/changelog.txt @@ -3,7 +3,7 @@ jc changelog 2019xxxx v1.0.1 - Add w parser - Add uptime parser -- Add free parser +- Add history parser - Fix env parser 20191023 v0.9.1 From 6d047486d9f577bc04d79af839f5eef9657a9d43 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Oct 2019 17:53:56 -0700 Subject: [PATCH 08/33] doc fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96a6536f..b48d27b7 100755 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ $ pip3 install jc ## Usage ``` -jc [parser] [options] +jc PARSER [OPTIONS] ``` `jc` accepts piped input from `STDIN` and outputs a JSON representation of the previous command's output to `STDOUT`. The JSON output can be compact or pretty formatted. From c8c5564b29588094721949b96bff1e623437835d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 09:46:03 -0700 Subject: [PATCH 09/33] change buff/cache key to buff_cache --- README.md | 2 +- jc/parsers/free.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b48d27b7..8a6862c5 100755 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ $ free | jc --free -p "used": "213104", "free": "1148452", "shared": "1176", - "buff/cache": "655744", + "buff_cache": "655744", "available": "1622204" }, { diff --git a/jc/parsers/free.py b/jc/parsers/free.py index fb206746..9f24c5a9 100644 --- a/jc/parsers/free.py +++ b/jc/parsers/free.py @@ -13,7 +13,7 @@ $ free | jc --free -p "used": "213104", "free": "1148452", "shared": "1176", - "buff/cache": "655744", + "buff_cache": "655744", "available": "1622204" }, { @@ -33,9 +33,12 @@ def parse(data): cleandata = data.splitlines() headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] - headers.insert(0, "type") + # clean up 'buff/cache' header + # even though forward slash in a key is valid json, it can make things difficult + headers = ['buff_cache' if x == 'buff/cache' else x for x in headers] + raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) output = [dict(zip(headers, r)) for r in raw_data] From 4fa88c1ba38b34f9750625458c465d66f6531bc3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 09:53:44 -0700 Subject: [PATCH 10/33] clear out non-ascii chars from data --- jc/parsers/history.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jc/parsers/history.py b/jc/parsers/history.py index a2b02450..a272c236 100644 --- a/jc/parsers/history.py +++ b/jc/parsers/history.py @@ -19,7 +19,8 @@ $ history | jc --history -p def parse(data): output = {} - linedata = data.splitlines() + # split lines and clear out any non-ascii chars + linedata = data.splitlines().encode('ascii', errors='ignore').decode() # Clear any blank lines cleandata = list(filter(None, linedata)) From b3996cb4dfed908d71b292dc0b0136c6675351e6 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 09:54:42 -0700 Subject: [PATCH 11/33] change MAJ:MIN key to MAJ_MIN --- README.md | 10 +++++----- jc/parsers/lsblk.py | 14 +++++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8a6862c5..4f67814e 100755 --- a/README.md +++ b/README.md @@ -650,7 +650,7 @@ $ lsblk | jc --lsblk -p [ { "NAME": "loop0", - "MAJ:MIN": "7:0", + "MAJ_MIN": "7:0", "RM": "0", "SIZE": "54.5M", "RO": "1", @@ -659,7 +659,7 @@ $ lsblk | jc --lsblk -p }, { "NAME": "sda", - "MAJ:MIN": "8:0", + "MAJ_MIN": "8:0", "RM": "0", "SIZE": "20G", "RO": "0", @@ -667,7 +667,7 @@ $ lsblk | jc --lsblk -p }, { "NAME": "sda1", - "MAJ:MIN": "8:1", + "MAJ_MIN": "8:1", "RM": "0", "SIZE": "1M", "RO": "0", @@ -675,7 +675,7 @@ $ lsblk | jc --lsblk -p }, { "NAME": "sda2", - "MAJ:MIN": "8:2", + "MAJ_MIN": "8:2", "RM": "0", "SIZE": "20G", "RO": "0", @@ -684,7 +684,7 @@ $ lsblk | jc --lsblk -p }, { "NAME": "sr0", - "MAJ:MIN": "11:0", + "MAJ_MIN": "11:0", "RM": "1", "SIZE": "64.8M", "RO": "0", diff --git a/jc/parsers/lsblk.py b/jc/parsers/lsblk.py index eb78975f..0d02fef3 100644 --- a/jc/parsers/lsblk.py +++ b/jc/parsers/lsblk.py @@ -9,7 +9,7 @@ $ lsblk | jc --lsblk -p [ { "NAME": "loop0", - "MAJ:MIN": "7:0", + "MAJ_MIN": "7:0", "RM": "0", "SIZE": "54.5M", "RO": "1", @@ -18,7 +18,7 @@ $ lsblk | jc --lsblk -p }, { "NAME": "sda", - "MAJ:MIN": "8:0", + "MAJ_MIN": "8:0", "RM": "0", "SIZE": "20G", "RO": "0", @@ -26,7 +26,7 @@ $ lsblk | jc --lsblk -p }, { "NAME": "sda1", - "MAJ:MIN": "8:1", + "MAJ_MIN": "8:1", "RM": "0", "SIZE": "1M", "RO": "0", @@ -34,7 +34,7 @@ $ lsblk | jc --lsblk -p }, { "NAME": "sda2", - "MAJ:MIN": "8:2", + "MAJ_MIN": "8:2", "RM": "0", "SIZE": "20G", "RO": "0", @@ -43,7 +43,7 @@ $ lsblk | jc --lsblk -p }, { "NAME": "sr0", - "MAJ:MIN": "11:0", + "MAJ_MIN": "11:0", "RM": "1", "SIZE": "64.8M", "RO": "0", @@ -61,6 +61,10 @@ def parse(data): cleandata = data.splitlines() headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + # clean up 'MAJ:MIN' header + # even though colon in a key is valid json, it can make things difficult + headers = ['MAJ_MIN' if x == 'MAJ:MIN' else x for x in headers] + raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) output = [dict(zip(headers, r)) for r in raw_data] From a07d9a0e4bb76d369b4a9bf0d77c7590dc3c2861 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 10:04:29 -0700 Subject: [PATCH 12/33] change SIZE/OFF key to SIZE_OFF --- README.md | 10 +++++----- jc/parsers/lsof.py | 25 +++++++++++++++++-------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4f67814e..4b72d834 100755 --- a/README.md +++ b/README.md @@ -758,7 +758,7 @@ $ sudo lsof | jc --lsof -p "FD": "cwd", "TYPE": "DIR", "DEVICE": "253,0", - "SIZE/OFF": "224", + "SIZE_OFF": "224", "NODE": "64", "NAME": "/" }, @@ -770,7 +770,7 @@ $ sudo lsof | jc --lsof -p "FD": "rtd", "TYPE": "DIR", "DEVICE": "253,0", - "SIZE/OFF": "224", + "SIZE_OFF": "224", "NODE": "64", "NAME": "/" }, @@ -782,7 +782,7 @@ $ sudo lsof | jc --lsof -p "FD": "txt", "TYPE": "REG", "DEVICE": "253,0", - "SIZE/OFF": "1624520", + "SIZE_OFF": "1624520", "NODE": "50360451", "NAME": "/usr/lib/systemd/systemd" }, @@ -794,7 +794,7 @@ $ sudo lsof | jc --lsof -p "FD": "mem", "TYPE": "REG", "DEVICE": "253,0", - "SIZE/OFF": "20064", + "SIZE_OFF": "20064", "NODE": "8146", "NAME": "/usr/lib64/libuuid.so.1.3.0" }, @@ -806,7 +806,7 @@ $ sudo lsof | jc --lsof -p "FD": "mem", "TYPE": "REG", "DEVICE": "253,0", - "SIZE/OFF": "265600", + "SIZE_OFF": "265600", "NODE": "8147", "NAME": "/usr/lib64/libblkid.so.1.1.0" }, diff --git a/jc/parsers/lsof.py b/jc/parsers/lsof.py index bda21b4f..382e801b 100644 --- a/jc/parsers/lsof.py +++ b/jc/parsers/lsof.py @@ -15,7 +15,7 @@ $ sudo lsof | jc --lsof -p "FD": "cwd", "TYPE": "DIR", "DEVICE": "253,0", - "SIZE/OFF": "224", + "SIZE_OFF": "224", "NODE": "64", "NAME": "/" }, @@ -27,7 +27,7 @@ $ sudo lsof | jc --lsof -p "FD": "rtd", "TYPE": "DIR", "DEVICE": "253,0", - "SIZE/OFF": "224", + "SIZE_OFF": "224", "NODE": "64", "NAME": "/" }, @@ -39,7 +39,7 @@ $ sudo lsof | jc --lsof -p "FD": "txt", "TYPE": "REG", "DEVICE": "253,0", - "SIZE/OFF": "1624520", + "SIZE_OFF": "1624520", "NODE": "50360451", "NAME": "/usr/lib/systemd/systemd" }, @@ -51,7 +51,7 @@ $ sudo lsof | jc --lsof -p "FD": "mem", "TYPE": "REG", "DEVICE": "253,0", - "SIZE/OFF": "20064", + "SIZE_OFF": "20064", "NODE": "8146", "NAME": "/usr/lib64/libuuid.so.1.3.0" }, @@ -63,13 +63,14 @@ $ sudo lsof | jc --lsof -p "FD": "mem", "TYPE": "REG", "DEVICE": "253,0", - "SIZE/OFF": "265600", + "SIZE_OFF": "265600", "NODE": "8147", "NAME": "/usr/lib64/libblkid.so.1.1.0" }, ... ] """ +import string def parse(data): @@ -85,6 +86,9 @@ def parse(data): # find column value of last character of each header header_row = cleandata.pop(0) headers = header_row.split() + # clean up 'SIZE/OFF' header + # even though forward slash in a key is valid json, it can make things difficult + headers = ['SIZE_OFF' if x == 'SIZE/OFF' else x for x in headers] header_spec = [] for i, h in enumerate(headers): @@ -99,10 +103,15 @@ def parse(data): temp_line = entry.split(maxsplit=len(headers) - 1) for spec in header_spec: - if spec[1] == 'COMMAND' or spec[1] == 'NAME': + + index = spec[0] + header_name = spec[1] + col = spec[2] + + if header_name == 'COMMAND' or header_name == 'NAME': continue - if entry[spec[2] - 1] == ' ': - temp_line.insert(spec[0], None) + if entry[col - 1] == string.whitespace: + temp_line.insert(index, None) name = ' '.join(temp_line[9:]) fixed_line = temp_line[0:9] From 57d0ab2ed7f444862546da17cbbe8f8ce67bca8c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 10:22:10 -0700 Subject: [PATCH 13/33] change LOGIN@ to LOGIN_AT --- README.md | 4 ++-- jc/parsers/w.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4b72d834..d448f9d4 100755 --- a/README.md +++ b/README.md @@ -1097,7 +1097,7 @@ $ w | jc --w -p "USER": "root", "TTY": "ttyS0", "FROM": "-", - "LOGIN@": "Mon20", + "LOGIN_AT": "Mon20", "IDLE": "2:27", "JCPU": "10.61s", "PCPU": "10.53s", @@ -1107,7 +1107,7 @@ $ w | jc --w -p "USER": "root", "TTY": "pts/0", "FROM": "192.168.71.1", - "LOGIN@": "22:58", + "LOGIN_AT": "22:58", "IDLE": "2.00s", "JCPU": "0.04s", "PCPU": "0.00s", diff --git a/jc/parsers/w.py b/jc/parsers/w.py index 1ed1a555..ec35d374 100644 --- a/jc/parsers/w.py +++ b/jc/parsers/w.py @@ -11,7 +11,7 @@ $ w | jc --w -p "USER": "root", "TTY": "ttyS0", "FROM": "-", - "LOGIN@": "Mon20", + "LOGIN_AT": "Mon20", "IDLE": "2:27", "JCPU": "10.61s", "PCPU": "10.53s", @@ -21,7 +21,7 @@ $ w | jc --w -p "USER": "root", "TTY": "pts/0", "FROM": "192.168.71.1", - "LOGIN@": "22:58", + "LOGIN_AT": "22:58", "IDLE": "2.00s", "JCPU": "0.04s", "PCPU": "0.00s", @@ -38,5 +38,10 @@ def parse(data): cleandata = data.splitlines()[1:] headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + + # clean up 'LOGIN@' header + # even though @ in a key is valid json, it can make things difficult + headers = ['LOGIN_AT' if x == 'LOGIN@' else x for x in headers] + raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) return [dict(zip(headers, r)) for r in raw_data] From 0897c96ef3c180a1707e8f56c545f59b3a4e0672 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 10:22:23 -0700 Subject: [PATCH 14/33] formatting --- jc/parsers/lsof.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jc/parsers/lsof.py b/jc/parsers/lsof.py index 382e801b..e1dedd5d 100644 --- a/jc/parsers/lsof.py +++ b/jc/parsers/lsof.py @@ -86,9 +86,11 @@ def parse(data): # find column value of last character of each header header_row = cleandata.pop(0) headers = header_row.split() + # clean up 'SIZE/OFF' header # even though forward slash in a key is valid json, it can make things difficult headers = ['SIZE_OFF' if x == 'SIZE/OFF' else x for x in headers] + header_spec = [] for i, h in enumerate(headers): From 8c7b3193d131411f8766508fc27b2a6589bba1f2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 10:28:19 -0700 Subject: [PATCH 15/33] documentation change --- jc/parsers/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/history.py b/jc/parsers/history.py index a272c236..33dd19b8 100644 --- a/jc/parsers/history.py +++ b/jc/parsers/history.py @@ -29,7 +29,7 @@ def parse(data): for entry in cleandata: try: parsed_line = entry.split(maxsplit=1) - # prepend a alpha character n to be more json compliant + # prepend alpha character n to key so the resulting JSON is easier to work with output['n' + parsed_line[0]] = parsed_line[1] except IndexError: # need to catch indexerror in case there is weird input from prior commands From 5b532b9b71fdb9de575da0af9b7989ac537877c4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 10:31:03 -0700 Subject: [PATCH 16/33] minor cleanup --- jc/parsers/lsof.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jc/parsers/lsof.py b/jc/parsers/lsof.py index e1dedd5d..da2243b5 100644 --- a/jc/parsers/lsof.py +++ b/jc/parsers/lsof.py @@ -92,7 +92,6 @@ def parse(data): headers = ['SIZE_OFF' if x == 'SIZE/OFF' else x for x in headers] header_spec = [] - for i, h in enumerate(headers): # header tuple is (index, header_name, col) header_spec.append((i, h, header_row.find(h) + len(h))) @@ -108,11 +107,11 @@ def parse(data): index = spec[0] header_name = spec[1] - col = spec[2] + col = spec[2] - 1 if header_name == 'COMMAND' or header_name == 'NAME': continue - if entry[col - 1] == string.whitespace: + if entry[col] == string.whitespace: temp_line.insert(index, None) name = ' '.join(temp_line[9:]) From 75c084153845757e22c149ea4ae7909d42ec7118 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 10:32:54 -0700 Subject: [PATCH 17/33] changelog update --- changelog.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.txt b/changelog.txt index fae1a996..7c92e80c 100644 --- a/changelog.txt +++ b/changelog.txt @@ -4,7 +4,8 @@ jc changelog - Add w parser - Add uptime parser - Add history parser -- Fix env parser +- Flatten env parser output +- Remove problematic characters from key names in: free, history, lsblk, lsof, and w 20191023 v0.9.1 - Add jobs parser From 25b90546c652cafa9409f02bae1654cb523add88 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 10:39:05 -0700 Subject: [PATCH 18/33] change 'Use%' to 'Use_percent' --- README.md | 8 ++++---- changelog.txt | 2 +- jc/parsers/df.py | 13 +++++++++---- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d448f9d4..9a75a4bd 100755 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ $ df | jc --df -p "1K-blocks": "977500", "Used": "0", "Available": "977500", - "Use%": "0%", + "Use_percent": "0%", "Mounted": "/dev" }, { @@ -100,7 +100,7 @@ $ df | jc --df -p "1K-blocks": "201732", "Used": "1180", "Available": "200552", - "Use%": "1%", + "Use_percent": "1%", "Mounted": "/run" }, { @@ -108,7 +108,7 @@ $ df | jc --df -p "1K-blocks": "20508240", "Used": "5747284", "Available": "13696152", - "Use%": "30%", + "Use_percent": "30%", "Mounted": "/" }, { @@ -116,7 +116,7 @@ $ df | jc --df -p "1K-blocks": "1008648", "Used": "0", "Available": "1008648", - "Use%": "0%", + "Use_percent": "0%", "Mounted": "/dev/shm" }, ... diff --git a/changelog.txt b/changelog.txt index 7c92e80c..079e487f 100644 --- a/changelog.txt +++ b/changelog.txt @@ -5,7 +5,7 @@ jc changelog - Add uptime parser - Add history parser - Flatten env parser output -- Remove problematic characters from key names in: free, history, lsblk, lsof, and w +- Remove problematic characters from key names in: df, free, history, lsblk, lsof, and w 20191023 v0.9.1 - Add jobs parser diff --git a/jc/parsers/df.py b/jc/parsers/df.py index 32a9af01..7d04a68d 100644 --- a/jc/parsers/df.py +++ b/jc/parsers/df.py @@ -12,7 +12,7 @@ $ df | jc --df -p "1K-blocks": "977500", "Used": "0", "Available": "977500", - "Use%": "0%", + "Use_percent": "0%", "Mounted": "/dev" }, { @@ -20,7 +20,7 @@ $ df | jc --df -p "1K-blocks": "201732", "Used": "1180", "Available": "200552", - "Use%": "1%", + "Use_percent": "1%", "Mounted": "/run" }, { @@ -28,7 +28,7 @@ $ df | jc --df -p "1K-blocks": "20508240", "Used": "5747284", "Available": "13696152", - "Use%": "30%", + "Use_percent": "30%", "Mounted": "/" }, { @@ -36,7 +36,7 @@ $ df | jc --df -p "1K-blocks": "1008648", "Used": "0", "Available": "1008648", - "Use%": "0%", + "Use_percent": "0%", "Mounted": "/dev/shm" }, ... @@ -51,5 +51,10 @@ def parse(data): cleandata = data.splitlines() headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + + # clean up 'Use%' header + # even though % in a key is valid json, it can make things difficult + headers = ['Use_percent' if x == 'Use%' else x for x in headers] + raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) return [dict(zip(headers, r)) for r in raw_data] From 5f8e70d73054f1a106c0e75eee621bc0cefd1c6b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 10:55:09 -0700 Subject: [PATCH 19/33] convert headers to lowercase --- changelog.txt | 1 + jc/parsers/df.py | 4 ++-- jc/parsers/free.py | 2 +- jc/parsers/lsblk.py | 8 ++++---- jc/parsers/lsmod.py | 9 +++------ jc/parsers/lsof.py | 8 ++++---- jc/parsers/ps.py | 2 +- jc/parsers/route.py | 2 +- jc/parsers/w.py | 6 +++--- 9 files changed, 20 insertions(+), 22 deletions(-) diff --git a/changelog.txt b/changelog.txt index 079e487f..250a6382 100644 --- a/changelog.txt +++ b/changelog.txt @@ -6,6 +6,7 @@ jc changelog - Add history parser - Flatten env parser output - Remove problematic characters from key names in: df, free, history, lsblk, lsof, and w +- Where possible, lowercase all keys (except cases like env where the key is the variable name) 20191023 v0.9.1 - Add jobs parser diff --git a/jc/parsers/df.py b/jc/parsers/df.py index 7d04a68d..3bb0a60d 100644 --- a/jc/parsers/df.py +++ b/jc/parsers/df.py @@ -50,11 +50,11 @@ def parse(data): # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 cleandata = data.splitlines() - headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + headers = [h for h in ' '.join(cleandata[0].lower().strip().split()).split() if h] # clean up 'Use%' header # even though % in a key is valid json, it can make things difficult - headers = ['Use_percent' if x == 'Use%' else x for x in headers] + headers = ['use_percent' if x == 'use%' else x for x in headers] raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) return [dict(zip(headers, r)) for r in raw_data] diff --git a/jc/parsers/free.py b/jc/parsers/free.py index 9f24c5a9..5d775a3d 100644 --- a/jc/parsers/free.py +++ b/jc/parsers/free.py @@ -32,7 +32,7 @@ def parse(data): # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 cleandata = data.splitlines() - headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + headers = [h for h in ' '.join(cleandata[0].lower().strip().split()).split() if h] headers.insert(0, "type") # clean up 'buff/cache' header diff --git a/jc/parsers/lsblk.py b/jc/parsers/lsblk.py index 0d02fef3..3b2e1423 100644 --- a/jc/parsers/lsblk.py +++ b/jc/parsers/lsblk.py @@ -59,16 +59,16 @@ def parse(data): # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 cleandata = data.splitlines() - headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + headers = [h for h in ' '.join(cleandata[0].lower().strip().split()).split() if h] - # clean up 'MAJ:MIN' header + # clean up 'maj:min' header # even though colon in a key is valid json, it can make things difficult - headers = ['MAJ_MIN' if x == 'MAJ:MIN' else x for x in headers] + headers = ['maj_min' if x == 'maj:min' else x for x in headers] raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) output = [dict(zip(headers, r)) for r in raw_data] for entry in output: - entry['NAME'] = entry['NAME'].encode('ascii', errors='ignore').decode() + entry['name'] = entry['name'].encode('ascii', errors='ignore').decode() return output diff --git a/jc/parsers/lsmod.py b/jc/parsers/lsmod.py index 625e38cd..14fde7c9 100644 --- a/jc/parsers/lsmod.py +++ b/jc/parsers/lsmod.py @@ -65,16 +65,13 @@ def parse(data): # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 cleandata = data.splitlines() - headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] - - headers.pop(-1) - headers.append('By') + headers = [h for h in ' '.join(cleandata[0].lower().strip().split()).split() if h] raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) output = [dict(zip(headers, r)) for r in raw_data] for mod in output: - if 'By' in mod: - mod['By'] = mod['By'].split(',') + if 'by' in mod: + mod['by'] = mod['by'].split(',') return output diff --git a/jc/parsers/lsof.py b/jc/parsers/lsof.py index da2243b5..60481b15 100644 --- a/jc/parsers/lsof.py +++ b/jc/parsers/lsof.py @@ -85,11 +85,11 @@ def parse(data): # find column value of last character of each header header_row = cleandata.pop(0) - headers = header_row.split() + headers = header_row.lower().split() - # clean up 'SIZE/OFF' header + # clean up 'size/off' header # even though forward slash in a key is valid json, it can make things difficult - headers = ['SIZE_OFF' if x == 'SIZE/OFF' else x for x in headers] + headers = ['size_off' if x == 'size/off' else x for x in headers] header_spec = [] for i, h in enumerate(headers): @@ -109,7 +109,7 @@ def parse(data): header_name = spec[1] col = spec[2] - 1 - if header_name == 'COMMAND' or header_name == 'NAME': + if header_name == 'command' or header_name == 'name': continue if entry[col] == string.whitespace: temp_line.insert(index, None) diff --git a/jc/parsers/ps.py b/jc/parsers/ps.py index 14fc45f0..202fe24c 100644 --- a/jc/parsers/ps.py +++ b/jc/parsers/ps.py @@ -62,6 +62,6 @@ def parse(data): # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 cleandata = data.splitlines() - headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + headers = [h for h in ' '.join(cleandata[0].lower().strip().split()).split() if h] raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) return [dict(zip(headers, r)) for r in raw_data] diff --git a/jc/parsers/route.py b/jc/parsers/route.py index 46c8f2dd..0ac1a5c9 100644 --- a/jc/parsers/route.py +++ b/jc/parsers/route.py @@ -58,6 +58,6 @@ def parse(data): # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 cleandata = data.splitlines()[1:] - headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + headers = [h for h in ' '.join(cleandata[0].lower().strip().split()).split() if h] raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) return [dict(zip(headers, r)) for r in raw_data] diff --git a/jc/parsers/w.py b/jc/parsers/w.py index ec35d374..9ecbe157 100644 --- a/jc/parsers/w.py +++ b/jc/parsers/w.py @@ -37,11 +37,11 @@ def parse(data): # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 cleandata = data.splitlines()[1:] - headers = [h for h in ' '.join(cleandata[0].strip().split()).split() if h] + headers = [h for h in ' '.join(cleandata[0].lower().strip().split()).split() if h] - # clean up 'LOGIN@' header + # clean up 'login@' header # even though @ in a key is valid json, it can make things difficult - headers = ['LOGIN_AT' if x == 'LOGIN@' else x for x in headers] + headers = ['login_at' if x == 'login@' else x for x in headers] raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) return [dict(zip(headers, r)) for r in raw_data] From eda726c4a3864008e2f700374b679ae68a965ef8 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 10:55:26 -0700 Subject: [PATCH 20/33] documentation update --- jc/parsers/ifconfig.py | 81 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/jc/parsers/ifconfig.py b/jc/parsers/ifconfig.py index e716ba05..245dc038 100644 --- a/jc/parsers/ifconfig.py +++ b/jc/parsers/ifconfig.py @@ -8,7 +8,86 @@ Usage: Example: $ ifconfig | jc --ifconfig -p - +[ + { + "name": "docker0", + "flags": "4099", + "state": "UP,BROADCAST,MULTICAST", + "mtu": "1500", + "ipv4_addr": "172.17.0.1", + "ipv4_mask": "255.255.0.0", + "ipv4_bcast": "0.0.0.0", + "mac_addr": "02:42:53:18:31:cc", + "type": "Ethernet", + "rx_packets": "0", + "rx_errors": "0", + "rx_dropped": "0", + "rx_overruns": "0", + "rx_frame": "0", + "tx_packets": "0", + "tx_errors": "0", + "tx_dropped": "0", + "tx_overruns": "0", + "tx_carrier": "0", + "tx_collisions": "0", + "ipv6_addr": null, + "ipv6_mask": null, + "ipv6_scope": null, + "metric": null + }, + { + "name": "ens33", + "flags": "4163", + "state": "UP,BROADCAST,RUNNING,MULTICAST", + "mtu": "1500", + "ipv4_addr": "192.168.71.135", + "ipv4_mask": "255.255.255.0", + "ipv4_bcast": "192.168.71.255", + "ipv6_addr": "fe80::c1cb:715d:bc3e:b8a0", + "ipv6_mask": "64", + "ipv6_scope": "link", + "mac_addr": "00:0c:29:3b:58:0e", + "type": "Ethernet", + "rx_packets": "26348", + "rx_errors": "0", + "rx_dropped": "0", + "rx_overruns": "0", + "rx_frame": "0", + "tx_packets": "5308", + "tx_errors": "0", + "tx_dropped": "0", + "tx_overruns": "0", + "tx_carrier": "0", + "tx_collisions": "0", + "metric": null + }, + { + "name": "lo", + "flags": "73", + "state": "UP,LOOPBACK,RUNNING", + "mtu": "65536", + "ipv4_addr": "127.0.0.1", + "ipv4_mask": "255.0.0.0", + "ipv4_bcast": null, + "ipv6_addr": "::1", + "ipv6_mask": "128", + "ipv6_scope": "host", + "mac_addr": null, + "type": "Local Loopback", + "rx_packets": "64", + "rx_errors": "0", + "rx_dropped": "0", + "rx_overruns": "0", + "rx_frame": "0", + "tx_packets": "64", + "tx_errors": "0", + "tx_dropped": "0", + "tx_overruns": "0", + "tx_carrier": "0", + "tx_collisions": "0", + "metric": null + } +] """ from collections import namedtuple from ifconfigparser import IfconfigParser From 7e44c4278a75c7223d99200023160e8d77fde54d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 10:55:38 -0700 Subject: [PATCH 21/33] formatting --- jc/parsers/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/jobs.py b/jc/parsers/jobs.py index bc3ec39b..bc94b71d 100644 --- a/jc/parsers/jobs.py +++ b/jc/parsers/jobs.py @@ -73,7 +73,7 @@ def parse(data): remainder = parsed_line.pop(1) job_number = parsed_line.pop(0) remainder = remainder.split(maxsplit=1) - + # rebuild parsed_line parsed_line = [] From 5f1ec6734874651bee595961edf1fafaf5acf001 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 14:58:15 -0700 Subject: [PATCH 22/33] lower() headers --- README.md | 461 +++++++++++++++++++++-------------------- jc/parsers/df.py | 52 ++--- jc/parsers/history.py | 2 +- jc/parsers/iptables.py | 2 +- jc/parsers/lsblk.py | 73 ++++--- jc/parsers/lsmod.py | 69 +++--- jc/parsers/lsof.py | 113 +++++----- jc/parsers/ps.py | 81 +++++--- jc/parsers/route.py | 60 +++--- jc/parsers/w.py | 32 +-- 10 files changed, 481 insertions(+), 464 deletions(-) diff --git a/README.md b/README.md index 9a75a4bd..794dd804 100755 --- a/README.md +++ b/README.md @@ -88,37 +88,37 @@ jc PARSER [OPTIONS] $ df | jc --df -p [ { - "Filesystem": "udev", - "1K-blocks": "977500", - "Used": "0", - "Available": "977500", - "Use_percent": "0%", - "Mounted": "/dev" + "filesystem": "udev", + "1k-blocks": "977500", + "used": "0", + "available": "977500", + "use_percent": "0%", + "mounted": "/dev" }, { - "Filesystem": "tmpfs", - "1K-blocks": "201732", - "Used": "1180", - "Available": "200552", - "Use_percent": "1%", - "Mounted": "/run" + "filesystem": "tmpfs", + "1k-blocks": "201732", + "used": "1204", + "available": "200528", + "use_percent": "1%", + "mounted": "/run" }, { - "Filesystem": "/dev/sda2", - "1K-blocks": "20508240", - "Used": "5747284", - "Available": "13696152", - "Use_percent": "30%", - "Mounted": "/" + "filesystem": "/dev/sda2", + "1k-blocks": "20508240", + "used": "5748312", + "available": "13695124", + "use_percent": "30%", + "mounted": "/" }, { - "Filesystem": "tmpfs", - "1K-blocks": "1008648", - "Used": "0", - "Available": "1008648", - "Use_percent": "0%", - "Mounted": "/dev/shm" - }, + "filesystem": "tmpfs", + "1k-blocks": "1008648", + "used": "0", + "available": "1008648", + "use_percent": "0%", + "mounted": "/dev/shm" + } ... ] ``` @@ -649,46 +649,55 @@ $ ls -l /bin | jc --ls -p $ lsblk | jc --lsblk -p [ { - "NAME": "loop0", - "MAJ_MIN": "7:0", - "RM": "0", - "SIZE": "54.5M", - "RO": "1", - "TYPE": "loop", - "MOUNTPOINT": "/snap/core18/1223" + "name": "sda", + "maj_min": "8:0", + "rm": "0", + "size": "20G", + "ro": "0", + "type": "disk" }, { - "NAME": "sda", - "MAJ_MIN": "8:0", - "RM": "0", - "SIZE": "20G", - "RO": "0", - "TYPE": "disk" + "name": "sda1", + "maj_min": "8:1", + "rm": "0", + "size": "1G", + "ro": "0", + "type": "part", + "mountpoint": "/boot" }, { - "NAME": "sda1", - "MAJ_MIN": "8:1", - "RM": "0", - "SIZE": "1M", - "RO": "0", - "TYPE": "part" + "name": "sda2", + "maj_min": "8:2", + "rm": "0", + "size": "19G", + "ro": "0", + "type": "part" }, { - "NAME": "sda2", - "MAJ_MIN": "8:2", - "RM": "0", - "SIZE": "20G", - "RO": "0", - "TYPE": "part", - "MOUNTPOINT": "/" + "name": "centos-root", + "maj_min": "253:0", + "rm": "0", + "size": "17G", + "ro": "0", + "type": "lvm", + "mountpoint": "/" }, { - "NAME": "sr0", - "MAJ_MIN": "11:0", - "RM": "1", - "SIZE": "64.8M", - "RO": "0", - "TYPE": "rom" + "name": "centos-swap", + "maj_min": "253:1", + "rm": "0", + "size": "2G", + "ro": "0", + "type": "lvm", + "mountpoint": "[SWAP]" + }, + { + "name": "sr0", + "maj_min": "11:0", + "rm": "1", + "size": "1024M", + "ro": "0", + "type": "rom" } ] ``` @@ -696,44 +705,12 @@ $ lsblk | jc --lsblk -p ``` $ lsmod | jc --lsmod -p [ - { - "Module": "nf_nat_ipv4", - "Size": "14115", - "Used": "1", - "By": [ - "iptable_nat" - ] - }, + ... { - "Module": "nf_nat", - "Size": "26583", - "Used": "3", - "By": [ - "nf_nat_ipv4", - "nf_nat_ipv6", - "nf_nat_masquerade_ipv4" - ] - }, - { - "Module": "iptable_mangle", - "Size": "12695", - "Used": "1" - }, - { - "Module": "iptable_security", - "Size": "12705", - "Used": "1" - }, - { - "Module": "iptable_raw", - "Size": "12678", - "Used": "1" - }, - { - "Module": "nf_conntrack", - "Size": "139224", - "Used": "7", - "By": [ + "module": "nf_conntrack", + "size": "139224", + "used": "7", + "by": [ "nf_nat", "nf_nat_ipv4", "nf_nat_ipv6", @@ -743,72 +720,99 @@ $ lsmod | jc --lsmod -p "nf_conntrack_ipv6" ] }, + { + "module": "ip_set", + "size": "45799", + "used": "0" + }, + { + "module": "nfnetlink", + "size": "14519", + "used": "1", + "by": [ + "ip_set" + ] + }, + { + "module": "ebtable_filter", + "size": "12827", + "used": "1" + }, + { + "module": "ebtables", + "size": "35009", + "used": "2", + "by": [ + "ebtable_nat", + "ebtable_filter" + ] + }, ... ] ``` ### lsof ``` -$ sudo lsof | jc --lsof -p +$ sudo lsof | jc --lsof -p | more [ { - "COMMAND": "systemd", - "PID": "1", - "TID": null, - "USER": "root", - "FD": "cwd", - "TYPE": "DIR", - "DEVICE": "253,0", - "SIZE_OFF": "224", - "NODE": "64", - "NAME": "/" + "command": "systemd", + "pid": "1", + "tid": null, + "user": "root", + "fd": "cwd", + "type": "DIR", + "device": "8,2", + "size_off": "4096", + "node": "2", + "name": "/" }, { - "COMMAND": "systemd", - "PID": "1", - "TID": null, - "USER": "root", - "FD": "rtd", - "TYPE": "DIR", - "DEVICE": "253,0", - "SIZE_OFF": "224", - "NODE": "64", - "NAME": "/" + "command": "systemd", + "pid": "1", + "tid": null, + "user": "root", + "fd": "rtd", + "type": "DIR", + "device": "8,2", + "size_off": "4096", + "node": "2", + "name": "/" }, { - "COMMAND": "systemd", - "PID": "1", - "TID": null, - "USER": "root", - "FD": "txt", - "TYPE": "REG", - "DEVICE": "253,0", - "SIZE_OFF": "1624520", - "NODE": "50360451", - "NAME": "/usr/lib/systemd/systemd" + "command": "systemd", + "pid": "1", + "tid": null, + "user": "root", + "fd": "txt", + "type": "REG", + "device": "8,2", + "size_off": "1595792", + "node": "668802", + "name": "/lib/systemd/systemd" }, { - "COMMAND": "systemd", - "PID": "1", - "TID": null, - "USER": "root", - "FD": "mem", - "TYPE": "REG", - "DEVICE": "253,0", - "SIZE_OFF": "20064", - "NODE": "8146", - "NAME": "/usr/lib64/libuuid.so.1.3.0" + "command": "systemd", + "pid": "1", + "tid": null, + "user": "root", + "fd": "mem", + "type": "REG", + "device": "8,2", + "size_off": "1700792", + "node": "656167", + "name": "/lib/x86_64-linux-gnu/libm-2.27.so" }, { - "COMMAND": "systemd", - "PID": "1", - "TID": null, - "USER": "root", - "FD": "mem", - "TYPE": "REG", - "DEVICE": "253,0", - "SIZE_OFF": "265600", - "NODE": "8147", - "NAME": "/usr/lib64/libblkid.so.1.1.0" + "command": "systemd", + "pid": "1", + "tid": null, + "user": "root", + "fd": "mem", + "type": "REG", + "device": "8,2", + "size_off": "121016", + "node": "655394", + "name": "/lib/x86_64-linux-gnu/libudev.so.1.6.9" }, ... ] @@ -974,92 +978,93 @@ $ netstat -lpn | jc --netstat -p ``` $ ps -ef | jc --ps -p [ + ... { - "UID": "root", - "PID": "1", - "PPID": "0", - "C": "0", - "STIME": "13:58", - "TTY": "?", - "TIME": "00:00:05", - "CMD": "/lib/systemd/systemd --system --deserialize 35" + "uid": "root", + "pid": "545", + "ppid": "1", + "c": "0", + "stime": "Oct21", + "tty": "?", + "time": "00:00:03", + "cmd": "/usr/lib/systemd/systemd-journald" }, { - "UID": "root", - "PID": "2", - "PPID": "0", - "C": "0", - "STIME": "13:58", - "TTY": "?", - "TIME": "00:00:00", - "CMD": "[kthreadd]" + "uid": "root", + "pid": "566", + "ppid": "1", + "c": "0", + "stime": "Oct21", + "tty": "?", + "time": "00:00:00", + "cmd": "/usr/sbin/lvmetad -f" }, { - "UID": "root", - "PID": "4", - "PPID": "2", - "C": "0", - "STIME": "13:58", - "TTY": "?", - "TIME": "00:00:00", - "CMD": "[kworker/0:0H]" + "uid": "root", + "pid": "580", + "ppid": "1", + "c": "0", + "stime": "Oct21", + "tty": "?", + "time": "00:00:00", + "cmd": "/usr/lib/systemd/systemd-udevd" }, { - "UID": "root", - "PID": "6", - "PPID": "2", - "C": "0", - "STIME": "13:58", - "TTY": "?", - "TIME": "00:00:00", - "CMD": "[mm_percpu_wq]" + "uid": "root", + "pid": "659", + "ppid": "2", + "c": "0", + "stime": "Oct21", + "tty": "?", + "time": "00:00:00", + "cmd": "[kworker/u257:0]" + }, + { + "uid": "root", + "pid": "666", + "ppid": "2", + "c": "0", + "stime": "Oct21", + "tty": "?", + "time": "00:00:00", + "cmd": "[hci0]" }, ... ] ``` ### route ``` -$ route -n | jc --route -p +$ route | jc --route -p [ { - "Destination": "0.0.0.0", - "Gateway": "192.168.71.2", - "Genmask": "0.0.0.0", - "Flags": "UG", - "Metric": "100", - "Ref": "0", - "Use": "0", - "Iface": "ens33" + "destination": "default", + "gateway": "gateway", + "genmask": "0.0.0.0", + "flags": "UG", + "metric": "100", + "ref": "0", + "use": "0", + "iface": "ens33" }, { - "Destination": "172.17.0.0", - "Gateway": "0.0.0.0", - "Genmask": "255.255.0.0", - "Flags": "U", - "Metric": "0", - "Ref": "0", - "Use": "0", - "Iface": "docker0" + "destination": "172.17.0.0", + "gateway": "0.0.0.0", + "genmask": "255.255.0.0", + "flags": "U", + "metric": "0", + "ref": "0", + "use": "0", + "iface": "docker0" }, { - "Destination": "192.168.71.0", - "Gateway": "0.0.0.0", - "Genmask": "255.255.255.0", - "Flags": "U", - "Metric": "0", - "Ref": "0", - "Use": "0", - "Iface": "ens33" - }, - { - "Destination": "192.168.71.2", - "Gateway": "0.0.0.0", - "Genmask": "255.255.255.255", - "Flags": "UH", - "Metric": "100", - "Ref": "0", - "Use": "0", - "Iface": "ens33" + "destination": "192.168.71.0", + "gateway": "0.0.0.0", + "genmask": "255.255.255.0", + "flags": "U", + "metric": "100", + "ref": "0", + "use": "0", + "iface": "ens33" } ] ``` @@ -1094,24 +1099,24 @@ $ uptime | jc --uptime -p $ w | jc --w -p [ { - "USER": "root", - "TTY": "ttyS0", - "FROM": "-", - "LOGIN_AT": "Mon20", - "IDLE": "2:27", - "JCPU": "10.61s", - "PCPU": "10.53s", - "WHAT": "-bash" + "user": "root", + "tty": "ttyS0", + "from": "-", + "login_at": "Mon20", + "idle": "0.00s", + "jcpu": "14.70s", + "pcpu": "0.00s", + "what": "bash" }, { - "USER": "root", - "TTY": "pts/0", - "FROM": "192.168.71.1", - "LOGIN_AT": "22:58", - "IDLE": "2.00s", - "JCPU": "0.04s", - "PCPU": "0.00s", - "WHAT": "w" + "user": "root", + "tty": "pts/0", + "from": "192.168.71.1", + "login_at": "Thu22", + "idle": "22:46m", + "jcpu": "0.05s", + "pcpu": "0.05s", + "what": "-bash" } ] ``` diff --git a/jc/parsers/df.py b/jc/parsers/df.py index 3bb0a60d..08cbfcce 100644 --- a/jc/parsers/df.py +++ b/jc/parsers/df.py @@ -8,37 +8,37 @@ Example: $ df | jc --df -p [ { - "Filesystem": "udev", - "1K-blocks": "977500", - "Used": "0", - "Available": "977500", - "Use_percent": "0%", - "Mounted": "/dev" + "filesystem": "udev", + "1k-blocks": "977500", + "used": "0", + "available": "977500", + "use_percent": "0%", + "mounted": "/dev" }, { - "Filesystem": "tmpfs", - "1K-blocks": "201732", - "Used": "1180", - "Available": "200552", - "Use_percent": "1%", - "Mounted": "/run" + "filesystem": "tmpfs", + "1k-blocks": "201732", + "used": "1204", + "available": "200528", + "use_percent": "1%", + "mounted": "/run" }, { - "Filesystem": "/dev/sda2", - "1K-blocks": "20508240", - "Used": "5747284", - "Available": "13696152", - "Use_percent": "30%", - "Mounted": "/" + "filesystem": "/dev/sda2", + "1k-blocks": "20508240", + "used": "5748312", + "available": "13695124", + "use_percent": "30%", + "mounted": "/" }, { - "Filesystem": "tmpfs", - "1K-blocks": "1008648", - "Used": "0", - "Available": "1008648", - "Use_percent": "0%", - "Mounted": "/dev/shm" - }, + "filesystem": "tmpfs", + "1k-blocks": "1008648", + "used": "0", + "available": "1008648", + "use_percent": "0%", + "mounted": "/dev/shm" + } ... ] """ @@ -52,7 +52,7 @@ def parse(data): cleandata = data.splitlines() headers = [h for h in ' '.join(cleandata[0].lower().strip().split()).split() if h] - # clean up 'Use%' header + # clean up 'use%' header # even though % in a key is valid json, it can make things difficult headers = ['use_percent' if x == 'use%' else x for x in headers] diff --git a/jc/parsers/history.py b/jc/parsers/history.py index 33dd19b8..bce56ca4 100644 --- a/jc/parsers/history.py +++ b/jc/parsers/history.py @@ -20,7 +20,7 @@ def parse(data): output = {} # split lines and clear out any non-ascii chars - linedata = data.splitlines().encode('ascii', errors='ignore').decode() + linedata = data.encode('ascii', errors='ignore').decode().splitlines() # Clear any blank lines cleandata = list(filter(None, linedata)) diff --git a/jc/parsers/iptables.py b/jc/parsers/iptables.py index bb0a1328..b3f801ae 100644 --- a/jc/parsers/iptables.py +++ b/jc/parsers/iptables.py @@ -348,7 +348,7 @@ def parse(data): elif line.find('target') == 0 or line.find('pkts') == 1: headers = [] - headers = [h for h in ' '.join(line.strip().split()).split() if h] + headers = [h for h in ' '.join(line.lower().strip().split()).split() if h] headers.append("options") continue diff --git a/jc/parsers/lsblk.py b/jc/parsers/lsblk.py index 3b2e1423..664eaba9 100644 --- a/jc/parsers/lsblk.py +++ b/jc/parsers/lsblk.py @@ -8,46 +8,55 @@ Example: $ lsblk | jc --lsblk -p [ { - "NAME": "loop0", - "MAJ_MIN": "7:0", - "RM": "0", - "SIZE": "54.5M", - "RO": "1", - "TYPE": "loop", - "MOUNTPOINT": "/snap/core18/1223" + "name": "sda", + "maj_min": "8:0", + "rm": "0", + "size": "20G", + "ro": "0", + "type": "disk" }, { - "NAME": "sda", - "MAJ_MIN": "8:0", - "RM": "0", - "SIZE": "20G", - "RO": "0", - "TYPE": "disk" + "name": "sda1", + "maj_min": "8:1", + "rm": "0", + "size": "1G", + "ro": "0", + "type": "part", + "mountpoint": "/boot" }, { - "NAME": "sda1", - "MAJ_MIN": "8:1", - "RM": "0", - "SIZE": "1M", - "RO": "0", - "TYPE": "part" + "name": "sda2", + "maj_min": "8:2", + "rm": "0", + "size": "19G", + "ro": "0", + "type": "part" }, { - "NAME": "sda2", - "MAJ_MIN": "8:2", - "RM": "0", - "SIZE": "20G", - "RO": "0", - "TYPE": "part", - "MOUNTPOINT": "/" + "name": "centos-root", + "maj_min": "253:0", + "rm": "0", + "size": "17G", + "ro": "0", + "type": "lvm", + "mountpoint": "/" }, { - "NAME": "sr0", - "MAJ_MIN": "11:0", - "RM": "1", - "SIZE": "64.8M", - "RO": "0", - "TYPE": "rom" + "name": "centos-swap", + "maj_min": "253:1", + "rm": "0", + "size": "2G", + "ro": "0", + "type": "lvm", + "mountpoint": "[SWAP]" + }, + { + "name": "sr0", + "maj_min": "11:0", + "rm": "1", + "size": "1024M", + "ro": "0", + "type": "rom" } ] """ diff --git a/jc/parsers/lsmod.py b/jc/parsers/lsmod.py index 14fde7c9..1ff10f6c 100644 --- a/jc/parsers/lsmod.py +++ b/jc/parsers/lsmod.py @@ -7,44 +7,12 @@ Example: $ lsmod | jc --lsmod -p [ - { - "Module": "nf_nat_ipv4", - "Size": "14115", - "Used": "1", - "By": [ - "iptable_nat" - ] - }, + ... { - "Module": "nf_nat", - "Size": "26583", - "Used": "3", - "By": [ - "nf_nat_ipv4", - "nf_nat_ipv6", - "nf_nat_masquerade_ipv4" - ] - }, - { - "Module": "iptable_mangle", - "Size": "12695", - "Used": "1" - }, - { - "Module": "iptable_security", - "Size": "12705", - "Used": "1" - }, - { - "Module": "iptable_raw", - "Size": "12678", - "Used": "1" - }, - { - "Module": "nf_conntrack", - "Size": "139224", - "Used": "7", - "By": [ + "module": "nf_conntrack", + "size": "139224", + "used": "7", + "by": [ "nf_nat", "nf_nat_ipv4", "nf_nat_ipv6", @@ -54,6 +22,33 @@ $ lsmod | jc --lsmod -p "nf_conntrack_ipv6" ] }, + { + "module": "ip_set", + "size": "45799", + "used": "0" + }, + { + "module": "nfnetlink", + "size": "14519", + "used": "1", + "by": [ + "ip_set" + ] + }, + { + "module": "ebtable_filter", + "size": "12827", + "used": "1" + }, + { + "module": "ebtables", + "size": "35009", + "used": "2", + "by": [ + "ebtable_nat", + "ebtable_filter" + ] + }, ... ] """ diff --git a/jc/parsers/lsof.py b/jc/parsers/lsof.py index 60481b15..e5636fe5 100644 --- a/jc/parsers/lsof.py +++ b/jc/parsers/lsof.py @@ -5,67 +5,67 @@ Usage: Example: -$ sudo lsof | jc --lsof -p +$ sudo lsof | jc --lsof -p | more [ { - "COMMAND": "systemd", - "PID": "1", - "TID": null, - "USER": "root", - "FD": "cwd", - "TYPE": "DIR", - "DEVICE": "253,0", - "SIZE_OFF": "224", - "NODE": "64", - "NAME": "/" + "command": "systemd", + "pid": "1", + "tid": null, + "user": "root", + "fd": "cwd", + "type": "DIR", + "device": "8,2", + "size_off": "4096", + "node": "2", + "name": "/" }, { - "COMMAND": "systemd", - "PID": "1", - "TID": null, - "USER": "root", - "FD": "rtd", - "TYPE": "DIR", - "DEVICE": "253,0", - "SIZE_OFF": "224", - "NODE": "64", - "NAME": "/" + "command": "systemd", + "pid": "1", + "tid": null, + "user": "root", + "fd": "rtd", + "type": "DIR", + "device": "8,2", + "size_off": "4096", + "node": "2", + "name": "/" }, { - "COMMAND": "systemd", - "PID": "1", - "TID": null, - "USER": "root", - "FD": "txt", - "TYPE": "REG", - "DEVICE": "253,0", - "SIZE_OFF": "1624520", - "NODE": "50360451", - "NAME": "/usr/lib/systemd/systemd" + "command": "systemd", + "pid": "1", + "tid": null, + "user": "root", + "fd": "txt", + "type": "REG", + "device": "8,2", + "size_off": "1595792", + "node": "668802", + "name": "/lib/systemd/systemd" }, { - "COMMAND": "systemd", - "PID": "1", - "TID": null, - "USER": "root", - "FD": "mem", - "TYPE": "REG", - "DEVICE": "253,0", - "SIZE_OFF": "20064", - "NODE": "8146", - "NAME": "/usr/lib64/libuuid.so.1.3.0" + "command": "systemd", + "pid": "1", + "tid": null, + "user": "root", + "fd": "mem", + "type": "REG", + "device": "8,2", + "size_off": "1700792", + "node": "656167", + "name": "/lib/x86_64-linux-gnu/libm-2.27.so" }, { - "COMMAND": "systemd", - "PID": "1", - "TID": null, - "USER": "root", - "FD": "mem", - "TYPE": "REG", - "DEVICE": "253,0", - "SIZE_OFF": "265600", - "NODE": "8147", - "NAME": "/usr/lib64/libblkid.so.1.1.0" + "command": "systemd", + "pid": "1", + "tid": null, + "user": "root", + "fd": "mem", + "type": "REG", + "device": "8,2", + "size_off": "121016", + "node": "655394", + "name": "/lib/x86_64-linux-gnu/libudev.so.1.6.9" }, ... ] @@ -84,12 +84,13 @@ def parse(data): if cleandata: # find column value of last character of each header - header_row = cleandata.pop(0) - headers = header_row.lower().split() + header_text = cleandata.pop(0).lower() # clean up 'size/off' header # even though forward slash in a key is valid json, it can make things difficult - headers = ['size_off' if x == 'size/off' else x for x in headers] + header_row = header_text.replace('size/off', 'size_off') + + headers = header_row.split() header_spec = [] for i, h in enumerate(headers): @@ -107,11 +108,11 @@ def parse(data): index = spec[0] header_name = spec[1] - col = spec[2] - 1 + col = spec[2] - 1 # subtract one since column starts at 0 instead of 1 if header_name == 'command' or header_name == 'name': continue - if entry[col] == string.whitespace: + if entry[col] in string.whitespace: temp_line.insert(index, None) name = ' '.join(temp_line[9:]) diff --git a/jc/parsers/ps.py b/jc/parsers/ps.py index 202fe24c..b536f264 100644 --- a/jc/parsers/ps.py +++ b/jc/parsers/ps.py @@ -11,45 +11,56 @@ Example: $ ps -ef | jc --ps -p [ + ... { - "UID": "root", - "PID": "1", - "PPID": "0", - "C": "0", - "STIME": "13:58", - "TTY": "?", - "TIME": "00:00:05", - "CMD": "/lib/systemd/systemd --system --deserialize 35" + "uid": "root", + "pid": "545", + "ppid": "1", + "c": "0", + "stime": "Oct21", + "tty": "?", + "time": "00:00:03", + "cmd": "/usr/lib/systemd/systemd-journald" }, { - "UID": "root", - "PID": "2", - "PPID": "0", - "C": "0", - "STIME": "13:58", - "TTY": "?", - "TIME": "00:00:00", - "CMD": "[kthreadd]" + "uid": "root", + "pid": "566", + "ppid": "1", + "c": "0", + "stime": "Oct21", + "tty": "?", + "time": "00:00:00", + "cmd": "/usr/sbin/lvmetad -f" }, { - "UID": "root", - "PID": "4", - "PPID": "2", - "C": "0", - "STIME": "13:58", - "TTY": "?", - "TIME": "00:00:00", - "CMD": "[kworker/0:0H]" + "uid": "root", + "pid": "580", + "ppid": "1", + "c": "0", + "stime": "Oct21", + "tty": "?", + "time": "00:00:00", + "cmd": "/usr/lib/systemd/systemd-udevd" }, { - "UID": "root", - "PID": "6", - "PPID": "2", - "C": "0", - "STIME": "13:58", - "TTY": "?", - "TIME": "00:00:00", - "CMD": "[mm_percpu_wq]" + "uid": "root", + "pid": "659", + "ppid": "2", + "c": "0", + "stime": "Oct21", + "tty": "?", + "time": "00:00:00", + "cmd": "[kworker/u257:0]" + }, + { + "uid": "root", + "pid": "666", + "ppid": "2", + "c": "0", + "stime": "Oct21", + "tty": "?", + "time": "00:00:00", + "cmd": "[hci0]" }, ... ] @@ -63,5 +74,11 @@ def parse(data): cleandata = data.splitlines() headers = [h for h in ' '.join(cleandata[0].lower().strip().split()).split() if h] + + # clean up '%cpu' and '%mem' headers + # even though % in a key is valid json, it can make things difficult + headers = ['cpu_percent' if x == '%cpu' else x for x in headers] + headers = ['mem_percent' if x == '%mem' else x for x in headers] + raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), cleandata[1:]) return [dict(zip(headers, r)) for r in raw_data] diff --git a/jc/parsers/route.py b/jc/parsers/route.py index 0ac1a5c9..80bcf6d8 100644 --- a/jc/parsers/route.py +++ b/jc/parsers/route.py @@ -6,47 +6,37 @@ Usage: Example: -$ route -n | jc --route -p +$ route | jc --route -p [ { - "Destination": "0.0.0.0", - "Gateway": "192.168.71.2", - "Genmask": "0.0.0.0", - "Flags": "UG", - "Metric": "100", - "Ref": "0", - "Use": "0", - "Iface": "ens33" + "destination": "default", + "gateway": "gateway", + "genmask": "0.0.0.0", + "flags": "UG", + "metric": "100", + "ref": "0", + "use": "0", + "iface": "ens33" }, { - "Destination": "172.17.0.0", - "Gateway": "0.0.0.0", - "Genmask": "255.255.0.0", - "Flags": "U", - "Metric": "0", - "Ref": "0", - "Use": "0", - "Iface": "docker0" + "destination": "172.17.0.0", + "gateway": "0.0.0.0", + "genmask": "255.255.0.0", + "flags": "U", + "metric": "0", + "ref": "0", + "use": "0", + "iface": "docker0" }, { - "Destination": "192.168.71.0", - "Gateway": "0.0.0.0", - "Genmask": "255.255.255.0", - "Flags": "U", - "Metric": "0", - "Ref": "0", - "Use": "0", - "Iface": "ens33" - }, - { - "Destination": "192.168.71.2", - "Gateway": "0.0.0.0", - "Genmask": "255.255.255.255", - "Flags": "UH", - "Metric": "100", - "Ref": "0", - "Use": "0", - "Iface": "ens33" + "destination": "192.168.71.0", + "gateway": "0.0.0.0", + "genmask": "255.255.255.0", + "flags": "U", + "metric": "100", + "ref": "0", + "use": "0", + "iface": "ens33" } ] """ diff --git a/jc/parsers/w.py b/jc/parsers/w.py index 9ecbe157..1e343ada 100644 --- a/jc/parsers/w.py +++ b/jc/parsers/w.py @@ -8,24 +8,24 @@ Example: $ w | jc --w -p [ { - "USER": "root", - "TTY": "ttyS0", - "FROM": "-", - "LOGIN_AT": "Mon20", - "IDLE": "2:27", - "JCPU": "10.61s", - "PCPU": "10.53s", - "WHAT": "-bash" + "user": "root", + "tty": "ttyS0", + "from": "-", + "login_at": "Mon20", + "idle": "0.00s", + "jcpu": "14.70s", + "pcpu": "0.00s", + "what": "bash" }, { - "USER": "root", - "TTY": "pts/0", - "FROM": "192.168.71.1", - "LOGIN_AT": "22:58", - "IDLE": "2.00s", - "JCPU": "0.04s", - "PCPU": "0.00s", - "WHAT": "w" + "user": "root", + "tty": "pts/0", + "from": "192.168.71.1", + "login_at": "Thu22", + "idle": "22:46m", + "jcpu": "0.05s", + "pcpu": "0.05s", + "what": "-bash" } ] """ From 7ca2a4bdb939293e8d7364f8901ad1fd6d3ee808 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 15:39:48 -0700 Subject: [PATCH 23/33] remove integer values --- README.md | 209 ++++++++++++++++++++---------------------- changelog.txt | 1 + jc/parsers/jobs.py | 4 +- jc/parsers/ls.py | 95 +++++++++++-------- jc/parsers/netstat.py | 84 ++++++----------- 5 files changed, 187 insertions(+), 206 deletions(-) diff --git a/README.md b/README.md index 794dd804..719887b5 100755 --- a/README.md +++ b/README.md @@ -6,22 +6,22 @@ JSON CLI output utility This allows further command line processing of output with tools like `jq` simply by piping commands: ``` -$ ls -l /usr/bin | jc --ls | jq '.[] | select(.bytes > 50000000)' +$ ls -l /usr/bin | jc --ls | jq '.[] | select(.size|tonumber > 50000000)' { "filename": "emacs", "flags": "-r-xr-xr-x", "links": 1, "owner": "root", "group": "wheel", - "bytes": 117164432, + "size": "117164432", "date": "May 3 22:26" } ``` -The `jc` parsers can also be used as python modules: +The `jc` parsers can also be used as python modules. In this case the output will be a python dictionary instead of JSON: ``` >>> import jc.parsers.ls ->>> +>>> >>> data='''-rwxr-xr-x 1 root wheel 23648 May 3 22:26 cat ... -rwxr-xr-x 1 root wheel 30016 May 3 22:26 chmod ... -rwxr-xr-x 1 root wheel 29024 May 3 22:26 cp @@ -30,22 +30,12 @@ The `jc` parsers can also be used as python modules: ... -rwxr-xr-x 1 root wheel 32000 May 3 22:26 dd ... -rwxr-xr-x 1 root wheel 23392 May 3 22:26 df ... -rwxr-xr-x 1 root wheel 18128 May 3 22:26 echo''' ->>> +>>> >>> jc.parsers.ls.parse(data) -[{'filename': 'cat', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', -'bytes': 23648, 'date': 'May 3 22:26'}, {'filename': 'chmod', 'flags': '-rwxr-xr-x', 'links': 1, -'owner': 'root', 'group': 'wheel', 'bytes': 30016, 'date': 'May 3 22:26'}, {'filename': 'cp', -'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'bytes': 29024, -'date': 'May 3 22:26'}, {'filename': 'csh', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', -'group': 'wheel', 'bytes': 375824, 'date': 'May 3 22:26'}, {'filename': 'date', -'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'bytes': 28608, -'date': 'May 3 22:26'}, {'filename': 'dd', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', -'group': 'wheel', 'bytes': 32000, 'date': 'May 3 22:26'}, {'filename': 'df', 'flags': '-rwxr-xr-x', -'links': 1, 'owner': 'root', 'group': 'wheel', 'bytes': 23392, 'date': 'May 3 22:26'}, -{'filename': 'echo', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', -'bytes': 18128, 'date': 'May 3 22:26'}] +[{'filename': 'cat', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '23648', 'date': 'May 3 22:26'}, {'filename': 'chmod', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '30016', 'date': 'May 3 22:26'}, {'filename': 'cp', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '29024', 'date': 'May 3 22:26'}, {'filename': 'csh', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '375824', 'date': 'May 3 22:26'}, {'filename': 'date', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '28608', 'date': 'May 3 22:26'}, {'filename': 'dd', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '32000', 'date': 'May 3 22:26'}, {'filename': 'df', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '23392', 'date': 'May 3 22:26'}, {'filename': 'echo', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '18128', 'date': 'May 3 22:26'}] ``` -In this case the output will be a python dictionary instead of JSON. + +The goal is to keep the resulting JSON as flat and simple as possible. Also, keys have been converted to lowercase and special characters are replaced whenever possible. Numbers are kept as strings because, depending on context or the output options, numbers can sometimes turn into strings. (e.g 'human readable' options) ## Installation ``` @@ -577,69 +567,92 @@ $ sudo iptables -vnL -t filter | jc --iptables -p $ jobs -l | jc --jobs -p [ { - "job_number": 1, - "pid": 14798, + "job_number": "1", + "pid": "19510", "status": "Running", - "command": "sleep 10000 &" + "command": "sleep 1000 &" }, { - "job_number": 2, - "pid": 14799, + "job_number": "2", + "pid": "19511", "status": "Running", - "command": "sleep 10001 &" + "command": "sleep 1001 &" }, { - "job_number": 3, - "pid": 14800, - "status": "Running", - "command": "sleep 10002 &" - }, - { - "job_number": 4, - "pid": 14814, + "job_number": "3", + "pid": "19512", "history": "previous", "status": "Running", - "command": "sleep 10003 &" + "command": "sleep 1002 &" }, { - "job_number": 5, - "pid": 14815, + "job_number": "4", + "pid": "19513", "history": "current", "status": "Running", - "command": "sleep 10004 &" + "command": "sleep 1003 &" } ] ``` ### ls ``` -$ ls -l /bin | jc --ls -p +$ ls -l /usr/bin | jc --ls -p [ + { + "filename": "apropos", + "link_to": "whatis", + "flags": "lrwxrwxrwx.", + "links": "1", + "owner": "root", + "group": "root", + "size": "6", + "date": "Aug 15 10:53" + }, + { + "filename": "arch", + "flags": "-rwxr-xr-x.", + "links": "1", + "owner": "root", + "group": "root", + "size": "33080", + "date": "Aug 19 23:25" + }, + { + "filename": "awk", + "link_to": "gawk", + "flags": "lrwxrwxrwx.", + "links": "1", + "owner": "root", + "group": "root", + "size": "4", + "date": "Aug 15 10:53" + }, + { + "filename": "base64", + "flags": "-rwxr-xr-x.", + "links": "1", + "owner": "root", + "group": "root", + "size": "37360", + "date": "Aug 19 23:25" + }, + { + "filename": "basename", + "flags": "-rwxr-xr-x.", + "links": "1", + "owner": "root", + "group": "root", + "size": "29032", + "date": "Aug 19 23:25" + }, { "filename": "bash", - "flags": "-r-xr-xr-x", - "links": 1, + "flags": "-rwxr-xr-x.", + "links": "1", "owner": "root", - "group": "wheel", - "bytes": 618416, - "date": "May 3 22:26" - }, - { - "filename": "cat", - "flags": "-rwxr-xr-x", - "links": 1, - "owner": "root", - "group": "wheel", - "bytes": 23648, - "date": "May 3 22:26" - }, - { - "filename": "chmod", - "flags": "-rwxr-xr-x", - "links": 1, - "owner": "root", - "group": "wheel", - "bytes": 30016, - "date": "May 3 22:26" + "group": "root", + "size": "964600", + "date": "Aug 8 05:06" }, ... ] @@ -875,8 +888,8 @@ $ netstat -p | jc --netstat -p "state": "ESTABLISHED", "pid": 53550, "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", @@ -888,39 +901,26 @@ $ netstat -p | jc --netstat -p "state": "ESTABLISHED", "pid": 53550, "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" } ] ``` ``` -$ netstat -lpn | jc --netstat -p +$ sudo netstat -lpn | jc --netstat -p [ { "transport_protocol": "tcp", "network_protocol": "ipv4", "local_address": "127.0.0.1", - "local_port": "42351", + "local_port": "25", "foreign_address": "0.0.0.0", "foreign_port": "*", "state": "LISTEN", - "pid": 1112, - "program_name": "containerd", - "receive_q": 0, - "send_q": 0 - }, - { - "transport_protocol": "tcp", - "network_protocol": "ipv4", - "local_address": "127.0.0.53", - "local_port": "53", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "state": "LISTEN", - "pid": 885, - "program_name": "systemd-resolve", - "receive_q": 0, - "send_q": 0 + "pid": "1584", + "program_name": "master", + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", @@ -930,48 +930,37 @@ $ netstat -lpn | jc --netstat -p "foreign_address": "0.0.0.0", "foreign_port": "*", "state": "LISTEN", - "pid": 1127, + "pid": "1213", "program_name": "sshd", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", "network_protocol": "ipv6", - "local_address": "::", - "local_port": "22", + "local_address": "::1", + "local_port": "25", "foreign_address": "::", "foreign_port": "*", "state": "LISTEN", - "pid": 1127, - "program_name": "sshd", - "receive_q": 0, - "send_q": 0 + "pid": "1584", + "program_name": "master", + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "udp", "network_protocol": "ipv4", - "local_address": "127.0.0.53", - "local_port": "53", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "pid": 885, - "program_name": "systemd-resolve", - "receive_q": 0, - "send_q": 0 - }, - { - "transport_protocol": "udp", - "network_protocol": "ipv4", - "local_address": "192.168.71.131", + "local_address": "0.0.0.0", "local_port": "68", "foreign_address": "0.0.0.0", "foreign_port": "*", - "pid": 867, - "program_name": "systemd-network", - "receive_q": 0, - "send_q": 0 - } + "pid": "19177", + "program_name": "dhclient", + "receive_q": "0", + "send_q": "0" + }, + ... ] ``` ### ps diff --git a/changelog.txt b/changelog.txt index 250a6382..7348c302 100644 --- a/changelog.txt +++ b/changelog.txt @@ -7,6 +7,7 @@ jc changelog - Flatten env parser output - Remove problematic characters from key names in: df, free, history, lsblk, lsof, and w - Where possible, lowercase all keys (except cases like env where the key is the variable name) +- Remove integer values 20191023 v0.9.1 - Add jobs parser diff --git a/jc/parsers/jobs.py b/jc/parsers/jobs.py index bc94b71d..a7f8f679 100644 --- a/jc/parsers/jobs.py +++ b/jc/parsers/jobs.py @@ -95,9 +95,9 @@ def parse(data): parsed_line[0] = parsed_line[0].lstrip('[').rstrip(']') # create list of dictionaries - output_line['job_number'] = int(parsed_line[0]) + output_line['job_number'] = parsed_line[0] if pid: - output_line['pid'] = int(pid) + output_line['pid'] = pid if job_history: output_line['history'] = job_history output_line['status'] = parsed_line[1] diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index 318153aa..3e509515 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -5,81 +5,96 @@ Usage: ls options supported: - None - - l - - a + - lah Examples: -$ ls -a /usr/bin | jc --ls -p +$ ls /usr/bin | jc --ls -p [ { - "filename": "." + "filename": "apropos" }, { - "filename": ".." + "filename": "arch" }, { - "filename": "2to3-" + "filename": "awk" }, { - "filename": "2to3-2.7" - }, - { - "filename": "AssetCacheLocatorUtil" + "filename": "base64" }, ... ] -$ ls -al /usr/bin | jc --ls -p +$ ls -l /usr/bin | jc --ls -p [ { - "filename": ".", - "flags": "drwxr-xr-x", - "links": 970, + "filename": "apropos", + "link_to": "whatis", + "flags": "lrwxrwxrwx.", + "links": "1", "owner": "root", - "group": "wheel", - "bytes": 31040, - "date": "Aug 27 21:20" + "group": "root", + "size": "6", + "date": "Aug 15 10:53" }, { - "filename": "..", - "flags": "drwxr-xr-x@", - "links": 9, + "filename": "arch", + "flags": "-rwxr-xr-x.", + "links": "1", "owner": "root", - "group": "wheel", - "bytes": 288, - "date": "May 3 22:14" + "group": "root", + "size": "33080", + "date": "Aug 19 23:25" }, { - "filename": "2to3-", - "flags": "-rwxr-xr-x", - "links": 4, + "filename": "awk", + "link_to": "gawk", + "flags": "lrwxrwxrwx.", + "links": "1", "owner": "root", - "group": "wheel", - "bytes": 925, - "date": "Feb 22 2019" + "group": "root", + "size": "4", + "date": "Aug 15 10:53" }, { - "filename": "2to3-2.7", - "link_to": "../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7", - "flags": "lrwxr-xr-x", - "links": 1, + "filename": "base64", + "flags": "-rwxr-xr-x.", + "links": "1", "owner": "root", - "group": "wheel", - "bytes": 74, - "date": "May 4 02:12" + "group": "root", + "size": "37360", + "date": "Aug 19 23:25" + }, + { + "filename": "basename", + "flags": "-rwxr-xr-x.", + "links": "1", + "owner": "root", + "group": "root", + "size": "29032", + "date": "Aug 19 23:25" + }, + { + "filename": "bash", + "flags": "-rwxr-xr-x.", + "links": "1", + "owner": "root", + "group": "root", + "size": "964600", + "date": "Aug 8 05:06" }, ... ] -$ $ ls -l /usr/bin | jc --ls | jq '.[] | 'select(.bytes > 50000000)' +$ ls -l /usr/bin | jc --ls | jq '.[] | select(.size|tonumber > 50000000)' { "filename": "emacs", "flags": "-r-xr-xr-x", "links": 1, "owner": "root", "group": "wheel", - "bytes": 117164432, + "size": "117164432", "date": "May 3 22:26" } """ @@ -117,10 +132,10 @@ def parse(data): output_line['link_to'] = filename_field[1] output_line['flags'] = parsed_line[0] - output_line['links'] = int(parsed_line[1]) + output_line['links'] = parsed_line[1] output_line['owner'] = parsed_line[2] output_line['group'] = parsed_line[3] - output_line['bytes'] = int(parsed_line[4]) + output_line['size'] = parsed_line[4] output_line['date'] = ' '.join(parsed_line[5:8]) output.append(output_line) else: diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index dafc2f37..54c7e9c4 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -22,8 +22,8 @@ $ netstat -p | jc --netstat -p "state": "ESTABLISHED", "pid": 53550, "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", @@ -35,38 +35,25 @@ $ netstat -p | jc --netstat -p "state": "ESTABLISHED", "pid": 53550, "program_name": "git-remote-ht", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" } ] -$ netstat -lpn | jc --netstat -p +$ sudo netstat -lpn | jc --netstat -p [ { "transport_protocol": "tcp", "network_protocol": "ipv4", "local_address": "127.0.0.1", - "local_port": "42351", + "local_port": "25", "foreign_address": "0.0.0.0", "foreign_port": "*", "state": "LISTEN", - "pid": 1112, - "program_name": "containerd", - "receive_q": 0, - "send_q": 0 - }, - { - "transport_protocol": "tcp", - "network_protocol": "ipv4", - "local_address": "127.0.0.53", - "local_port": "53", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "state": "LISTEN", - "pid": 885, - "program_name": "systemd-resolve", - "receive_q": 0, - "send_q": 0 + "pid": "1584", + "program_name": "master", + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", @@ -76,48 +63,37 @@ $ netstat -lpn | jc --netstat -p "foreign_address": "0.0.0.0", "foreign_port": "*", "state": "LISTEN", - "pid": 1127, + "pid": "1213", "program_name": "sshd", - "receive_q": 0, - "send_q": 0 + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "tcp", "network_protocol": "ipv6", - "local_address": "::", - "local_port": "22", + "local_address": "::1", + "local_port": "25", "foreign_address": "::", "foreign_port": "*", "state": "LISTEN", - "pid": 1127, - "program_name": "sshd", - "receive_q": 0, - "send_q": 0 + "pid": "1584", + "program_name": "master", + "receive_q": "0", + "send_q": "0" }, { "transport_protocol": "udp", "network_protocol": "ipv4", - "local_address": "127.0.0.53", - "local_port": "53", - "foreign_address": "0.0.0.0", - "foreign_port": "*", - "pid": 885, - "program_name": "systemd-resolve", - "receive_q": 0, - "send_q": 0 - }, - { - "transport_protocol": "udp", - "network_protocol": "ipv4", - "local_address": "192.168.71.131", + "local_address": "0.0.0.0", "local_port": "68", "foreign_address": "0.0.0.0", "foreign_port": "*", - "pid": 867, - "program_name": "systemd-network", - "receive_q": 0, - "send_q": 0 - } + "pid": "19177", + "program_name": "dhclient", + "receive_q": "0", + "send_q": "0" + }, + ... ] """ import string @@ -161,15 +137,15 @@ def parse_line(entry): output_line['state'] = parsed_line[5] if len(parsed_line) > 6 and parsed_line[6][0] in string.digits: - output_line['pid'] = int(parsed_line[6].split('/')[0]) + output_line['pid'] = parsed_line[6].split('/')[0] output_line['program_name'] = parsed_line[6].split('/')[1] else: if parsed_line[5][0] in string.digits: - output_line['pid'] = int(parsed_line[5].split('/')[0]) + output_line['pid'] = parsed_line[5].split('/')[0] output_line['program_name'] = parsed_line[5].split('/')[1] - output_line['receive_q'] = int(parsed_line[1]) - output_line['send_q'] = int(parsed_line[2]) + output_line['receive_q'] = parsed_line[1] + output_line['send_q'] = parsed_line[2] return output_line From 27245590ce28be933f1a55db4ee6b24783f4b63d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 15:40:53 -0700 Subject: [PATCH 24/33] remove integers --- jc/parsers/jobs.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/jc/parsers/jobs.py b/jc/parsers/jobs.py index a7f8f679..22fad6c9 100644 --- a/jc/parsers/jobs.py +++ b/jc/parsers/jobs.py @@ -10,41 +10,33 @@ Example: $ jobs -l | jc --jobs -p [ { - "job_number": 1, - "pid": 14798, + "job_number": "1", + "pid": "19510", "status": "Running", - "command": "sleep 10000 &" + "command": "sleep 1000 &" }, { - "job_number": 2, - "pid": 14799, + "job_number": "2", + "pid": "19511", "status": "Running", - "command": "sleep 10001 &" + "command": "sleep 1001 &" }, { - "job_number": 3, - "pid": 14800, - "status": "Running", - "command": "sleep 10002 &" - }, - { - "job_number": 4, - "pid": 14814, + "job_number": "3", + "pid": "19512", "history": "previous", "status": "Running", - "command": "sleep 10003 &" + "command": "sleep 1002 &" }, { - "job_number": 5, - "pid": 14815, + "job_number": "4", + "pid": "19513", "history": "current", "status": "Running", - "command": "sleep 10004 &" + "command": "sleep 1003 &" } ] """ - - import string From 1acc4d6c29d136e286f82d607c4a3a59e2680b68 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 15:52:53 -0700 Subject: [PATCH 25/33] fix uptime parsing for short uptimes --- jc/parsers/uptime.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jc/parsers/uptime.py b/jc/parsers/uptime.py index 7116021a..ba95ea3e 100644 --- a/jc/parsers/uptime.py +++ b/jc/parsers/uptime.py @@ -25,6 +25,10 @@ def parse(data): if cleandata: parsed_line = cleandata[0].split() + # fix parsing if uptime is only a few minutes + if len(parsed_line) == 11: + parsed_line.insert(2, 'only') + output['time'] = parsed_line[0] output['uptime'] = ' '.join(parsed_line[2:5]).rstrip(',') output['users'] = parsed_line[5] From 0328e14c7c65a330093faa52fe48dae6cb173e42 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 16:05:34 -0700 Subject: [PATCH 26/33] handle ctrl-c gracefully --- jc/jc.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jc/jc.py b/jc/jc.py index be169920..7a1fb0b5 100755 --- a/jc/jc.py +++ b/jc/jc.py @@ -5,6 +5,7 @@ Main input module """ import sys +import signal import json import jc.parsers.df import jc.parsers.env @@ -53,7 +54,13 @@ def helptext(): print(' ls -al | jc --ls -p\n', file=sys.stderr) +def ctrlc(signum, frame): + exit() + + def main(): + signal.signal(signal.SIGINT, ctrlc) + if sys.stdin.isatty(): print('jc: missing piped data\n', file=sys.stderr) helptext() From 1a4fc204e21f1555ae67c3147322c8d73fab630b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 16:09:31 -0700 Subject: [PATCH 27/33] Documentation update --- README.md | 10 +++++++++- changelog.txt | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 719887b5..1e2de15f 100755 --- a/README.md +++ b/README.md @@ -32,7 +32,15 @@ The `jc` parsers can also be used as python modules. In this case the output wil ... -rwxr-xr-x 1 root wheel 18128 May 3 22:26 echo''' >>> >>> jc.parsers.ls.parse(data) -[{'filename': 'cat', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '23648', 'date': 'May 3 22:26'}, {'filename': 'chmod', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '30016', 'date': 'May 3 22:26'}, {'filename': 'cp', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '29024', 'date': 'May 3 22:26'}, {'filename': 'csh', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '375824', 'date': 'May 3 22:26'}, {'filename': 'date', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '28608', 'date': 'May 3 22:26'}, {'filename': 'dd', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '32000', 'date': 'May 3 22:26'}, {'filename': 'df', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '23392', 'date': 'May 3 22:26'}, {'filename': 'echo', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '18128', 'date': 'May 3 22:26'}] +[{'filename': 'cat', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '23648', +'date': 'May 3 22:26'}, {'filename': 'chmod', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', +'group': 'wheel', 'size': '30016', 'date': 'May 3 22:26'}, {'filename': 'cp', 'flags': '-rwxr-xr-x', +'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '29024', 'date': 'May 3 22:26'}, {'filename': 'csh', +'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '375824', 'date': 'May 3 22:26'}, {'filename': 'date', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '28608', +'date': 'May 3 22:26'}, {'filename': 'dd', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', +'size': '32000', 'date': 'May 3 22:26'}, {'filename': 'df', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', +'group': 'wheel', 'size': '23392', 'date': 'May 3 22:26'}, {'filename': 'echo', 'flags': '-rwxr-xr-x', +'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '18128', 'date': 'May 3 22:26'}] ``` The goal is to keep the resulting JSON as flat and simple as possible. Also, keys have been converted to lowercase and special characters are replaced whenever possible. Numbers are kept as strings because, depending on context or the output options, numbers can sometimes turn into strings. (e.g 'human readable' options) diff --git a/changelog.txt b/changelog.txt index 7348c302..fc06ce22 100644 --- a/changelog.txt +++ b/changelog.txt @@ -8,6 +8,7 @@ jc changelog - Remove problematic characters from key names in: df, free, history, lsblk, lsof, and w - Where possible, lowercase all keys (except cases like env where the key is the variable name) - Remove integer values +- Handle CTRL-C gracefully 20191023 v0.9.1 - Add jobs parser From d7913070315e5e0c89006942250aaf37e9e3e18e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 16:12:45 -0700 Subject: [PATCH 28/33] documentation update --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e2de15f..b76927df 100755 --- a/README.md +++ b/README.md @@ -36,7 +36,8 @@ The `jc` parsers can also be used as python modules. In this case the output wil 'date': 'May 3 22:26'}, {'filename': 'chmod', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '30016', 'date': 'May 3 22:26'}, {'filename': 'cp', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '29024', 'date': 'May 3 22:26'}, {'filename': 'csh', -'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '375824', 'date': 'May 3 22:26'}, {'filename': 'date', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '28608', +'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '375824', 'date': 'May 3 22:26'}, +{'filename': 'date', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '28608', 'date': 'May 3 22:26'}, {'filename': 'dd', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '32000', 'date': 'May 3 22:26'}, {'filename': 'df', 'flags': '-rwxr-xr-x', 'links': 1, 'owner': 'root', 'group': 'wheel', 'size': '23392', 'date': 'May 3 22:26'}, {'filename': 'echo', 'flags': '-rwxr-xr-x', From de647bba4aff84f7b513cf7f6fec8d3933b0d9da Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 16:14:32 -0700 Subject: [PATCH 29/33] documentation update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b76927df..2b9671e5 100755 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ The goal is to keep the resulting JSON as flat and simple as possible. Also, key ## Installation ``` -$ pip3 install jc +$ pip3 install --upgrade jc ``` ## Usage From f4d9c1b699fa30bd297a40ccf3d5f6e7ba3d4253 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 17:16:02 -0700 Subject: [PATCH 30/33] fix uptime for minutes and hours long uptime --- jc/parsers/uptime.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jc/parsers/uptime.py b/jc/parsers/uptime.py index ba95ea3e..558d484f 100644 --- a/jc/parsers/uptime.py +++ b/jc/parsers/uptime.py @@ -26,14 +26,18 @@ def parse(data): parsed_line = cleandata[0].split() # fix parsing if uptime is only a few minutes + if len(parsed_line) < 12: + parsed_line.insert(2, ' ') + + # fix parsing if uptime is only a few hours if len(parsed_line) == 11: - parsed_line.insert(2, 'only') + parsed_line.insert(2, ' ') output['time'] = parsed_line[0] - output['uptime'] = ' '.join(parsed_line[2:5]).rstrip(',') + output['uptime'] = ' '.join(parsed_line[2:5]).lstrip().rstrip(',') output['users'] = parsed_line[5] - output['load_1m'] = parsed_line[9] - output['load_5m'] = parsed_line[10] + output['load_1m'] = parsed_line[9].rstrip(',') + output['load_5m'] = parsed_line[10].rstrip(',') output['load_15m'] = parsed_line[11] return output From a3d43f27f7bb0835b3593b7c303fe0ee8d896a58 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 18:25:33 -0700 Subject: [PATCH 31/33] fix odd uptime text parsing --- jc/parsers/uptime.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/jc/parsers/uptime.py b/jc/parsers/uptime.py index 558d484f..c5d6c362 100644 --- a/jc/parsers/uptime.py +++ b/jc/parsers/uptime.py @@ -25,19 +25,21 @@ def parse(data): if cleandata: parsed_line = cleandata[0].split() - # fix parsing if uptime is only a few minutes - if len(parsed_line) < 12: + # allow space for odd times + while len(parsed_line) < 20: parsed_line.insert(2, ' ') - # fix parsing if uptime is only a few hours - if len(parsed_line) == 11: - parsed_line.insert(2, ' ') + # find first part of time + for i, word in enumerate(parsed_line[2:]): + if word != ' ': + marker = i + 2 + break output['time'] = parsed_line[0] - output['uptime'] = ' '.join(parsed_line[2:5]).lstrip().rstrip(',') - output['users'] = parsed_line[5] - output['load_1m'] = parsed_line[9].rstrip(',') - output['load_5m'] = parsed_line[10].rstrip(',') - output['load_15m'] = parsed_line[11] + output['uptime'] = ' '.join(parsed_line[marker:13]).lstrip().rstrip(',') + output['users'] = parsed_line[13] + output['load_1m'] = parsed_line[17].rstrip(',') + output['load_5m'] = parsed_line[18].rstrip(',') + output['load_15m'] = parsed_line[19] return output From d3c89a3092973a7a8765463fff41c2469ed675cd Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 19:27:02 -0700 Subject: [PATCH 32/33] check for enough info to parse --- jc/parsers/uname.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/jc/parsers/uname.py b/jc/parsers/uname.py index 892ec042..91632a85 100644 --- a/jc/parsers/uname.py +++ b/jc/parsers/uname.py @@ -26,17 +26,19 @@ def parse(data): output = {} parsed_line = data.split(maxsplit=3) - output['kernel_name'] = parsed_line.pop(0) - output['node_name'] = parsed_line.pop(0) - output['kernel_release'] = parsed_line.pop(0) + if len(parsed_line) > 1: - parsed_line = parsed_line[-1].rsplit(maxsplit=4) + output['kernel_name'] = parsed_line.pop(0) + output['node_name'] = parsed_line.pop(0) + output['kernel_release'] = parsed_line.pop(0) - output['operating_system'] = parsed_line.pop(-1) - output['hardware_platform'] = parsed_line.pop(-1) - output['processor'] = parsed_line.pop(-1) - output['machine'] = parsed_line.pop(-1) + parsed_line = parsed_line[-1].rsplit(maxsplit=4) - output['kernel_version'] = parsed_line.pop(0) + output['operating_system'] = parsed_line.pop(-1) + output['hardware_platform'] = parsed_line.pop(-1) + output['processor'] = parsed_line.pop(-1) + output['machine'] = parsed_line.pop(-1) + + output['kernel_version'] = parsed_line.pop(0) return output From c9849ce0db1f864d33739f8f6f942bf5672bc3b7 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Oct 2019 19:41:35 -0700 Subject: [PATCH 33/33] changelog update --- changelog.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.txt b/changelog.txt index fc06ce22..160030a9 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,9 +1,10 @@ jc changelog -2019xxxx v1.0.1 +20191025 v1.0.1 - Add w parser - Add uptime parser - Add history parser +- Fix uptime parser - Flatten env parser output - Remove problematic characters from key names in: df, free, history, lsblk, lsof, and w - Where possible, lowercase all keys (except cases like env where the key is the variable name)