From 6869133f532bda10138aa0e101fb742c0517a06b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 25 May 2022 10:31:41 -0700 Subject: [PATCH] working top parser and doc update --- README.md | 1 + docs/parsers/ping_s.md | 2 +- docs/parsers/top.md | 322 ++++++++++++++++++ jc/parsers/top.py | 260 ++++++++++++-- man/jc.1 | 7 +- .../centos-7.7/top-b-n1-gib-allfields-w.out | 2 +- 6 files changed, 555 insertions(+), 39 deletions(-) create mode 100644 docs/parsers/top.md diff --git a/README.md b/README.md index 0a96d406..259e5a31 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,7 @@ option. | ` --systeminfo` | `systeminfo` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/systeminfo) | | ` --time` | `/usr/bin/time` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/time) | | ` --timedatectl` | `timedatectl status` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/timedatectl) | +| ` --top` | `top -b` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/top) | | ` --tracepath` | `tracepath` and `tracepath6` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/tracepath) | | ` --traceroute` | `traceroute` and `traceroute6` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/traceroute) | | ` --ufw` | `ufw status` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/ufw) | diff --git a/docs/parsers/ping_s.md b/docs/parsers/ping_s.md index 9ba85e53..176aa00d 100644 --- a/docs/parsers/ping_s.md +++ b/docs/parsers/ping_s.md @@ -12,7 +12,7 @@ Supports `ping` and `ping6` output. Usage (cli): - $ ping | jc --ping-s + $ ping 1.2.3.4 | jc --ping-s > Note: When piping `jc` converted `ping` output to other processes it may appear the output is hanging due to the OS pipe buffers. This is because diff --git a/docs/parsers/top.md b/docs/parsers/top.md new file mode 100644 index 00000000..dd395b82 --- /dev/null +++ b/docs/parsers/top.md @@ -0,0 +1,322 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.top + +jc - JSON Convert `top -b` command output parser + +Requires batch mode (`-b`). The `-n` option must also be used to limit +the number of times `top` is run. + +Warning messages will be printed to STDERR if truncated fields are detected. +These warnings an be suppressed with the `-q` or `quiet=True` option. + +Usage (cli): + + $ top -b -n 3 | jc --top + + or + + $ jc top -b -n 3 + +Usage (module): + + import jc + result = jc.parse('top', top_command_output) + +Schema: + + All `-` values are converted to `null` + + [ + { + "time": string, + "uptime": integer, + "users": integer, + "load_1m": float, + "load_5m": float, + "load_15m": float, + "tasks_total": integer, + "tasks_running": integer, + "tasks_sleeping": integer, + "tasks_stopped": integer, + "tasks_zombie": integer, + "cpu_user": float, + "cpu_sys": float, + "cpu_nice": float, + "cpu_idle": float, + "cpu_wait": float, + "cpu_hardware": float, + "cpu_software": float, + "cpu_steal": float, + "mem_total": integer, + "mem_free": integer, + "mem_used": integer, + "mem_buff_cache": integer, + "swap_total": integer, + "swap_free": integer, + "swap_used": integer, + "mem_available": integer, + "processes": [ + { + "pid": integer, + "user": string, + "priority": integer, + "nice": integer, + "virtual_mem": string, + "resident_mem": string, + "shared_mem": string, + "status": string, + "percent_cpu": float, + "percent_mem": float, + "time_hundredths": string, + "command": string, + "parent_pid": integer, + "uid": integer, + "real_uid": integer, + "real_user": string, + "saved_uid": integer, + "saved_user": string, + "gid": integer, + "group": string, + "pgrp": integer, + "tty": string, + "tty_process_gid": integer, + "session_id": integer, + "thread_count": integer, + "last_used_processor": integer, + "time": string, + "swap": string, + "code": string, + "data": string, + "major_page_fault_count": integer, + "minor_page_fault_count": integer, + "dirty_pages_count": integer, + "sleeping_in_function": string, + "flags": string, + "cgroups": string, + "supplementary_gids": [ + integer + ], + "supplementary_groups": [ + string + ], + "thread_gid": integer, + "environment_variables": [ + string + ] + "major_page_fault_count_delta": integer, + "minor_page_fault_count_delta": integer, + "used": string, + "ipc_namespace_inode": integer, + "mount_namespace_inode": integer, + "net_namespace_inode": integer, + "pid_namespace_inode": integer, + "user_namespace_inode": integer, + "nts_namespace_inode": integer + } + ] + } + ] + +Examples: + + $ top -b -n 3 | jc --top -p + [ + { + "time": "11:20:43", + "uptime": 118, + "users": 2, + "load_1m": 0.0, + "load_5m": 0.01, + "load_15m": 0.05, + "tasks_total": 108, + "tasks_running": 2, + "tasks_sleeping": 106, + "tasks_stopped": 0, + "tasks_zombie": 0, + "cpu_user": 5.6, + "cpu_sys": 11.1, + "cpu_nice": 0.0, + "cpu_idle": 83.3, + "cpu_wait": 0.0, + "cpu_hardware": 0.0, + "cpu_software": 0.0, + "cpu_steal": 0.0, + "swap_total": 2, + "swap_free": 2, + "swap_used": 0, + "mem_available": 3, + "processes": [ + { + "pid": 2225, + "user": "kbrazil", + "priority": 20, + "nice": 0, + "virtual_mem": 158, + "resident_mem": 2, + "shared_mem": 1, + "status": "running", + "percent_cpu": 12.5, + "percent_mem": 0.1, + "time_hundredths": "0:00.02", + "command": "top", + "parent_pid": 1884, + "uid": 1000, + "real_uid": 1000, + "real_user": "kbrazil", + "saved_uid": 1000, + "saved_user": "kbrazil", + "gid": 1000, + "group": "kbrazil", + "pgrp": 2225, + "tty": "pts/0", + "tty_process_gid": 2225, + "session_id": 1884, + "thread_count": 1, + "last_used_processor": 0, + "time": "0:00", + "swap": "0.0m", + "code": "0.1m", + "data": "1.0m", + "major_page_fault_count": 0, + "minor_page_fault_count": 736, + "dirty_pages_count": 0, + "sleeping_in_function": null, + "flags": "..4.2...", + "cgroups": "1:name=systemd:/user.slice/user-1000.+", + "supplementary_gids": [ + 10, + 1000 + ], + "supplementary_groups": [ + "wheel", + "kbrazil" + ], + "thread_gid": 2225, + "environment_variables": [ + "XDG_SESSION_ID=2", + "HOSTNAME=localhost" + ], + "major_page_fault_count_delta": 0, + "minor_page_fault_count_delta": 4, + "used": "2.2m", + "ipc_namespace_inode": 4026531839, + "mount_namespace_inode": 4026531840, + "net_namespace_inode": 4026531956, + "pid_namespace_inode": 4026531836, + "user_namespace_inode": 4026531837, + "nts_namespace_inode": 4026531838 + }, + ... + ] + } + ] + + $ top -b -n 3 | jc --top -p -r + [ + { + "time": "11:20:43", + "uptime": "1:18", + "users": "2", + "load_1m": "0.00", + "load_5m": "0.01", + "load_15m": "0.05", + "tasks_total": "108", + "tasks_running": "2", + "tasks_sleeping": "106", + "tasks_stopped": "0", + "tasks_zombie": "0", + "cpu_user": "5.6", + "cpu_sys": "11.1", + "cpu_nice": "0.0", + "cpu_idle": "83.3", + "cpu_wait": "0.0", + "cpu_hardware": "0.0", + "cpu_software": "0.0", + "cpu_steal": "0.0", + "swap_total": "2.0", + "swap_free": "2.0", + "swap_used": "0.0", + "mem_available": "3.3", + "processes": [ + { + "PID": "2225", + "USER": "kbrazil", + "PR": "20", + "NI": "0", + "VIRT": "158.1m", + "RES": "2.2m", + "SHR": "1.6m", + "S": "R", + "%CPU": "12.5", + "%MEM": "0.1", + "TIME+": "0:00.02", + "COMMAND": "top", + "PPID": "1884", + "UID": "1000", + "RUID": "1000", + "RUSER": "kbrazil", + "SUID": "1000", + "SUSER": "kbrazil", + "GID": "1000", + "GROUP": "kbrazil", + "PGRP": "2225", + "TTY": "pts/0", + "TPGID": "2225", + "SID": "1884", + "nTH": "1", + "P": "0", + "TIME": "0:00", + "SWAP": "0.0m", + "CODE": "0.1m", + "DATA": "1.0m", + "nMaj": "0", + "nMin": "736", + "nDRT": "0", + "WCHAN": "-", + "Flags": "..4.2...", + "CGROUPS": "1:name=systemd:/user.slice/user-1000.+", + "SUPGIDS": "10,1000", + "SUPGRPS": "wheel,kbrazil", + "TGID": "2225", + "ENVIRON": "XDG_SESSION_ID=2 HOSTNAME=localhost S+", + "vMj": "0", + "vMn": "4", + "USED": "2.2m", + "nsIPC": "4026531839", + "nsMNT": "4026531840", + "nsNET": "4026531956", + "nsPID": "4026531836", + "nsUSER": "4026531837", + "nsUTS": "4026531838" + }, + ... + ] + } + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/top.py b/jc/parsers/top.py index 5f4b4445..e55b3eed 100644 --- a/jc/parsers/top.py +++ b/jc/parsers/top.py @@ -1,8 +1,11 @@ -"""jc - JSON Convert `top` command output parser +"""jc - JSON Convert `top -b` command output parser -Requires batch mode (`-b`). The `-n` option should also be used to limit +Requires batch mode (`-b`). The `-n` option must also be used to limit the number of times `top` is run. +Warning messages will be printed to STDERR if truncated fields are detected. +These warnings an be suppressed with the `-q` or `quiet=True` option. + Usage (cli): $ top -b -n 3 | jc --top @@ -18,6 +21,8 @@ Usage (module): Schema: + All `-` values are converted to `null` + [ { "time": string, @@ -59,7 +64,7 @@ Schema: "status": string, "percent_cpu": float, "percent_mem": float, - "time_hundredths ": string, + "time_hundredths": string, "command": string, "parent_pid": integer, "uid": integer, @@ -112,10 +117,179 @@ Schema: Examples: $ top -b -n 3 | jc --top -p - [] + [ + { + "time": "11:20:43", + "uptime": 118, + "users": 2, + "load_1m": 0.0, + "load_5m": 0.01, + "load_15m": 0.05, + "tasks_total": 108, + "tasks_running": 2, + "tasks_sleeping": 106, + "tasks_stopped": 0, + "tasks_zombie": 0, + "cpu_user": 5.6, + "cpu_sys": 11.1, + "cpu_nice": 0.0, + "cpu_idle": 83.3, + "cpu_wait": 0.0, + "cpu_hardware": 0.0, + "cpu_software": 0.0, + "cpu_steal": 0.0, + "swap_total": 2, + "swap_free": 2, + "swap_used": 0, + "mem_available": 3, + "processes": [ + { + "pid": 2225, + "user": "kbrazil", + "priority": 20, + "nice": 0, + "virtual_mem": 158, + "resident_mem": 2, + "shared_mem": 1, + "status": "running", + "percent_cpu": 12.5, + "percent_mem": 0.1, + "time_hundredths": "0:00.02", + "command": "top", + "parent_pid": 1884, + "uid": 1000, + "real_uid": 1000, + "real_user": "kbrazil", + "saved_uid": 1000, + "saved_user": "kbrazil", + "gid": 1000, + "group": "kbrazil", + "pgrp": 2225, + "tty": "pts/0", + "tty_process_gid": 2225, + "session_id": 1884, + "thread_count": 1, + "last_used_processor": 0, + "time": "0:00", + "swap": "0.0m", + "code": "0.1m", + "data": "1.0m", + "major_page_fault_count": 0, + "minor_page_fault_count": 736, + "dirty_pages_count": 0, + "sleeping_in_function": null, + "flags": "..4.2...", + "cgroups": "1:name=systemd:/user.slice/user-1000.+", + "supplementary_gids": [ + 10, + 1000 + ], + "supplementary_groups": [ + "wheel", + "kbrazil" + ], + "thread_gid": 2225, + "environment_variables": [ + "XDG_SESSION_ID=2", + "HOSTNAME=localhost" + ], + "major_page_fault_count_delta": 0, + "minor_page_fault_count_delta": 4, + "used": "2.2m", + "ipc_namespace_inode": 4026531839, + "mount_namespace_inode": 4026531840, + "net_namespace_inode": 4026531956, + "pid_namespace_inode": 4026531836, + "user_namespace_inode": 4026531837, + "nts_namespace_inode": 4026531838 + }, + ... + ] + } + ] $ top -b -n 3 | jc --top -p -r - [] + [ + { + "time": "11:20:43", + "uptime": "1:18", + "users": "2", + "load_1m": "0.00", + "load_5m": "0.01", + "load_15m": "0.05", + "tasks_total": "108", + "tasks_running": "2", + "tasks_sleeping": "106", + "tasks_stopped": "0", + "tasks_zombie": "0", + "cpu_user": "5.6", + "cpu_sys": "11.1", + "cpu_nice": "0.0", + "cpu_idle": "83.3", + "cpu_wait": "0.0", + "cpu_hardware": "0.0", + "cpu_software": "0.0", + "cpu_steal": "0.0", + "swap_total": "2.0", + "swap_free": "2.0", + "swap_used": "0.0", + "mem_available": "3.3", + "processes": [ + { + "PID": "2225", + "USER": "kbrazil", + "PR": "20", + "NI": "0", + "VIRT": "158.1m", + "RES": "2.2m", + "SHR": "1.6m", + "S": "R", + "%CPU": "12.5", + "%MEM": "0.1", + "TIME+": "0:00.02", + "COMMAND": "top", + "PPID": "1884", + "UID": "1000", + "RUID": "1000", + "RUSER": "kbrazil", + "SUID": "1000", + "SUSER": "kbrazil", + "GID": "1000", + "GROUP": "kbrazil", + "PGRP": "2225", + "TTY": "pts/0", + "TPGID": "2225", + "SID": "1884", + "nTH": "1", + "P": "0", + "TIME": "0:00", + "SWAP": "0.0m", + "CODE": "0.1m", + "DATA": "1.0m", + "nMaj": "0", + "nMin": "736", + "nDRT": "0", + "WCHAN": "-", + "Flags": "..4.2...", + "CGROUPS": "1:name=systemd:/user.slice/user-1000.+", + "SUPGIDS": "10,1000", + "SUPGRPS": "wheel,kbrazil", + "TGID": "2225", + "ENVIRON": "XDG_SESSION_ID=2 HOSTNAME=localhost S+", + "vMj": "0", + "vMn": "4", + "USED": "2.2m", + "nsIPC": "4026531839", + "nsMNT": "4026531840", + "nsNET": "4026531956", + "nsPID": "4026531836", + "nsUSER": "4026531837", + "nsUTS": "4026531838" + }, + ... + ] + } + ] """ from typing import List, Dict import jc.utils @@ -141,7 +315,7 @@ def _safe_split(string: str, path: str, delim: str = ' ', quiet=False) -> List[s split_string = [x for x in split_string if not x.endswith('+')] if string.endswith('+') and not quiet: - jc.utils.warning_message([f'{path} list was truncated']) + jc.utils.warning_message([f'{path} list was truncated by top']) return split_string @@ -171,7 +345,7 @@ def _process(proc_data: List[Dict], quiet=False) -> List[Dict]: 'PID': 'pid', 'SWAP': 'swap', 'TIME': 'time', - 'TIME+': 'time_hundredths ', + 'TIME+': 'time_hundredths', 'TTY': 'tty', 'UID': 'uid', 'USED': 'used', @@ -219,41 +393,29 @@ def _process(proc_data: List[Dict], quiet=False) -> List[Dict]: } int_list: List = [ - 'uptime', - 'users', - 'tasks_total', - 'tasks_running', - 'tasks_sleeping', - 'tasks_stopped', - 'tasks_zombie', - 'mem_total', - 'mem_free', - 'mem_used', - 'mem_buff_cache', - 'swap_total', - 'swap_free', - 'swap_used', - 'mem_available' + 'uptime', 'users', 'tasks_total', 'tasks_running', 'tasks_sleeping', 'tasks_stopped', + 'tasks_zombie', 'mem_total', 'mem_free', 'mem_used', 'mem_buff_cache', 'swap_total', + 'swap_free', 'swap_used', 'mem_available', 'pid', 'priority', 'nice', 'parent_pid', 'uid', + 'real_uid', 'saved_uid', 'gid', 'pgrp', 'tty_process_gid', 'session_id', 'thread_count', + 'last_used_processor', 'major_page_fault_count', 'minor_page_fault_count', + 'dirty_pages_count', 'thread_gid', 'major_page_fault_count_delta', + 'minor_page_fault_count_delta', 'ipc_namespace_inode', 'mount_namespace_inode', + 'net_namespace_inode', 'pid_namespace_inode', 'user_namespace_inode', 'nts_namespace_inode', + 'virtual_mem', 'resident_mem', 'shared_mem' ] float_list: List = [ - 'load_1m', - 'load_5m', - 'load_15m', - 'cpu_user', - 'cpu_sys', - 'cpu_nice', - 'cpu_idle', - 'cpu_wait', - 'cpu_hardware', - 'cpu_software', - 'cpu_steal' + 'load_1m', 'load_5m', 'load_15m', 'cpu_user', 'cpu_sys', 'cpu_nice', 'cpu_idle', 'cpu_wait', + 'cpu_hardware', 'cpu_software', 'cpu_steal', 'percent_cpu', 'percent_mem' ] - for idx, item in enumerate(proc_data): - # root int and float conversions for key in item: + # root truncation warnings + if isinstance(item[key], str) and item[key].endswith('+') and not quiet: + jc.utils.warning_message([f'item[{idx}]["{key}"] was truncated by top']) + + # root int and float conversions if key in int_list: item[key] = jc.utils.convert_to_int(item[key]) @@ -266,11 +428,32 @@ def _process(proc_data: List[Dict], quiet=False) -> List[Dict]: for old_key in proc_copy.keys(): proc[key_map[old_key]] = proc.pop(old_key) - # set dashes to nulls + # cleanup values for key in proc.keys(): + + # set dashes to nulls if proc[key] == '-': proc[key] = None + # because of ambiguous column spacing (right-justified numbers + # with left-justified dashes for null values) there are some hanging + # dashes that need to be cleaned up in some values. Seems the correct + # values are kept in the assigned columns, so this should not affect + # data integrity. + if proc[key] and proc[key].endswith(' -'): + new_val = proc[key][::-1] + new_val = new_val.replace('- ', '') + new_val = new_val[::-1] + proc[key] = new_val + + # do int/float conversions for the process objects + if proc[key]: + if key in int_list: + proc[key] = jc.utils.convert_to_int(proc[key]) + + if key in float_list: + proc[key] = jc.utils.convert_to_float(proc[key]) + # set status string if proc.get('status'): proc['status'] = status_map[proc['status']] @@ -301,6 +484,11 @@ def _process(proc_data: List[Dict], quiet=False) -> List[Dict]: quiet=quiet ) + for key in proc.keys(): + # print final warnings for truncated string values + if isinstance(proc[key], str) and proc[key].endswith('+') and not quiet: + jc.utils.warning_message([f'item[{idx}]["processes"][{p_idx}]["{key}"] was truncated by top']) + return proc_data diff --git a/man/jc.1 b/man/jc.1 index 99d5ea43..8a769491 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-05-23 1.20.0 "JSON Convert" +.TH jc 1 2022-05-25 1.20.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -427,6 +427,11 @@ Key/Value file parser \fB--timedatectl\fP `timedatectl status` command parser +.TP +.B +\fB--top\fP +`top -b` command parser + .TP .B \fB--tracepath\fP diff --git a/tests/fixtures/centos-7.7/top-b-n1-gib-allfields-w.out b/tests/fixtures/centos-7.7/top-b-n1-gib-allfields-w.out index 0e078553..192ff003 100644 --- a/tests/fixtures/centos-7.7/top-b-n1-gib-allfields-w.out +++ b/tests/fixtures/centos-7.7/top-b-n1-gib-allfields-w.out @@ -112,4 +112,4 @@ GiB Swap: 2.0 total, 2.0 free, 0.0 used. 3.3 avail Mem 2139 root 20 0 0.0m 0.0m 0.0m S 0.0 0.0 0:00.36 kworker/0:2 2 0 0 root 0 root 0 root 0 ? -1 0 1 0 0:00 0.0m 0.0m 0.0m 0 0 0 worker_th+ .42.a.6. - - - 2139 - 0 0 0.0m - - - - - - 2197 root 20 0 0.0m 0.0m 0.0m S 0.0 0.0 0:00.01 kworker/0:3 2 0 0 root 0 root 0 root 0 ? -1 0 1 0 0:00 0.0m 0.0m 0.0m 0 0 0 worker_th+ .42.a.6. - - - 2197 - 0 0 0.0m - - - - - - 2205 root 20 0 0.0m 0.0m 0.0m S 0.0 0.0 0:00.01 kworker/0:0 2 0 0 root 0 root 0 root 0 ? -1 0 1 0 0:00 0.0m 0.0m 0.0m 0 0 0 worker_th+ .42.a.6. - - - 2205 - 0 0 0.0m - - - - - - - 2213 root 20 0 0.0m 0.0m 0.0m S 0.0 0.0 0:00.00 kworker/0:1 2 0 0 root 0 root 0 root 0 ? -1 0 1 0 0:00 0.0m 0.0m 0.0m 0 0 0 worker_th+ .42.a.6. - - - 2213 - 0 0 0.0m - - - - - - + 2213 root 20 0 0.0m 0.0m 0.0m S 0.0 0.0 0:00.00 kworker/0:1 2 0 0 root 0 root 0 root 0 ? -1 0 1 0 0:00 0.0m 0.0m 0.0m 0 0 0 worker_th+ .42.a.6. - - - 2213 - 0 0 0.0m - - 5678 - 1234 -