From f4242669bac3af5eb196c46d4146823aacec40ab Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 10:52:40 -0800 Subject: [PATCH 01/87] minor cleanup --- jc/parsers/arp.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/jc/parsers/arp.py b/jc/parsers/arp.py index 53f0c981..7524f265 100644 --- a/jc/parsers/arp.py +++ b/jc/parsers/arp.py @@ -123,7 +123,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.8' + version = '1.9' description = '`arp` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -208,11 +208,6 @@ def parse(data, raw=False, quiet=False): raw_output.append(output_line) - if raw: - return raw_output - else: - return _process(raw_output) - # detect if linux style was used elif cleandata[0].startswith('Address'): @@ -235,7 +230,4 @@ def parse(data, raw=False, quiet=False): } raw_output.append(output_line) - if raw: - return raw_output - else: - return _process(raw_output) + return raw_output if raw else _process(raw_output) From 8bfbf8f1bcd233a8ecd34012e140eb29164ca5fb Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 12:13:07 -0800 Subject: [PATCH 02/87] simplify error message --- jc/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/cli.py b/jc/cli.py index 149f5c6b..8bb3d846 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -452,7 +452,7 @@ def main(): error_msg = os.strerror(e.errno) utils.error_message([ - f'"{run_command_str}" command could not be run: {error_msg}. For details use the -d or -dd option.' + f'"{run_command_str}" command could not be run: {error_msg}.' ]) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) From 01f2c1e71f87ab401d487be4d6d150cb67c91437 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 12:13:32 -0800 Subject: [PATCH 03/87] add pidstat parser --- CHANGELOG | 3 + jc/lib.py | 3 +- jc/parsers/pidstat.py | 112 ++++++++++++++++++++++++++ setup.py | 2 +- tests/fixtures/centos-7.7/pidstat.out | 44 ++++++++++ 5 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 jc/parsers/pidstat.py create mode 100644 tests/fixtures/centos-7.7/pidstat.out diff --git a/CHANGELOG b/CHANGELOG index a97b7eab..163f9ced 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,8 @@ jc changelog +20220309 v1.18.6 (in progress) +- Add pidstat parser tested on linux + 20220305 v1.18.5 - Fix date parser to ensure AM/PM period string is always uppercase diff --git a/jc/lib.py b/jc/lib.py index ca025e08..678da5f4 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -9,7 +9,7 @@ import importlib from typing import Dict, List, Iterable, Union, Iterator from jc import appdirs -__version__ = '1.18.5' +__version__ = '1.18.6' parsers = [ 'acpi', @@ -63,6 +63,7 @@ parsers = [ 'nmcli', 'ntpq', 'passwd', + 'pidstat', 'ping', 'ping-s', 'pip-list', diff --git a/jc/parsers/pidstat.py b/jc/parsers/pidstat.py new file mode 100644 index 00000000..5c808614 --- /dev/null +++ b/jc/parsers/pidstat.py @@ -0,0 +1,112 @@ +"""jc - JSON Convert `pidstat` command output parser + +<> + +Usage (cli): + + $ pidstat | jc --pidstat + + or + + $ jc pidstat + +Usage (module): + + import jc + result = jc.parse('pidstat', pidstat_command_output) + + or + + import jc.parsers.pidstat + result = jc.parsers.pidstat.parse(pidstat_command_output) + +Schema: + + [ + { + "pidstat": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ pidstat | jc --pidstat -p + [] + + $ pidstat | jc --pidstat -p -r + [] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`pidstat` command parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + magic_commands = ['pidstat'] + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + + # process the data here + # rebuild output for added semantic information + # use helper functions in jc.utils for int, float, bool + # conversions and timestamps + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + # parse the content here + # check out helper functions in jc.utils + # and jc.parsers.universal + + pass + + return raw_output if raw else _process(raw_output) diff --git a/setup.py b/setup.py index 667d3d33..187a3a79 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='1.18.5', + version='1.18.6', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='Converts the output of popular command-line tools and file-types to JSON.', diff --git a/tests/fixtures/centos-7.7/pidstat.out b/tests/fixtures/centos-7.7/pidstat.out new file mode 100644 index 00000000..0b46a92a --- /dev/null +++ b/tests/fixtures/centos-7.7/pidstat.out @@ -0,0 +1,44 @@ +Linux 3.10.0-1062.1.2.el7.x86_64 (localhost) 03/09/2022 _x86_64_ (1 CPU) + +12:06:39 PM UID PID %usr %system %guest %CPU CPU Command +12:06:39 PM 0 1 0.00 0.05 0.00 0.05 0 systemd +12:06:39 PM 0 6 0.00 0.00 0.00 0.00 0 ksoftirqd/0 +12:06:39 PM 0 9 0.00 0.01 0.00 0.01 0 rcu_sched +12:06:39 PM 0 11 0.00 0.00 0.00 0.00 0 watchdog/0 +12:06:39 PM 0 32 0.00 0.00 0.00 0.00 0 khugepaged +12:06:39 PM 0 308 0.00 0.00 0.00 0.00 0 scsi_eh_1 +12:06:39 PM 0 309 0.00 0.00 0.00 0.00 0 kworker/u256:2 +12:06:39 PM 0 319 0.00 0.00 0.00 0.00 0 scsi_eh_2 +12:06:39 PM 0 357 0.00 0.00 0.00 0.00 0 irq/16-vmwgfx +12:06:39 PM 0 465 0.00 0.01 0.00 0.01 0 xfsaild/dm-0 +12:06:39 PM 0 466 0.00 0.00 0.00 0.00 0 kworker/0:1H +12:06:39 PM 0 543 0.00 0.00 0.00 0.01 0 systemd-journal +12:06:39 PM 0 564 0.00 0.00 0.00 0.00 0 lvmetad +12:06:39 PM 0 577 0.00 0.00 0.00 0.01 0 systemd-udevd +12:06:39 PM 0 752 0.00 0.00 0.00 0.00 0 auditd +12:06:39 PM 0 779 0.00 0.00 0.00 0.00 0 bluetoothd +12:06:39 PM 999 780 0.00 0.00 0.00 0.00 0 polkitd +12:06:39 PM 0 782 0.00 0.00 0.00 0.00 0 smartd +12:06:39 PM 0 784 0.00 0.00 0.00 0.00 0 systemd-logind +12:06:39 PM 81 787 0.00 0.00 0.00 0.01 0 dbus-daemon +12:06:39 PM 998 790 0.00 0.00 0.00 0.00 0 chronyd +12:06:39 PM 0 834 0.00 0.01 0.00 0.01 0 crond +12:06:39 PM 0 847 0.01 0.01 0.00 0.02 0 firewalld +12:06:39 PM 0 849 0.00 0.00 0.00 0.00 0 agetty +12:06:39 PM 0 852 0.00 0.00 0.00 0.00 0 login +12:06:39 PM 0 882 0.00 0.00 0.00 0.01 0 NetworkManager +12:06:39 PM 0 1031 0.00 0.00 0.00 0.00 0 dhclient +12:06:39 PM 0 1220 0.00 0.00 0.00 0.00 0 sshd +12:06:39 PM 0 1221 0.06 0.03 0.00 0.09 0 dockerd-current +12:06:39 PM 0 1222 0.01 0.01 0.00 0.02 0 tuned +12:06:39 PM 0 1225 0.00 0.01 0.00 0.01 0 rsyslogd +12:06:39 PM 0 1293 0.04 0.02 0.00 0.05 0 docker-containe +12:06:39 PM 0 1496 0.00 0.00 0.00 0.00 0 master +12:06:39 PM 89 1511 0.00 0.00 0.00 0.00 0 pickup +12:06:39 PM 1000 1852 0.00 0.00 0.00 0.00 0 bash +12:06:39 PM 0 1872 0.00 0.00 0.00 0.00 0 sshd +12:06:39 PM 1000 1876 0.00 0.00 0.00 0.00 0 sshd +12:06:39 PM 1000 1877 0.00 0.00 0.00 0.00 0 bash +12:06:39 PM 0 2062 0.00 0.01 0.00 0.01 0 kworker/0:0 +12:06:39 PM 0 2085 0.00 0.00 0.00 0.00 0 kworker/0:1 +12:06:39 PM 1000 2123 0.00 0.00 0.00 0.00 0 pidstat From 3389eb5debe0f0ba01ece01e20048608bf357e70 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 13:18:04 -0800 Subject: [PATCH 04/87] initial working parser --- jc/parsers/pidstat.py | 52 ++++++++++-- .../centos-7.7/pidstat-hdlrsuw-2-5.out | 47 +++++++++++ tests/fixtures/centos-7.7/pidstat-hdlrsuw.out | 83 +++++++++++++++++++ tests/fixtures/centos-7.7/pidstat-hl.out | 46 ++++++++++ 4 files changed, 219 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out create mode 100644 tests/fixtures/centos-7.7/pidstat-hdlrsuw.out create mode 100644 tests/fixtures/centos-7.7/pidstat-hl.out diff --git a/jc/parsers/pidstat.py b/jc/parsers/pidstat.py index 5c808614..22e7b6f4 100644 --- a/jc/parsers/pidstat.py +++ b/jc/parsers/pidstat.py @@ -1,6 +1,6 @@ """jc - JSON Convert `pidstat` command output parser -<> +Must use the `-h` option in `pidstat`. Usage (cli): @@ -24,9 +24,27 @@ Schema: [ { - "pidstat": string, - "bar": boolean, - "baz": integer + "time": "1646857494", + "uid": "1000", + "pid": "2201", + "percent_usr": "0.00", + "percent_system": "0.00", + "percent_guest": "0.00", + "percent_cpu": "0.00", + "cpu": "0", + "minflt_s": "0.09", + "majflt_s": "0.00", + "vsz": "108328", + "rss": "1040", + "percent_mem": "0.03", + "stksize": "132", + "stkref": "20", + "kb_rd_s": "0.00", + "kb_wr_s": "0.00", + "kb_ccwr_s": "0.00", + "cswch_s": "0.00", + "nvcswch_s": "0.00", + "command": "pidstat -dlrsuwh" } ] @@ -40,6 +58,8 @@ Examples: """ from typing import List, Dict import jc.utils +from jc.parsers.universal import simple_table_parse +from jc.exceptions import ParseError class info(): @@ -101,12 +121,26 @@ def parse( if jc.utils.has_data(data): - for line in filter(None, data.splitlines()): + # check for line starting with # as the start of the table + data_list = list(filter(None, data.splitlines())) + for line in data_list.copy(): + if line.startswith('#'): + break + else: + data_list.pop(0) - # parse the content here - # check out helper functions in jc.utils - # and jc.parsers.universal + if not data_list: + raise ParseError('Could not parse pidstat output. Make sure to use "pidstat -h".') - pass + # normalize headers + data_list[0] = data_list[0].replace('#', ' ')\ + .replace('/', '_')\ + .replace('%', 'percent_')\ + .lower() + + # remove remaining header lines (e.g. pidstat -h 2 5) + data_list = [i for i in data_list if not i.startswith('#')] + + raw_output = simple_table_parse(data_list) return raw_output if raw else _process(raw_output) diff --git a/tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out b/tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out new file mode 100644 index 00000000..5096001a --- /dev/null +++ b/tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out @@ -0,0 +1,47 @@ +Linux 3.10.0-1062.1.2.el7.x86_64 (localhost) 03/09/2022 _x86_64_ (1 CPU) + +# Time UID PID %usr %system %guest %CPU CPU minflt/s majflt/s VSZ RSS %MEM StkSize StkRef kB_rd/s kB_wr/s kB_ccwr/s cswch/s nvcswch/s Command + 1646859221 0 6 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 1.49 0.00 ksoftirqd/0 + 1646859221 0 9 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 1.98 0.00 rcu_sched + 1646859221 0 11 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.50 0.00 watchdog/0 + 1646859221 0 356 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.99 0.00 kworker/u256:4 + 1646859221 0 466 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.50 0.00 kworker/0:1H + 1646859221 0 2232 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 2.97 0.00 kworker/0:1 + 1646859221 0 2263 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.50 0.00 kworker/0:0 + 1646859221 1000 2297 0.00 0.50 0.00 0.50 0 74.75 0.00 108296 1072 0.03 132 20 0.00 0.00 0.00 0.50 0.50 pidstat -dlrsuwh 2 5 + +# Time UID PID %usr %system %guest %CPU CPU minflt/s majflt/s VSZ RSS %MEM StkSize StkRef kB_rd/s kB_wr/s kB_ccwr/s cswch/s nvcswch/s Command + 1646859223 0 6 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 1.00 0.00 ksoftirqd/0 + 1646859223 0 9 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 2.00 0.00 rcu_sched + 1646859223 0 356 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 1.00 0.00 kworker/u256:4 + 1646859223 0 2232 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 3.00 0.00 kworker/0:1 + 1646859223 0 2263 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.50 0.00 kworker/0:0 + 1646859223 1000 2297 0.50 0.50 0.00 1.00 0 82.50 0.00 108296 1152 0.03 132 24 0.00 0.00 0.00 0.50 0.50 pidstat -dlrsuwh 2 5 + +# Time UID PID %usr %system %guest %CPU CPU minflt/s majflt/s VSZ RSS %MEM StkSize StkRef kB_rd/s kB_wr/s kB_ccwr/s cswch/s nvcswch/s Command + 1646859225 0 6 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 1.01 0.00 ksoftirqd/0 + 1646859225 0 9 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 2.02 0.00 rcu_sched + 1646859225 0 11 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.51 0.00 watchdog/0 + 1646859225 0 32 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.51 0.00 khugepaged + 1646859225 0 356 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 1.01 0.00 kworker/u256:4 + 1646859225 0 2232 0.00 0.51 0.00 0.51 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 3.03 0.00 kworker/0:1 + 1646859225 0 2263 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.51 0.00 kworker/0:0 + 1646859225 1000 2297 0.00 0.00 0.00 0.00 0 73.23 0.00 108296 1152 0.03 132 24 0.00 0.00 0.00 0.51 0.51 pidstat -dlrsuwh 2 5 + +# Time UID PID %usr %system %guest %CPU CPU minflt/s majflt/s VSZ RSS %MEM StkSize StkRef kB_rd/s kB_wr/s kB_ccwr/s cswch/s nvcswch/s Command + 1646859227 0 6 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 1.00 0.00 ksoftirqd/0 + 1646859227 0 9 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 2.00 0.00 rcu_sched + 1646859227 0 356 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 1.00 0.00 kworker/u256:4 + 1646859227 0 2232 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 3.00 0.00 kworker/0:1 + 1646859227 0 2263 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.50 0.00 kworker/0:0 + 1646859227 1000 2297 0.50 0.50 0.00 1.00 0 72.50 0.00 108296 1152 0.03 132 24 0.00 0.00 0.00 0.50 0.50 pidstat -dlrsuwh 2 5 + +# Time UID PID %usr %system %guest %CPU CPU minflt/s majflt/s VSZ RSS %MEM StkSize StkRef kB_rd/s kB_wr/s kB_ccwr/s cswch/s nvcswch/s Command + 1646859229 0 6 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 1.00 0.00 ksoftirqd/0 + 1646859229 0 9 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 3.48 0.00 rcu_sched + 1646859229 0 11 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.50 0.00 watchdog/0 + 1646859229 0 356 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 1.00 0.00 kworker/u256:4 + 1646859229 0 466 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.50 0.00 kworker/0:1H + 1646859229 0 2232 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 2.99 0.00 kworker/0:1 + 1646859229 0 2263 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.50 0.00 kworker/0:0 + 1646859229 1000 2297 0.00 0.50 0.00 0.50 0 72.14 0.00 108296 1152 0.03 132 24 0.00 0.00 0.00 0.50 0.00 pidstat -dlrsuwh 2 5 diff --git a/tests/fixtures/centos-7.7/pidstat-hdlrsuw.out b/tests/fixtures/centos-7.7/pidstat-hdlrsuw.out new file mode 100644 index 00000000..ddbd6c27 --- /dev/null +++ b/tests/fixtures/centos-7.7/pidstat-hdlrsuw.out @@ -0,0 +1,83 @@ +Linux 3.10.0-1062.1.2.el7.x86_64 (localhost) 03/09/2022 _x86_64_ (1 CPU) + +# Time UID PID %usr %system %guest %CPU CPU minflt/s majflt/s VSZ RSS %MEM StkSize StkRef kB_rd/s kB_wr/s kB_ccwr/s cswch/s nvcswch/s Command + 1646857494 0 2 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.03 0.00 kthreadd + 1646857494 0 4 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kworker/0:0H + 1646857494 0 6 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.77 0.00 ksoftirqd/0 + 1646857494 0 7 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.48 0.00 migration/0 + 1646857494 0 8 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 rcu_bh + 1646857494 0 9 0.00 0.01 0.00 0.01 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 4.93 0.00 rcu_sched + 1646857494 0 10 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 lru-add-drain + 1646857494 0 11 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.25 0.00 watchdog/0 + 1646857494 0 13 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.03 0.00 kdevtmpfs + 1646857494 0 14 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 netns + 1646857494 0 15 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.01 0.00 khungtaskd + 1646857494 0 16 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 writeback + 1646857494 0 17 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kintegrityd + 1646857494 0 18 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 bioset + 1646857494 0 19 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 bioset + 1646857494 0 20 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 bioset + 1646857494 0 21 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kblockd + 1646857494 0 22 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 md + 1646857494 0 23 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 edac-poller + 1646857494 0 24 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 watchdogd + 1646857494 0 30 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kswapd0 + 1646857494 0 31 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 ksmd + 1646857494 0 32 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.10 0.00 khugepaged + 1646857494 0 33 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 crypto + 1646857494 0 41 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kthrotld + 1646857494 0 43 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kmpath_rdacd + 1646857494 0 44 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kaluad + 1646857494 0 45 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kpsmoused + 1646857494 0 47 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 ipv6_addrconf + 1646857494 0 60 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 deferwq + 1646857494 0 95 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.09 0.00 kauditd + 1646857494 0 272 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 mpt_poll_0 + 1646857494 0 273 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 mpt/0 + 1646857494 0 274 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 nfit + 1646857494 0 275 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 ata_sff + 1646857494 0 285 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 scsi_eh_0 + 1646857494 0 288 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 scsi_tmf_0 + 1646857494 0 308 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 scsi_eh_1 + 1646857494 0 309 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.88 0.00 kworker/u256:2 + 1646857494 0 314 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 scsi_tmf_1 + 1646857494 0 319 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.01 0.00 scsi_eh_2 + 1646857494 0 323 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 scsi_tmf_2 + 1646857494 0 356 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.19 0.00 kworker/u256:4 + 1646857494 0 357 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.63 0.00 irq/16-vmwgfx + 1646857494 0 359 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 ttm_swap + 1646857494 0 431 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kdmflush + 1646857494 0 432 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 bioset + 1646857494 0 441 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kdmflush + 1646857494 0 442 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 bioset + 1646857494 0 455 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 bioset + 1646857494 0 456 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfsalloc + 1646857494 0 457 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs_mru_cache + 1646857494 0 458 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-buf/dm-0 + 1646857494 0 459 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-data/dm-0 + 1646857494 0 460 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-conv/dm-0 + 1646857494 0 461 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-cil/dm-0 + 1646857494 0 462 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-reclaim/dm- + 1646857494 0 463 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-log/dm-0 + 1646857494 0 464 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-eofblocks/d + 1646857494 0 465 0.00 0.01 0.00 0.01 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 3.04 0.00 xfsaild/dm-0 + 1646857494 0 466 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.20 0.00 kworker/0:1H + 1646857494 0 698 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 kworker/u257:0 + 1646857494 0 700 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 hci0 + 1646857494 0 701 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 hci0 + 1646857494 0 702 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.01 0.00 kworker/u257:1 + 1646857494 0 725 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-buf/sda1 + 1646857494 0 726 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-data/sda1 + 1646857494 0 727 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-conv/sda1 + 1646857494 0 728 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-cil/sda1 + 1646857494 0 729 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-reclaim/sda + 1646857494 0 730 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-log/sda1 + 1646857494 0 731 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfs-eofblocks/s + 1646857494 0 732 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.00 0.00 xfsaild/sda1 + 1646857494 1000 1852 0.00 0.00 0.00 0.00 0 0.25 0.00 115708 2324 0.06 264 264 0.08 0.00 0.00 0.01 0.00 -bash + 1646857494 1000 1877 0.00 0.00 0.00 0.00 0 0.33 0.00 115580 2216 0.06 132 24 0.64 0.00 0.00 0.03 0.00 -bash + 1646857494 0 2062 0.00 0.03 0.00 0.03 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.95 0.00 kworker/0:0 + 1646857494 0 2153 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.03 0.00 kworker/0:1 + 1646857494 0 2199 0.00 0.00 0.00 0.00 0 0.00 0.00 0 0 0.00 0 0 -1.00 -1.00 -1.00 0.03 0.00 kworker/0:2 + 1646857494 1000 2201 0.00 0.00 0.00 0.00 0 0.09 0.00 108328 1040 0.03 132 20 0.00 0.00 0.00 0.00 0.00 pidstat -dlrsuwh + diff --git a/tests/fixtures/centos-7.7/pidstat-hl.out b/tests/fixtures/centos-7.7/pidstat-hl.out new file mode 100644 index 00000000..b8924bbf --- /dev/null +++ b/tests/fixtures/centos-7.7/pidstat-hl.out @@ -0,0 +1,46 @@ +Linux 3.10.0-1062.1.2.el7.x86_64 (localhost) 03/09/2022 _x86_64_ (1 CPU) + +# Time UID PID %usr %system %guest %CPU CPU Command + 1646859134 0 1 0.00 0.03 0.00 0.03 0 /usr/lib/systemd/systemd --switched-root --system --deserialize 22 + 1646859134 0 6 0.00 0.00 0.00 0.00 0 ksoftirqd/0 + 1646859134 0 9 0.00 0.01 0.00 0.01 0 rcu_sched + 1646859134 0 11 0.00 0.00 0.00 0.00 0 watchdog/0 + 1646859134 0 32 0.00 0.00 0.00 0.00 0 khugepaged + 1646859134 0 308 0.00 0.00 0.00 0.00 0 scsi_eh_1 + 1646859134 0 309 0.00 0.00 0.00 0.00 0 kworker/u256:2 + 1646859134 0 319 0.00 0.00 0.00 0.00 0 scsi_eh_2 + 1646859134 0 356 0.00 0.00 0.00 0.00 0 kworker/u256:4 + 1646859134 0 357 0.00 0.00 0.00 0.00 0 irq/16-vmwgfx + 1646859134 0 465 0.00 0.01 0.00 0.01 0 xfsaild/dm-0 + 1646859134 0 466 0.00 0.00 0.00 0.00 0 kworker/0:1H + 1646859134 0 543 0.00 0.00 0.00 0.00 0 /usr/lib/systemd/systemd-journald + 1646859134 0 564 0.00 0.00 0.00 0.00 0 /usr/sbin/lvmetad -f + 1646859134 0 577 0.00 0.00 0.00 0.00 0 /usr/lib/systemd/systemd-udevd + 1646859134 0 752 0.00 0.00 0.00 0.00 0 /sbin/auditd + 1646859134 0 779 0.00 0.00 0.00 0.00 0 /usr/libexec/bluetooth/bluetoothd + 1646859134 999 780 0.00 0.00 0.00 0.00 0 /usr/lib/polkit-1/polkitd --no-debug + 1646859134 0 782 0.00 0.00 0.00 0.00 0 /usr/sbin/smartd -n -q never + 1646859134 0 784 0.00 0.00 0.00 0.00 0 /usr/lib/systemd/systemd-logind + 1646859134 81 787 0.00 0.00 0.00 0.00 0 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation + 1646859134 998 790 0.00 0.00 0.00 0.00 0 /usr/sbin/chronyd + 1646859134 0 834 0.00 0.01 0.00 0.01 0 /usr/sbin/crond -n + 1646859134 0 847 0.01 0.00 0.00 0.01 0 /usr/bin/python2 -Es /usr/sbin/firewalld --nofork --nopid + 1646859134 0 849 0.00 0.00 0.00 0.00 0 /sbin/agetty --keep-baud 115200,38400,9600 ttyS0 vt220 + 1646859134 0 852 0.00 0.00 0.00 0.00 0 login -- kbrazil + 1646859134 0 882 0.00 0.00 0.00 0.00 0 /usr/sbin/NetworkManager --no-daemon + 1646859134 0 1031 0.00 0.00 0.00 0.00 0 /sbin/dhclient -d -q -sf /usr/libexec/nm-dhcp-helper -pf /var/run/dhclient-ens33.pid -lf /var/lib/NetworkManager/dhclient-d92ec + 1646859134 0 1220 0.00 0.00 0.00 0.00 0 /usr/sbin/sshd -D + 1646859134 0 1221 0.06 0.03 0.00 0.09 0 /usr/bin/dockerd-current --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current --default-runtime=docker-runc --exec + 1646859134 0 1222 0.01 0.00 0.00 0.01 0 /usr/bin/python2 -Es /usr/sbin/tuned -l -P + 1646859134 0 1225 0.00 0.00 0.00 0.01 0 /usr/sbin/rsyslogd -n + 1646859134 0 1293 0.04 0.02 0.00 0.05 0 /usr/bin/docker-containerd-current -l unix:///var/run/docker/libcontainerd/docker-containerd.sock --metrics-interval=0 --start- + 1646859134 0 1496 0.00 0.00 0.00 0.00 0 /usr/libexec/postfix/master -w + 1646859134 89 1512 0.00 0.00 0.00 0.00 0 qmgr -l -t unix -u + 1646859134 1000 1852 0.00 0.00 0.00 0.00 0 -bash + 1646859134 0 1872 0.00 0.00 0.00 0.00 0 sshd: kbrazil [priv] + 1646859134 1000 1876 0.00 0.00 0.00 0.00 0 sshd: kbrazil@pts/0 + 1646859134 1000 1877 0.00 0.00 0.00 0.00 0 -bash + 1646859134 0 2232 0.00 0.01 0.00 0.01 0 kworker/0:1 + 1646859134 0 2239 0.00 0.00 0.00 0.00 0 kworker/0:3 + 1646859134 89 2240 0.00 0.00 0.00 0.00 0 pickup -l -t unix -u + 1646859134 0 2263 0.00 0.00 0.00 0.00 0 kworker/0:0 From ae754a84bf5fc35b248bbc5f7b0fa8e871765819 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 14:06:34 -0800 Subject: [PATCH 05/87] doc update --- docs/parsers/arp.md | 2 +- docs/parsers/pidstat.md | 156 ++++++++++++++++++++++++++++++++++++++++ jc/parsers/pidstat.py | 140 +++++++++++++++++++++++++++--------- 3 files changed, 264 insertions(+), 34 deletions(-) create mode 100644 docs/parsers/pidstat.md diff --git a/docs/parsers/arp.md b/docs/parsers/arp.md index ad76015f..c036899f 100644 --- a/docs/parsers/arp.md +++ b/docs/parsers/arp.md @@ -145,4 +145,4 @@ Returns: ### Parser Information Compatibility: linux, aix, freebsd, darwin -Version 1.8 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.9 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/pidstat.md b/docs/parsers/pidstat.md new file mode 100644 index 00000000..c2c946b5 --- /dev/null +++ b/docs/parsers/pidstat.md @@ -0,0 +1,156 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.pidstat + +jc - JSON Convert `pidstat` command output parser + +Must use the `-h` option in `pidstat`. All other `pidstat` options are +supported in combination with `-h`. + +Usage (cli): + + $ pidstat -h | jc --pidstat + + or + + $ jc pidstat -h + +Usage (module): + + import jc + result = jc.parse('pidstat', pidstat_command_output) + + or + + import jc.parsers.pidstat + result = jc.parsers.pidstat.parse(pidstat_command_output) + +Schema: + + [ + { + "time": integer, + "uid": integer, + "pid": integer, + "percent_usr": float, + "percent_system": float, + "percent_guest": float, + "percent_cpu": float, + "cpu": integer, + "minflt_s": float, + "majflt_s": float, + "vsz": integer, + "rss": integer, + "percent_mem": float, + "stksize": integer, + "stkref": integer, + "kb_rd_s": float, + "kb_wr_s": float, + "kb_ccwr_s": float, + "cswch_s": float, + "nvcswch_s": float, + "command": string + } + ] + +Examples: + + $ pidstat -hl | jc --pidstat -p + [ + { + "time": 1646859134, + "uid": 0, + "pid": 1, + "percent_usr": 0.0, + "percent_system": 0.03, + "percent_guest": 0.0, + "percent_cpu": 0.03, + "cpu": 0, + "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + }, + { + "time": 1646859134, + "uid": 0, + "pid": 6, + "percent_usr": 0.0, + "percent_system": 0.0, + "percent_guest": 0.0, + "percent_cpu": 0.0, + "cpu": 0, + "command": "ksoftirqd/0" + }, + { + "time": 1646859134, + "uid": 0, + "pid": 2263, + "percent_usr": 0.0, + "percent_system": 0.0, + "percent_guest": 0.0, + "percent_cpu": 0.0, + "cpu": 0, + "command": "kworker/0:0" + } + ] + + $ pidstat -hl | jc --pidstat -p -r + [ + { + "time": "1646859134", + "uid": "0", + "pid": "1", + "percent_usr": "0.00", + "percent_system": "0.03", + "percent_guest": "0.00", + "percent_cpu": "0.03", + "cpu": "0", + "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + }, + { + "time": "1646859134", + "uid": "0", + "pid": "6", + "percent_usr": "0.00", + "percent_system": "0.00", + "percent_guest": "0.00", + "percent_cpu": "0.00", + "cpu": "0", + "command": "ksoftirqd/0" + }, + { + "time": "1646859134", + "uid": "0", + "pid": "2263", + "percent_usr": "0.00", + "percent_system": "0.00", + "percent_guest": "0.00", + "percent_cpu": "0.00", + "cpu": "0", + "command": "kworker/0:0" + } + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/pidstat.py b/jc/parsers/pidstat.py index 22e7b6f4..5b6e79da 100644 --- a/jc/parsers/pidstat.py +++ b/jc/parsers/pidstat.py @@ -1,14 +1,15 @@ """jc - JSON Convert `pidstat` command output parser -Must use the `-h` option in `pidstat`. +Must use the `-h` option in `pidstat`. All other `pidstat` options are +supported in combination with `-h`. Usage (cli): - $ pidstat | jc --pidstat + $ pidstat -h | jc --pidstat or - $ jc pidstat + $ jc pidstat -h Usage (module): @@ -24,37 +25,105 @@ Schema: [ { - "time": "1646857494", - "uid": "1000", - "pid": "2201", - "percent_usr": "0.00", - "percent_system": "0.00", - "percent_guest": "0.00", - "percent_cpu": "0.00", - "cpu": "0", - "minflt_s": "0.09", - "majflt_s": "0.00", - "vsz": "108328", - "rss": "1040", - "percent_mem": "0.03", - "stksize": "132", - "stkref": "20", - "kb_rd_s": "0.00", - "kb_wr_s": "0.00", - "kb_ccwr_s": "0.00", - "cswch_s": "0.00", - "nvcswch_s": "0.00", - "command": "pidstat -dlrsuwh" + "time": integer, + "uid": integer, + "pid": integer, + "percent_usr": float, + "percent_system": float, + "percent_guest": float, + "percent_cpu": float, + "cpu": integer, + "minflt_s": float, + "majflt_s": float, + "vsz": integer, + "rss": integer, + "percent_mem": float, + "stksize": integer, + "stkref": integer, + "kb_rd_s": float, + "kb_wr_s": float, + "kb_ccwr_s": float, + "cswch_s": float, + "nvcswch_s": float, + "command": string } ] Examples: - $ pidstat | jc --pidstat -p - [] + $ pidstat -hl | jc --pidstat -p + [ + { + "time": 1646859134, + "uid": 0, + "pid": 1, + "percent_usr": 0.0, + "percent_system": 0.03, + "percent_guest": 0.0, + "percent_cpu": 0.03, + "cpu": 0, + "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + }, + { + "time": 1646859134, + "uid": 0, + "pid": 6, + "percent_usr": 0.0, + "percent_system": 0.0, + "percent_guest": 0.0, + "percent_cpu": 0.0, + "cpu": 0, + "command": "ksoftirqd/0" + }, + { + "time": 1646859134, + "uid": 0, + "pid": 2263, + "percent_usr": 0.0, + "percent_system": 0.0, + "percent_guest": 0.0, + "percent_cpu": 0.0, + "cpu": 0, + "command": "kworker/0:0" + } + ] - $ pidstat | jc --pidstat -p -r - [] + $ pidstat -hl | jc --pidstat -p -r + [ + { + "time": "1646859134", + "uid": "0", + "pid": "1", + "percent_usr": "0.00", + "percent_system": "0.03", + "percent_guest": "0.00", + "percent_cpu": "0.03", + "cpu": "0", + "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + }, + { + "time": "1646859134", + "uid": "0", + "pid": "6", + "percent_usr": "0.00", + "percent_system": "0.00", + "percent_guest": "0.00", + "percent_cpu": "0.00", + "cpu": "0", + "command": "ksoftirqd/0" + }, + { + "time": "1646859134", + "uid": "0", + "pid": "2263", + "percent_usr": "0.00", + "percent_system": "0.00", + "percent_guest": "0.00", + "percent_cpu": "0.00", + "cpu": "0", + "command": "kworker/0:0" + } + ] """ from typing import List, Dict import jc.utils @@ -87,11 +156,16 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - - # process the data here - # rebuild output for added semantic information - # use helper functions in jc.utils for int, float, bool - # conversions and timestamps + int_list = ['time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'] + float_list = ['percent_usr', 'percent_system', 'percent_guest', 'percent_cpu', + 'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s', + 'kb_ccwr_s', 'cswch_s', 'nvcswch_s'] + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + if key in float_list: + entry[key] = jc.utils.convert_to_float(entry[key]) return proc_data From bbd4afa73542ba6596aed77426bb494fbcf23196 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 14:09:58 -0800 Subject: [PATCH 06/87] add pidstat to docs --- README.md | 1 + man/jc.1 | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a5db4980..3732ab20 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ option. - `--nmcli` enables the `nmcli` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/nmcli)) - `--ntpq` enables the `ntpq -p` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ntpq)) - `--passwd` enables the `/etc/passwd` file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/passwd)) +- `--pidstat` enables the `pidstat` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/pidstat)) - `--ping` enables the `ping` and `ping6` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ping)) - `--ping-s` enables the `ping` and `ping6` command streaming parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ping_s)) - `--pip-list` enables the `pip list` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_list)) diff --git a/man/jc.1 b/man/jc.1 index 7982a291..c223da4d 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-03-05 1.18.5 "JSON Convert" +.TH jc 1 2022-03-09 1.18.6 "JSON Convert" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -272,6 +272,11 @@ Key/Value file parser \fB--passwd\fP `/etc/passwd` file parser +.TP +.B +\fB--pidstat\fP +`pidstat` command parser + .TP .B \fB--ping\fP From c3198a58744803f52113f80967c7259d781e2328 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 14:17:48 -0800 Subject: [PATCH 07/87] formatting --- man/jc.1 | 17 +++++++++++------ templates/manpage_template | 17 +++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/man/jc.1 b/man/jc.1 index c223da4d..04710341 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -600,7 +600,7 @@ Most operating systems will buffer output that is being piped from process to pr .RS .na .nf -$ ping 1.1.1.1 | jc --ping-s | jq +$ ping 1.1.1.1 | jc \fB--ping-s\fP | jq .fi .RE @@ -610,7 +610,7 @@ This is because the OS engages the 4KB buffer between \fBjc\fP and \fBjq\fP in t .RS .na .nf -$ ping 1.1.1.1 | jc --ping-s -u | jq +$ ping 1.1.1.1 | jc \fB--ping-s\fP \fB-u\fP | jq {"type":"reply","pattern":null,"timestamp":null,"bytes":"64",...} {"type":"reply","pattern":null,"timestamp":null,"bytes":"64",...} etc... @@ -623,11 +623,12 @@ Note: Unbuffered output can be slower for large data streams. Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your local "App data directory": .RS +.na +.nf - Linux/unix: \fB$HOME/.local/share/jc/jcparsers\fP - - macOS: \fB$HOME/Library/Application Support/jc/jcparsers\fP - - Windows: \fB$LOCALAPPDATA\\jc\\jc\\jcparsers\fP +.fi .RE Local parser plugins are standard python module files. Use the \fBjc/parsers/foo.py\fP parser as a template and simply place a \fB.py\fP file in the \fBjcparsers\fP subfolder. @@ -639,11 +640,15 @@ Note: The application data directory follows the XDG Base Directory Specificatio .SH CAVEATS \fBLocale:\fP For best results set the \fBLANG\fP locale environment variable to \fBC\fP or \fBen_US.UTF-8\fP. For example, either by setting directly on the command-line: -\fB$ LANG=C date | jc --date\fP +.RS +$ LANG=C date | jc \fB--date\fP +.RE or by exporting to the environment before running commands: -\fB$ export LANG=C\fP +.RS +$ export LANG=C +.RE \fBTimezones:\fP Some parsers have calculated epoch timestamp fields added to the output. Unless a timestamp field name has a \fB_utc\fP suffix it is considered naive. (i.e. based on the local timezone of the system the \fBjc\fP parser was run on). diff --git a/templates/manpage_template b/templates/manpage_template index fe1bc4fc..f37ccdbc 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -155,7 +155,7 @@ Most operating systems will buffer output that is being piped from process to pr .RS .na .nf -$ ping 1.1.1.1 | jc --ping-s | jq +$ ping 1.1.1.1 | jc \fB--ping-s\fP | jq .fi .RE @@ -165,7 +165,7 @@ This is because the OS engages the 4KB buffer between \fBjc\fP and \fBjq\fP in t .RS .na .nf -$ ping 1.1.1.1 | jc --ping-s -u | jq +$ ping 1.1.1.1 | jc \fB--ping-s\fP \fB-u\fP | jq {"type":"reply","pattern":null,"timestamp":null,"bytes":"64",...} {"type":"reply","pattern":null,"timestamp":null,"bytes":"64",...} etc... @@ -178,11 +178,12 @@ Note: Unbuffered output can be slower for large data streams. Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your local "App data directory": .RS +.na +.nf - Linux/unix: \fB$HOME/.local/share/jc/jcparsers\fP - - macOS: \fB$HOME/Library/Application Support/jc/jcparsers\fP - - Windows: \fB$LOCALAPPDATA\\jc\\jc\\jcparsers\fP +.fi .RE Local parser plugins are standard python module files. Use the \fBjc/parsers/foo.py\fP parser as a template and simply place a \fB.py\fP file in the \fBjcparsers\fP subfolder. @@ -194,11 +195,15 @@ Note: The application data directory follows the XDG Base Directory Specificatio .SH CAVEATS \fBLocale:\fP For best results set the \fBLANG\fP locale environment variable to \fBC\fP or \fBen_US.UTF-8\fP. For example, either by setting directly on the command-line: -\fB$ LANG=C date | jc --date\fP +.RS +$ LANG=C date | jc \fB--date\fP +.RE or by exporting to the environment before running commands: -\fB$ export LANG=C\fP +.RS +$ export LANG=C +.RE \fBTimezones:\fP Some parsers have calculated epoch timestamp fields added to the output. Unless a timestamp field name has a \fB_utc\fP suffix it is considered naive. (i.e. based on the local timezone of the system the \fBjc\fP parser was run on). From f5c7d52ec72f5ff4b77d5ac4491f330575b9f10f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 14:43:56 -0800 Subject: [PATCH 08/87] formatting --- man/jc.1 | 6 +----- templates/manpage_template | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/man/jc.1 b/man/jc.1 index 04710341..cd19fb2a 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -568,7 +568,6 @@ You may want to ignore parsing errors when using streaming parsers since these m .RS Successfully parsed line with \fB-qq\fP option: .RS -.na .nf { "command_data": "data", @@ -576,11 +575,11 @@ Successfully parsed line with \fB-qq\fP option: "success": true } } +.fi .RE Unsuccessfully parsed line with \fB-qq\fP option: .RS -.na .nf { "_jc_meta": { @@ -598,7 +597,6 @@ Unsuccessfully parsed line with \fB-qq\fP option: Most operating systems will buffer output that is being piped from process to process. The buffer is usually around 4KB. When viewing the output in the terminal the OS buffer is not engaged so output is immediately displayed on the screen. When piping multiple processes together, though, it may seem as if the output is hanging when the input data is very slow (e.g. \fBping\fP): .RS -.na .nf $ ping 1.1.1.1 | jc \fB--ping-s\fP | jq @@ -608,7 +606,6 @@ $ ping 1.1.1.1 | jc \fB--ping-s\fP | jq This is because the OS engages the 4KB buffer between \fBjc\fP and \fBjq\fP in this example. To display the data on the terminal in realtime, you can disable the buffer with the \fB-u\fP (unbuffer) cli option: .RS -.na .nf $ ping 1.1.1.1 | jc \fB--ping-s\fP \fB-u\fP | jq {"type":"reply","pattern":null,"timestamp":null,"bytes":"64",...} @@ -623,7 +620,6 @@ Note: Unbuffered output can be slower for large data streams. Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your local "App data directory": .RS -.na .nf - Linux/unix: \fB$HOME/.local/share/jc/jcparsers\fP - macOS: \fB$HOME/Library/Application Support/jc/jcparsers\fP diff --git a/templates/manpage_template b/templates/manpage_template index f37ccdbc..43c6b585 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -123,7 +123,6 @@ You may want to ignore parsing errors when using streaming parsers since these m .RS Successfully parsed line with \fB-qq\fP option: .RS -.na .nf { "command_data": "data", @@ -131,11 +130,11 @@ Successfully parsed line with \fB-qq\fP option: "success": true } } +.fi .RE Unsuccessfully parsed line with \fB-qq\fP option: .RS -.na .nf { "_jc_meta": { @@ -153,7 +152,6 @@ Unsuccessfully parsed line with \fB-qq\fP option: Most operating systems will buffer output that is being piped from process to process. The buffer is usually around 4KB. When viewing the output in the terminal the OS buffer is not engaged so output is immediately displayed on the screen. When piping multiple processes together, though, it may seem as if the output is hanging when the input data is very slow (e.g. \fBping\fP): .RS -.na .nf $ ping 1.1.1.1 | jc \fB--ping-s\fP | jq @@ -163,7 +161,6 @@ $ ping 1.1.1.1 | jc \fB--ping-s\fP | jq This is because the OS engages the 4KB buffer between \fBjc\fP and \fBjq\fP in this example. To display the data on the terminal in realtime, you can disable the buffer with the \fB-u\fP (unbuffer) cli option: .RS -.na .nf $ ping 1.1.1.1 | jc \fB--ping-s\fP \fB-u\fP | jq {"type":"reply","pattern":null,"timestamp":null,"bytes":"64",...} @@ -178,7 +175,6 @@ Note: Unbuffered output can be slower for large data streams. Custom local parser plugins may be placed in a \fBjc/jcparsers\fP folder in your local "App data directory": .RS -.na .nf - Linux/unix: \fB$HOME/.local/share/jc/jcparsers\fP - macOS: \fB$HOME/Library/Application Support/jc/jcparsers\fP From 845d763829acd03fff1d2fad6e9691f04bd1a2aa Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 15:09:22 -0800 Subject: [PATCH 09/87] format docs to fit 80 columns --- docs/parsers/pidstat.md | 4 ++-- jc/parsers/pidstat.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/parsers/pidstat.md b/docs/parsers/pidstat.md index c2c946b5..8921b870 100644 --- a/docs/parsers/pidstat.md +++ b/docs/parsers/pidstat.md @@ -67,7 +67,7 @@ Examples: "percent_guest": 0.0, "percent_cpu": 0.03, "cpu": 0, - "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "command": "/usr/lib/systemd/systemd --switched-root --system..." }, { "time": 1646859134, @@ -104,7 +104,7 @@ Examples: "percent_guest": "0.00", "percent_cpu": "0.03", "cpu": "0", - "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "command": "/usr/lib/systemd/systemd --switched-root --system..." }, { "time": "1646859134", diff --git a/jc/parsers/pidstat.py b/jc/parsers/pidstat.py index 5b6e79da..5e9b7467 100644 --- a/jc/parsers/pidstat.py +++ b/jc/parsers/pidstat.py @@ -62,7 +62,7 @@ Examples: "percent_guest": 0.0, "percent_cpu": 0.03, "cpu": 0, - "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "command": "/usr/lib/systemd/systemd --switched-root --system..." }, { "time": 1646859134, @@ -99,7 +99,7 @@ Examples: "percent_guest": "0.00", "percent_cpu": "0.03", "cpu": "0", - "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "command": "/usr/lib/systemd/systemd --switched-root --system..." }, { "time": "1646859134", From 5e7a87f39782d12070a895440ff541800df91b9c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 15:37:04 -0800 Subject: [PATCH 10/87] add test template --- tests/_test_foo.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/_test_foo.py diff --git a/tests/_test_foo.py b/tests/_test_foo.py new file mode 100644 index 00000000..9ca7b75c --- /dev/null +++ b/tests/_test_foo.py @@ -0,0 +1,35 @@ +import os +import unittest +import json +import jc.parsers.foo + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/foo.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_foo = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/foo.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_foo_json = json.loads(f.read()) + + + def test_foo_nodata(self): + """ + Test 'foo' with no data + """ + self.assertEqual(jc.parsers.foo.parse('', quiet=True), []) + + def test_foo_centos_7_7(self): + """ + Test 'foo' on Centos 7.7 + """ + self.assertEqual(jc.parsers.foo.parse(self.centos_7_7_foo, quiet=True), self.centos_7_7_foo_json) + + +if __name__ == '__main__': + unittest.main() From 21f27f26c81c75234221a5193c7d3903323e2032 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 15:37:10 -0800 Subject: [PATCH 11/87] add pidstat tests --- .../centos-7.7/pidstat-hdlrsuw-2-5.json | 1 + .../fixtures/centos-7.7/pidstat-hdlrsuw.json | 1 + tests/fixtures/centos-7.7/pidstat-hl.json | 1 + tests/test_pidstat.py | 69 +++++++++++++++++++ 4 files changed, 72 insertions(+) create mode 100644 tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.json create mode 100644 tests/fixtures/centos-7.7/pidstat-hdlrsuw.json create mode 100644 tests/fixtures/centos-7.7/pidstat-hl.json create mode 100644 tests/test_pidstat.py diff --git a/tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.json b/tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.json new file mode 100644 index 00000000..11f389e8 --- /dev/null +++ b/tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.json @@ -0,0 +1 @@ +[{"time":1646859221,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.49,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646859221,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.98,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646859221,"uid":0,"pid":11,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"watchdog/0"},{"time":1646859221,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.99,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646859221,"uid":0,"pid":466,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:1H"},{"time":1646859221,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":2.97,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646859221,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646859221,"uid":1000,"pid":2297,"percent_usr":0.0,"percent_system":0.5,"percent_guest":0.0,"percent_cpu":0.5,"cpu":0,"minflt_s":74.75,"majflt_s":0.0,"vsz":108296,"rss":1072,"percent_mem":0.03,"stksize":132,"stkref":20,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.5,"nvcswch_s":0.5,"command":"pidstat -dlrsuwh 2 5"},{"time":1646859223,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646859223,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":2.0,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646859223,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646859223,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":3.0,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646859223,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646859223,"uid":1000,"pid":2297,"percent_usr":0.5,"percent_system":0.5,"percent_guest":0.0,"percent_cpu":1.0,"cpu":0,"minflt_s":82.5,"majflt_s":0.0,"vsz":108296,"rss":1152,"percent_mem":0.03,"stksize":132,"stkref":24,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.5,"nvcswch_s":0.5,"command":"pidstat -dlrsuwh 2 5"},{"time":1646859225,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.01,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646859225,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":2.02,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646859225,"uid":0,"pid":11,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.51,"nvcswch_s":0.0,"command":"watchdog/0"},{"time":1646859225,"uid":0,"pid":32,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.51,"nvcswch_s":0.0,"command":"khugepaged"},{"time":1646859225,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.01,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646859225,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.51,"percent_guest":0.0,"percent_cpu":0.51,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":3.03,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646859225,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.51,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646859225,"uid":1000,"pid":2297,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":73.23,"majflt_s":0.0,"vsz":108296,"rss":1152,"percent_mem":0.03,"stksize":132,"stkref":24,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.51,"nvcswch_s":0.51,"command":"pidstat -dlrsuwh 2 5"},{"time":1646859227,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646859227,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":2.0,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646859227,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646859227,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":3.0,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646859227,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646859227,"uid":1000,"pid":2297,"percent_usr":0.5,"percent_system":0.5,"percent_guest":0.0,"percent_cpu":1.0,"cpu":0,"minflt_s":72.5,"majflt_s":0.0,"vsz":108296,"rss":1152,"percent_mem":0.03,"stksize":132,"stkref":24,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.5,"nvcswch_s":0.5,"command":"pidstat -dlrsuwh 2 5"},{"time":1646859229,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646859229,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":3.48,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646859229,"uid":0,"pid":11,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"watchdog/0"},{"time":1646859229,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646859229,"uid":0,"pid":466,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:1H"},{"time":1646859229,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":2.99,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646859229,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646859229,"uid":1000,"pid":2297,"percent_usr":0.0,"percent_system":0.5,"percent_guest":0.0,"percent_cpu":0.5,"cpu":0,"minflt_s":72.14,"majflt_s":0.0,"vsz":108296,"rss":1152,"percent_mem":0.03,"stksize":132,"stkref":24,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"pidstat -dlrsuwh 2 5"}] diff --git a/tests/fixtures/centos-7.7/pidstat-hdlrsuw.json b/tests/fixtures/centos-7.7/pidstat-hdlrsuw.json new file mode 100644 index 00000000..e7926c3c --- /dev/null +++ b/tests/fixtures/centos-7.7/pidstat-hdlrsuw.json @@ -0,0 +1 @@ +[{"time":1646857494,"uid":0,"pid":2,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.03,"nvcswch_s":0.0,"command":"kthreadd"},{"time":1646857494,"uid":0,"pid":4,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kworker/0:0H"},{"time":1646857494,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.77,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646857494,"uid":0,"pid":7,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.48,"nvcswch_s":0.0,"command":"migration/0"},{"time":1646857494,"uid":0,"pid":8,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"rcu_bh"},{"time":1646857494,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":4.93,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646857494,"uid":0,"pid":10,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"lru-add-drain"},{"time":1646857494,"uid":0,"pid":11,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.25,"nvcswch_s":0.0,"command":"watchdog/0"},{"time":1646857494,"uid":0,"pid":13,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.03,"nvcswch_s":0.0,"command":"kdevtmpfs"},{"time":1646857494,"uid":0,"pid":14,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"netns"},{"time":1646857494,"uid":0,"pid":15,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.01,"nvcswch_s":0.0,"command":"khungtaskd"},{"time":1646857494,"uid":0,"pid":16,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"writeback"},{"time":1646857494,"uid":0,"pid":17,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kintegrityd"},{"time":1646857494,"uid":0,"pid":18,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":19,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":20,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":21,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kblockd"},{"time":1646857494,"uid":0,"pid":22,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"md"},{"time":1646857494,"uid":0,"pid":23,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"edac-poller"},{"time":1646857494,"uid":0,"pid":24,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"watchdogd"},{"time":1646857494,"uid":0,"pid":30,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kswapd0"},{"time":1646857494,"uid":0,"pid":31,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"ksmd"},{"time":1646857494,"uid":0,"pid":32,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.1,"nvcswch_s":0.0,"command":"khugepaged"},{"time":1646857494,"uid":0,"pid":33,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"crypto"},{"time":1646857494,"uid":0,"pid":41,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kthrotld"},{"time":1646857494,"uid":0,"pid":43,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kmpath_rdacd"},{"time":1646857494,"uid":0,"pid":44,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kaluad"},{"time":1646857494,"uid":0,"pid":45,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kpsmoused"},{"time":1646857494,"uid":0,"pid":47,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"ipv6_addrconf"},{"time":1646857494,"uid":0,"pid":60,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"deferwq"},{"time":1646857494,"uid":0,"pid":95,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.09,"nvcswch_s":0.0,"command":"kauditd"},{"time":1646857494,"uid":0,"pid":272,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"mpt_poll_0"},{"time":1646857494,"uid":0,"pid":273,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"mpt/0"},{"time":1646857494,"uid":0,"pid":274,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"nfit"},{"time":1646857494,"uid":0,"pid":275,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"ata_sff"},{"time":1646857494,"uid":0,"pid":285,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"scsi_eh_0"},{"time":1646857494,"uid":0,"pid":288,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"scsi_tmf_0"},{"time":1646857494,"uid":0,"pid":308,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"scsi_eh_1"},{"time":1646857494,"uid":0,"pid":309,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.88,"nvcswch_s":0.0,"command":"kworker/u256:2"},{"time":1646857494,"uid":0,"pid":314,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"scsi_tmf_1"},{"time":1646857494,"uid":0,"pid":319,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.01,"nvcswch_s":0.0,"command":"scsi_eh_2"},{"time":1646857494,"uid":0,"pid":323,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"scsi_tmf_2"},{"time":1646857494,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.19,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646857494,"uid":0,"pid":357,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.63,"nvcswch_s":0.0,"command":"irq/16-vmwgfx"},{"time":1646857494,"uid":0,"pid":359,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"ttm_swap"},{"time":1646857494,"uid":0,"pid":431,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kdmflush"},{"time":1646857494,"uid":0,"pid":432,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":441,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kdmflush"},{"time":1646857494,"uid":0,"pid":442,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":455,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":456,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfsalloc"},{"time":1646857494,"uid":0,"pid":457,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs_mru_cache"},{"time":1646857494,"uid":0,"pid":458,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-buf/dm-0"},{"time":1646857494,"uid":0,"pid":459,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-data/dm-0"},{"time":1646857494,"uid":0,"pid":460,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-conv/dm-0"},{"time":1646857494,"uid":0,"pid":461,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-cil/dm-0"},{"time":1646857494,"uid":0,"pid":462,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-reclaim/dm-"},{"time":1646857494,"uid":0,"pid":463,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-log/dm-0"},{"time":1646857494,"uid":0,"pid":464,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-eofblocks/d"},{"time":1646857494,"uid":0,"pid":465,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":3.04,"nvcswch_s":0.0,"command":"xfsaild/dm-0"},{"time":1646857494,"uid":0,"pid":466,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.2,"nvcswch_s":0.0,"command":"kworker/0:1H"},{"time":1646857494,"uid":0,"pid":698,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kworker/u257:0"},{"time":1646857494,"uid":0,"pid":700,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"hci0"},{"time":1646857494,"uid":0,"pid":701,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"hci0"},{"time":1646857494,"uid":0,"pid":702,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.01,"nvcswch_s":0.0,"command":"kworker/u257:1"},{"time":1646857494,"uid":0,"pid":725,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-buf/sda1"},{"time":1646857494,"uid":0,"pid":726,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-data/sda1"},{"time":1646857494,"uid":0,"pid":727,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-conv/sda1"},{"time":1646857494,"uid":0,"pid":728,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-cil/sda1"},{"time":1646857494,"uid":0,"pid":729,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-reclaim/sda"},{"time":1646857494,"uid":0,"pid":730,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-log/sda1"},{"time":1646857494,"uid":0,"pid":731,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-eofblocks/s"},{"time":1646857494,"uid":0,"pid":732,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfsaild/sda1"},{"time":1646857494,"uid":1000,"pid":1852,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.25,"majflt_s":0.0,"vsz":115708,"rss":2324,"percent_mem":0.06,"stksize":264,"stkref":264,"kb_rd_s":0.08,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.01,"nvcswch_s":0.0,"command":"-bash"},{"time":1646857494,"uid":1000,"pid":1877,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.33,"majflt_s":0.0,"vsz":115580,"rss":2216,"percent_mem":0.06,"stksize":132,"stkref":24,"kb_rd_s":0.64,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.03,"nvcswch_s":0.0,"command":"-bash"},{"time":1646857494,"uid":0,"pid":2062,"percent_usr":0.0,"percent_system":0.03,"percent_guest":0.0,"percent_cpu":0.03,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.95,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646857494,"uid":0,"pid":2153,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.03,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646857494,"uid":0,"pid":2199,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.03,"nvcswch_s":0.0,"command":"kworker/0:2"},{"time":1646857494,"uid":1000,"pid":2201,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.09,"majflt_s":0.0,"vsz":108328,"rss":1040,"percent_mem":0.03,"stksize":132,"stkref":20,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"pidstat -dlrsuwh"}] diff --git a/tests/fixtures/centos-7.7/pidstat-hl.json b/tests/fixtures/centos-7.7/pidstat-hl.json new file mode 100644 index 00000000..5502f9f2 --- /dev/null +++ b/tests/fixtures/centos-7.7/pidstat-hl.json @@ -0,0 +1 @@ +[{"time":1646859134,"uid":0,"pid":1,"percent_usr":0.0,"percent_system":0.03,"percent_guest":0.0,"percent_cpu":0.03,"cpu":0,"command":"/usr/lib/systemd/systemd --switched-root --system --deserialize 22"},{"time":1646859134,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"ksoftirqd/0"},{"time":1646859134,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"rcu_sched"},{"time":1646859134,"uid":0,"pid":11,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"watchdog/0"},{"time":1646859134,"uid":0,"pid":32,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"khugepaged"},{"time":1646859134,"uid":0,"pid":308,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"scsi_eh_1"},{"time":1646859134,"uid":0,"pid":309,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"kworker/u256:2"},{"time":1646859134,"uid":0,"pid":319,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"scsi_eh_2"},{"time":1646859134,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"kworker/u256:4"},{"time":1646859134,"uid":0,"pid":357,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"irq/16-vmwgfx"},{"time":1646859134,"uid":0,"pid":465,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"xfsaild/dm-0"},{"time":1646859134,"uid":0,"pid":466,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"kworker/0:1H"},{"time":1646859134,"uid":0,"pid":543,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/lib/systemd/systemd-journald"},{"time":1646859134,"uid":0,"pid":564,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/sbin/lvmetad -f"},{"time":1646859134,"uid":0,"pid":577,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/lib/systemd/systemd-udevd"},{"time":1646859134,"uid":0,"pid":752,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/sbin/auditd"},{"time":1646859134,"uid":0,"pid":779,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/libexec/bluetooth/bluetoothd"},{"time":1646859134,"uid":999,"pid":780,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/lib/polkit-1/polkitd --no-debug"},{"time":1646859134,"uid":0,"pid":782,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/sbin/smartd -n -q never"},{"time":1646859134,"uid":0,"pid":784,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/lib/systemd/systemd-logind"},{"time":1646859134,"uid":81,"pid":787,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation"},{"time":1646859134,"uid":998,"pid":790,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/sbin/chronyd"},{"time":1646859134,"uid":0,"pid":834,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"/usr/sbin/crond -n"},{"time":1646859134,"uid":0,"pid":847,"percent_usr":0.01,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"/usr/bin/python2 -Es /usr/sbin/firewalld --nofork --nopid"},{"time":1646859134,"uid":0,"pid":849,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/sbin/agetty --keep-baud 115200,38400,9600 ttyS0 vt220"},{"time":1646859134,"uid":0,"pid":852,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"login -- kbrazil"},{"time":1646859134,"uid":0,"pid":882,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/sbin/NetworkManager --no-daemon"},{"time":1646859134,"uid":0,"pid":1031,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/sbin/dhclient -d -q -sf /usr/libexec/nm-dhcp-helper -pf /var/run/dhclient-ens33.pid -lf /var/lib/NetworkManager/dhclient-d92ec"},{"time":1646859134,"uid":0,"pid":1220,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/sbin/sshd -D"},{"time":1646859134,"uid":0,"pid":1221,"percent_usr":0.06,"percent_system":0.03,"percent_guest":0.0,"percent_cpu":0.09,"cpu":0,"command":"/usr/bin/dockerd-current --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current --default-runtime=docker-runc --exec"},{"time":1646859134,"uid":0,"pid":1222,"percent_usr":0.01,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"/usr/bin/python2 -Es /usr/sbin/tuned -l -P"},{"time":1646859134,"uid":0,"pid":1225,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"/usr/sbin/rsyslogd -n"},{"time":1646859134,"uid":0,"pid":1293,"percent_usr":0.04,"percent_system":0.02,"percent_guest":0.0,"percent_cpu":0.05,"cpu":0,"command":"/usr/bin/docker-containerd-current -l unix:///var/run/docker/libcontainerd/docker-containerd.sock --metrics-interval=0 --start-"},{"time":1646859134,"uid":0,"pid":1496,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/libexec/postfix/master -w"},{"time":1646859134,"uid":89,"pid":1512,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"qmgr -l -t unix -u"},{"time":1646859134,"uid":1000,"pid":1852,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"-bash"},{"time":1646859134,"uid":0,"pid":1872,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"sshd: kbrazil [priv]"},{"time":1646859134,"uid":1000,"pid":1876,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"sshd: kbrazil@pts/0"},{"time":1646859134,"uid":1000,"pid":1877,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"-bash"},{"time":1646859134,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"kworker/0:1"},{"time":1646859134,"uid":0,"pid":2239,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"kworker/0:3"},{"time":1646859134,"uid":89,"pid":2240,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"pickup -l -t unix -u"},{"time":1646859134,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"kworker/0:0"}] diff --git a/tests/test_pidstat.py b/tests/test_pidstat.py new file mode 100644 index 00000000..01c2e60e --- /dev/null +++ b/tests/test_pidstat.py @@ -0,0 +1,69 @@ +import os +import unittest +import json +import jc.parsers.pidstat +from jc.exceptions import ParseError + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hl.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hl = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hdlrsuw = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hdlrsuw_2_5 = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hl.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hl_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hdlrsuw_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hdlrsuw_2_5_json = json.loads(f.read()) + + + def test_pidstat_nodata(self): + """ + Test 'pidstat' with no data + """ + self.assertEqual(jc.parsers.pidstat.parse('', quiet=True), []) + + def test_pidstat(self): + """ + Test 'pidstat' without -h... should raise ParseError + """ + self.assertRaises(ParseError, jc.parsers.pidstat.parse, self.centos_7_7_pidstat, quiet=True) + + def test_pidstat_hl_centos_7_7(self): + """ + Test 'pidstat -hl' on Centos 7.7 + """ + self.assertEqual(jc.parsers.pidstat.parse(self.centos_7_7_pidstat_hl, quiet=True), self.centos_7_7_pidstat_hl_json) + + def test_pidstat_hdlrsuw_centos_7_7(self): + """ + Test 'pidstat -hdlrsuw' on Centos 7.7 + """ + self.assertEqual(jc.parsers.pidstat.parse(self.centos_7_7_pidstat_hdlrsuw, quiet=True), self.centos_7_7_pidstat_hdlrsuw_json) + + def test_pidstat_hdlrsuw_2_5_centos_7_7(self): + """ + Test 'pidstat -hdlrsuw 2 5' on Centos 7.7 + """ + self.assertEqual(jc.parsers.pidstat.parse(self.centos_7_7_pidstat_hdlrsuw_2_5, quiet=True), self.centos_7_7_pidstat_hdlrsuw_2_5_json) + + +if __name__ == '__main__': + unittest.main() From c52ca20e285895d2afb685a2a5ca275de7901033 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 16:07:17 -0800 Subject: [PATCH 12/87] fix comment --- jc/parsers/pidstat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/pidstat.py b/jc/parsers/pidstat.py index 5e9b7467..b789cf54 100644 --- a/jc/parsers/pidstat.py +++ b/jc/parsers/pidstat.py @@ -206,7 +206,7 @@ def parse( if not data_list: raise ParseError('Could not parse pidstat output. Make sure to use "pidstat -h".') - # normalize headers + # normalize header data_list[0] = data_list[0].replace('#', ' ')\ .replace('/', '_')\ .replace('%', 'percent_')\ From 0d13909cf63d14008607c5640d074ab4073115e5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 16:07:29 -0800 Subject: [PATCH 13/87] add pidstat-s parser --- jc/lib.py | 1 + jc/parsers/pidstat_s.py | 167 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 jc/parsers/pidstat_s.py diff --git a/jc/lib.py b/jc/lib.py index 678da5f4..b61f39df 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -64,6 +64,7 @@ parsers = [ 'ntpq', 'passwd', 'pidstat', + 'pidstat-s', 'ping', 'ping-s', 'pip-list', diff --git a/jc/parsers/pidstat_s.py b/jc/parsers/pidstat_s.py new file mode 100644 index 00000000..281f82aa --- /dev/null +++ b/jc/parsers/pidstat_s.py @@ -0,0 +1,167 @@ +"""jc - JSON Convert `pidstat` command output streaming parser + +> This streaming parser outputs JSON Lines + +<> + +Usage (cli): + + $ pidstat | jc --pidstat-s + +Usage (module): + + import jc + # result is an iterable object (generator) + result = jc.parse('pidstat_s', pidstat_command_output.splitlines()) + for item in result: + # do something + + or + + import jc.parsers.pidstat_s + # result is an iterable object (generator) + result = jc.parsers.pidstat_s.parse(pidstat_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "pidstat": string, + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": + { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + +Examples: + + $ pidstat | jc --pidstat-s + {example output} + ... + + $ pidstat | jc --pidstat-s -r + {example output} + ... +""" +from typing import Dict, Iterable, Union +import jc.utils +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) +from jc.parsers.universal import simple_table_parse +from jc.exceptions import ParseError + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`pidstat` command streaming parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + streaming = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured data to conform to the schema. + """ + int_list = ['time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'] + float_list = ['percent_usr', 'percent_system', 'percent_guest', 'percent_cpu', + 'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s', + 'kb_ccwr_s', 'cswch_s', 'nvcswch_s'] + + for key in proc_data: + if key in int_list: + proc_data[key] = jc.utils.convert_to_int(proc_data[key]) + if key in float_list: + proc_data[key] = jc.utils.convert_to_float(proc_data[key]) + + return proc_data + + +@add_jc_meta +def parse( + data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False +) -> Union[Iterable[Dict], tuple]: + """ + Main text parsing generator function. Returns an iterator object. + + Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + + Yields: + + Dictionary. Raw or processed structured data. + + Returns: + + Iterator object (generator) + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + streaming_input_type_check(data) + + found_first_hash = False + header = '' + + for line in data: + try: + streaming_line_input_type_check(line) + output_line: Dict = {} + + if not line.strip(): + # skip blank lines + continue + + if not line.startswith('#') and not found_first_hash: + # skip preample lines before header row + continue + + if line.startswith('#') and not found_first_hash: + # normalize header + header = line.replace('#', ' ')\ + .replace('/', '_')\ + .replace('%', 'percent_')\ + .lower() + found_first_hash = True + continue + + if line.startswith('#') and found_first_hash: + # skip header lines after first one is found + continue + + output_line = simple_table_parse([header, line])[0] + + if output_line: + yield output_line if raw else _process(output_line) + else: + raise ParseError('Not pidstat data') + + except Exception as e: + yield raise_or_yield(ignore_exceptions, e, line) From d3727ea0906d6167aa48c340519242b010469d98 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 16:09:35 -0800 Subject: [PATCH 14/87] doc update --- CHANGELOG | 3 +- README.md | 1 + docs/parsers/pidstat_s.md | 91 +++++++++++++++++++++++++++++++++++++++ man/jc.1 | 5 +++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 docs/parsers/pidstat_s.md diff --git a/CHANGELOG b/CHANGELOG index 163f9ced..cd899fe9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,8 @@ jc changelog 20220309 v1.18.6 (in progress) -- Add pidstat parser tested on linux +- Add pidstat command parser tested on linux +- Add pidstat command streaming parser tested on linux 20220305 v1.18.5 - Fix date parser to ensure AM/PM period string is always uppercase diff --git a/README.md b/README.md index 3732ab20..354e373d 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ option. - `--ntpq` enables the `ntpq -p` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ntpq)) - `--passwd` enables the `/etc/passwd` file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/passwd)) - `--pidstat` enables the `pidstat` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/pidstat)) +- `--pidstat-s` enables the `pidstat` command streaming parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/pidstat_s)) - `--ping` enables the `ping` and `ping6` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ping)) - `--ping-s` enables the `ping` and `ping6` command streaming parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ping_s)) - `--pip-list` enables the `pip list` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_list)) diff --git a/docs/parsers/pidstat_s.md b/docs/parsers/pidstat_s.md new file mode 100644 index 00000000..cc049b34 --- /dev/null +++ b/docs/parsers/pidstat_s.md @@ -0,0 +1,91 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.pidstat\_s + +jc - JSON Convert `pidstat` command output streaming parser + +> This streaming parser outputs JSON Lines + +<> + +Usage (cli): + + $ pidstat | jc --pidstat-s + +Usage (module): + + import jc + # result is an iterable object (generator) + result = jc.parse('pidstat_s', pidstat_command_output.splitlines()) + for item in result: + # do something + + or + + import jc.parsers.pidstat_s + # result is an iterable object (generator) + result = jc.parsers.pidstat_s.parse(pidstat_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "pidstat": string, + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": + { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + +Examples: + + $ pidstat | jc --pidstat-s + {example output} + ... + + $ pidstat | jc --pidstat-s -r + {example output} + ... + + + +### parse + +```python +@add_jc_meta +def parse(data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] +``` + +Main text parsing generator function. Returns an iterator object. + +Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + +Yields: + + Dictionary. Raw or processed structured data. + +Returns: + + Iterator object (generator) + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/man/jc.1 b/man/jc.1 index cd19fb2a..ff95eb80 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -277,6 +277,11 @@ Key/Value file parser \fB--pidstat\fP `pidstat` command parser +.TP +.B +\fB--pidstat-s\fP +`pidstat` command streaming parser + .TP .B \fB--ping\fP From e4ae0fea63c0386c1b3239607fd6a58738d23633 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 9 Mar 2022 16:28:55 -0800 Subject: [PATCH 15/87] fix type annotation --- docs/parsers/pidstat_s.md | 2 +- docs/parsers/rsync_s.md | 2 +- jc/parsers/foo_s.py | 4 ++-- jc/parsers/pidstat_s.py | 4 ++-- jc/parsers/rsync_s.py | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/parsers/pidstat_s.md b/docs/parsers/pidstat_s.md index cc049b34..01118d3f 100644 --- a/docs/parsers/pidstat_s.md +++ b/docs/parsers/pidstat_s.md @@ -63,7 +63,7 @@ Examples: def parse(data: Iterable[str], raw: bool = False, quiet: bool = False, - ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] + ignore_exceptions: bool = False) -> Union[Iterable[Dict], NoReturn] ``` Main text parsing generator function. Returns an iterator object. diff --git a/docs/parsers/rsync_s.md b/docs/parsers/rsync_s.md index b3e41c30..810ac37b 100644 --- a/docs/parsers/rsync_s.md +++ b/docs/parsers/rsync_s.md @@ -102,7 +102,7 @@ Examples: def parse(data: Iterable[str], raw: bool = False, quiet: bool = False, - ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] + ignore_exceptions: bool = False) -> Union[Iterable[Dict], NoReturn] ``` Main text parsing generator function. Returns an iterator object. diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index 1113202d..ae40743c 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -49,7 +49,7 @@ Examples: {example output} ... """ -from typing import Dict, Iterable, Union +from typing import Dict, Iterable, Union, NoReturn import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -99,7 +99,7 @@ def parse( raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Union[Iterable[Dict], tuple]: +) -> Union[Iterable[Dict], NoReturn]: """ Main text parsing generator function. Returns an iterator object. diff --git a/jc/parsers/pidstat_s.py b/jc/parsers/pidstat_s.py index 281f82aa..4c79d612 100644 --- a/jc/parsers/pidstat_s.py +++ b/jc/parsers/pidstat_s.py @@ -49,7 +49,7 @@ Examples: {example output} ... """ -from typing import Dict, Iterable, Union +from typing import Dict, Iterable, Union, NoReturn import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -103,7 +103,7 @@ def parse( raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Union[Iterable[Dict], tuple]: +) -> Union[Iterable[Dict], NoReturn]: """ Main text parsing generator function. Returns an iterator object. diff --git a/jc/parsers/rsync_s.py b/jc/parsers/rsync_s.py index 94b874e3..8beec964 100644 --- a/jc/parsers/rsync_s.py +++ b/jc/parsers/rsync_s.py @@ -89,7 +89,7 @@ Examples: ... """ import re -from typing import Dict, Iterable, Union +from typing import Dict, Iterable, Union, NoReturn import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -148,7 +148,7 @@ def parse( raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Union[Iterable[Dict], tuple]: +) -> Union[Iterable[Dict], NoReturn]: """ Main text parsing generator function. Returns an iterator object. From 4fc04256a567c502a88ac8a1bd189746387c93f1 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 08:00:37 -0800 Subject: [PATCH 16/87] rollback noReturn to tuple --- jc/parsers/foo_s.py | 4 ++-- jc/parsers/pidstat_s.py | 4 ++-- jc/parsers/rsync_s.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index ae40743c..1113202d 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -49,7 +49,7 @@ Examples: {example output} ... """ -from typing import Dict, Iterable, Union, NoReturn +from typing import Dict, Iterable, Union import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -99,7 +99,7 @@ def parse( raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Union[Iterable[Dict], NoReturn]: +) -> Union[Iterable[Dict], tuple]: """ Main text parsing generator function. Returns an iterator object. diff --git a/jc/parsers/pidstat_s.py b/jc/parsers/pidstat_s.py index 4c79d612..281f82aa 100644 --- a/jc/parsers/pidstat_s.py +++ b/jc/parsers/pidstat_s.py @@ -49,7 +49,7 @@ Examples: {example output} ... """ -from typing import Dict, Iterable, Union, NoReturn +from typing import Dict, Iterable, Union import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -103,7 +103,7 @@ def parse( raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Union[Iterable[Dict], NoReturn]: +) -> Union[Iterable[Dict], tuple]: """ Main text parsing generator function. Returns an iterator object. diff --git a/jc/parsers/rsync_s.py b/jc/parsers/rsync_s.py index 8beec964..94b874e3 100644 --- a/jc/parsers/rsync_s.py +++ b/jc/parsers/rsync_s.py @@ -89,7 +89,7 @@ Examples: ... """ import re -from typing import Dict, Iterable, Union, NoReturn +from typing import Dict, Iterable, Union import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -148,7 +148,7 @@ def parse( raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Union[Iterable[Dict], NoReturn]: +) -> Union[Iterable[Dict], tuple]: """ Main text parsing generator function. Returns an iterator object. From 3278cb0de301ee542b1fe8245a6c359193373b6b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 08:14:46 -0800 Subject: [PATCH 17/87] add type hints --- docs/parsers/arp.md | 2 +- jc/parsers/arp.py | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/parsers/arp.md b/docs/parsers/arp.md index c036899f..a6c9b966 100644 --- a/docs/parsers/arp.md +++ b/docs/parsers/arp.md @@ -127,7 +127,7 @@ Examples: ### parse ```python -def parse(data, raw=False, quiet=False) +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] ``` Main text parsing function diff --git a/jc/parsers/arp.py b/jc/parsers/arp.py index 7524f265..26203490 100644 --- a/jc/parsers/arp.py +++ b/jc/parsers/arp.py @@ -117,6 +117,7 @@ Examples: } ] """ +from typing import List, Dict, Any import jc.utils import jc.parsers.universal @@ -134,7 +135,7 @@ class info(): __version__ = info.version -def _process(proc_data): +def _process(proc_data: List[Dict]) -> List[Dict]: """ Final processing to conform to the schema. @@ -160,7 +161,11 @@ def _process(proc_data): return proc_data -def parse(data, raw=False, quiet=False): +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: """ Main text parsing function @@ -190,7 +195,7 @@ def parse(data, raw=False, quiet=False): if cleandata[0][-1] == ']': for line in cleandata: splitline = line.split() - output_line = { + output_line: Dict[str, Any] = { 'name': splitline[0], 'address': splitline[1].lstrip('(').rstrip(')'), 'hwtype': splitline[-1].lstrip('[').rstrip(']'), @@ -220,13 +225,13 @@ def parse(data, raw=False, quiet=False): # otherwise, try bsd style else: for line in cleandata: - line = line.split() + splitline = line.split() output_line = { - 'name': line[0], - 'address': line[1].lstrip('(').rstrip(')'), - 'hwtype': line[4].lstrip('[').rstrip(']'), - 'hwaddress': line[3], - 'iface': line[6], + 'name': splitline[0], + 'address': splitline[1].lstrip('(').rstrip(')'), + 'hwtype': splitline[4].lstrip('[').rstrip(']'), + 'hwaddress': splitline[3], + 'iface': splitline[6], } raw_output.append(output_line) From 4f0616190bb842411353d65650b78a65d8cbdc28 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 08:14:58 -0800 Subject: [PATCH 18/87] doc update --- docs/parsers/pidstat_s.md | 2 +- docs/parsers/rsync_s.md | 2 +- man/jc.1 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/parsers/pidstat_s.md b/docs/parsers/pidstat_s.md index 01118d3f..cc049b34 100644 --- a/docs/parsers/pidstat_s.md +++ b/docs/parsers/pidstat_s.md @@ -63,7 +63,7 @@ Examples: def parse(data: Iterable[str], raw: bool = False, quiet: bool = False, - ignore_exceptions: bool = False) -> Union[Iterable[Dict], NoReturn] + ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] ``` Main text parsing generator function. Returns an iterator object. diff --git a/docs/parsers/rsync_s.md b/docs/parsers/rsync_s.md index 810ac37b..b3e41c30 100644 --- a/docs/parsers/rsync_s.md +++ b/docs/parsers/rsync_s.md @@ -102,7 +102,7 @@ Examples: def parse(data: Iterable[str], raw: bool = False, quiet: bool = False, - ignore_exceptions: bool = False) -> Union[Iterable[Dict], NoReturn] + ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] ``` Main text parsing generator function. Returns an iterator object. diff --git a/man/jc.1 b/man/jc.1 index ff95eb80..45711507 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-03-09 1.18.6 "JSON Convert" +.TH jc 1 2022-03-10 1.18.6 "JSON Convert" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 696338c1a3d462afbe677a6f2ebe08d06ef4046a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 09:04:12 -0800 Subject: [PATCH 19/87] add streaming test template --- tests/_test_foo_s.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/_test_foo_s.py diff --git a/tests/_test_foo_s.py b/tests/_test_foo_s.py new file mode 100644 index 00000000..b8a08609 --- /dev/null +++ b/tests/_test_foo_s.py @@ -0,0 +1,38 @@ +import os +import json +import unittest +import jc.parsers.foo_s + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +# To create streaming output use: +# $ cat foo.out | jc --foo-s | jello -c > foo-streaming.json + + +class MyTests(unittest.TestCase): + + def setUp(self): + pass + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/foo.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_foo = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/foo-streaming.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_foo_streaming_json = json.loads(f.read()) + + def test_foo_s_nodata(self): + """ + Test 'foo' with no data + """ + self.assertEqual(list(jc.parsers.foo_s.parse([], quiet=True)), []) + + def test_foo_s_centos_7_7(self): + """ + Test 'foo' on Centos 7.7 + """ + self.assertEqual(list(jc.parsers.foo_s.parse(self.centos_7_7_foo.splitlines(), quiet=True)), self.centos_7_7_foo_streaming_json) + + +if __name__ == '__main__': + unittest.main() From b9365e2ac28c3eacf53bc101857f7f1d7286ec20 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 09:23:36 -0800 Subject: [PATCH 20/87] add pidstat-s tests --- .../pidstat-hdlrsuw-2-5-streaming.json | 1 + .../centos-7.7/pidstat-hdlrsuw-streaming.json | 1 + .../centos-7.7/pidstat-hl-streaming.json | 1 + tests/test_pidstat_s.py | 71 +++++++++++++++++++ 4 files changed, 74 insertions(+) create mode 100644 tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5-streaming.json create mode 100644 tests/fixtures/centos-7.7/pidstat-hdlrsuw-streaming.json create mode 100644 tests/fixtures/centos-7.7/pidstat-hl-streaming.json create mode 100644 tests/test_pidstat_s.py diff --git a/tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5-streaming.json b/tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5-streaming.json new file mode 100644 index 00000000..11f389e8 --- /dev/null +++ b/tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5-streaming.json @@ -0,0 +1 @@ +[{"time":1646859221,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.49,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646859221,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.98,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646859221,"uid":0,"pid":11,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"watchdog/0"},{"time":1646859221,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.99,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646859221,"uid":0,"pid":466,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:1H"},{"time":1646859221,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":2.97,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646859221,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646859221,"uid":1000,"pid":2297,"percent_usr":0.0,"percent_system":0.5,"percent_guest":0.0,"percent_cpu":0.5,"cpu":0,"minflt_s":74.75,"majflt_s":0.0,"vsz":108296,"rss":1072,"percent_mem":0.03,"stksize":132,"stkref":20,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.5,"nvcswch_s":0.5,"command":"pidstat -dlrsuwh 2 5"},{"time":1646859223,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646859223,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":2.0,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646859223,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646859223,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":3.0,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646859223,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646859223,"uid":1000,"pid":2297,"percent_usr":0.5,"percent_system":0.5,"percent_guest":0.0,"percent_cpu":1.0,"cpu":0,"minflt_s":82.5,"majflt_s":0.0,"vsz":108296,"rss":1152,"percent_mem":0.03,"stksize":132,"stkref":24,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.5,"nvcswch_s":0.5,"command":"pidstat -dlrsuwh 2 5"},{"time":1646859225,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.01,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646859225,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":2.02,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646859225,"uid":0,"pid":11,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.51,"nvcswch_s":0.0,"command":"watchdog/0"},{"time":1646859225,"uid":0,"pid":32,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.51,"nvcswch_s":0.0,"command":"khugepaged"},{"time":1646859225,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.01,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646859225,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.51,"percent_guest":0.0,"percent_cpu":0.51,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":3.03,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646859225,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.51,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646859225,"uid":1000,"pid":2297,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":73.23,"majflt_s":0.0,"vsz":108296,"rss":1152,"percent_mem":0.03,"stksize":132,"stkref":24,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.51,"nvcswch_s":0.51,"command":"pidstat -dlrsuwh 2 5"},{"time":1646859227,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646859227,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":2.0,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646859227,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646859227,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":3.0,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646859227,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646859227,"uid":1000,"pid":2297,"percent_usr":0.5,"percent_system":0.5,"percent_guest":0.0,"percent_cpu":1.0,"cpu":0,"minflt_s":72.5,"majflt_s":0.0,"vsz":108296,"rss":1152,"percent_mem":0.03,"stksize":132,"stkref":24,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.5,"nvcswch_s":0.5,"command":"pidstat -dlrsuwh 2 5"},{"time":1646859229,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646859229,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":3.48,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646859229,"uid":0,"pid":11,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"watchdog/0"},{"time":1646859229,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":1.0,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646859229,"uid":0,"pid":466,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:1H"},{"time":1646859229,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":2.99,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646859229,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646859229,"uid":1000,"pid":2297,"percent_usr":0.0,"percent_system":0.5,"percent_guest":0.0,"percent_cpu":0.5,"cpu":0,"minflt_s":72.14,"majflt_s":0.0,"vsz":108296,"rss":1152,"percent_mem":0.03,"stksize":132,"stkref":24,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.5,"nvcswch_s":0.0,"command":"pidstat -dlrsuwh 2 5"}] diff --git a/tests/fixtures/centos-7.7/pidstat-hdlrsuw-streaming.json b/tests/fixtures/centos-7.7/pidstat-hdlrsuw-streaming.json new file mode 100644 index 00000000..e7926c3c --- /dev/null +++ b/tests/fixtures/centos-7.7/pidstat-hdlrsuw-streaming.json @@ -0,0 +1 @@ +[{"time":1646857494,"uid":0,"pid":2,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.03,"nvcswch_s":0.0,"command":"kthreadd"},{"time":1646857494,"uid":0,"pid":4,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kworker/0:0H"},{"time":1646857494,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.77,"nvcswch_s":0.0,"command":"ksoftirqd/0"},{"time":1646857494,"uid":0,"pid":7,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.48,"nvcswch_s":0.0,"command":"migration/0"},{"time":1646857494,"uid":0,"pid":8,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"rcu_bh"},{"time":1646857494,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":4.93,"nvcswch_s":0.0,"command":"rcu_sched"},{"time":1646857494,"uid":0,"pid":10,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"lru-add-drain"},{"time":1646857494,"uid":0,"pid":11,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.25,"nvcswch_s":0.0,"command":"watchdog/0"},{"time":1646857494,"uid":0,"pid":13,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.03,"nvcswch_s":0.0,"command":"kdevtmpfs"},{"time":1646857494,"uid":0,"pid":14,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"netns"},{"time":1646857494,"uid":0,"pid":15,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.01,"nvcswch_s":0.0,"command":"khungtaskd"},{"time":1646857494,"uid":0,"pid":16,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"writeback"},{"time":1646857494,"uid":0,"pid":17,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kintegrityd"},{"time":1646857494,"uid":0,"pid":18,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":19,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":20,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":21,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kblockd"},{"time":1646857494,"uid":0,"pid":22,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"md"},{"time":1646857494,"uid":0,"pid":23,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"edac-poller"},{"time":1646857494,"uid":0,"pid":24,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"watchdogd"},{"time":1646857494,"uid":0,"pid":30,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kswapd0"},{"time":1646857494,"uid":0,"pid":31,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"ksmd"},{"time":1646857494,"uid":0,"pid":32,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.1,"nvcswch_s":0.0,"command":"khugepaged"},{"time":1646857494,"uid":0,"pid":33,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"crypto"},{"time":1646857494,"uid":0,"pid":41,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kthrotld"},{"time":1646857494,"uid":0,"pid":43,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kmpath_rdacd"},{"time":1646857494,"uid":0,"pid":44,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kaluad"},{"time":1646857494,"uid":0,"pid":45,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kpsmoused"},{"time":1646857494,"uid":0,"pid":47,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"ipv6_addrconf"},{"time":1646857494,"uid":0,"pid":60,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"deferwq"},{"time":1646857494,"uid":0,"pid":95,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.09,"nvcswch_s":0.0,"command":"kauditd"},{"time":1646857494,"uid":0,"pid":272,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"mpt_poll_0"},{"time":1646857494,"uid":0,"pid":273,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"mpt/0"},{"time":1646857494,"uid":0,"pid":274,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"nfit"},{"time":1646857494,"uid":0,"pid":275,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"ata_sff"},{"time":1646857494,"uid":0,"pid":285,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"scsi_eh_0"},{"time":1646857494,"uid":0,"pid":288,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"scsi_tmf_0"},{"time":1646857494,"uid":0,"pid":308,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"scsi_eh_1"},{"time":1646857494,"uid":0,"pid":309,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.88,"nvcswch_s":0.0,"command":"kworker/u256:2"},{"time":1646857494,"uid":0,"pid":314,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"scsi_tmf_1"},{"time":1646857494,"uid":0,"pid":319,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.01,"nvcswch_s":0.0,"command":"scsi_eh_2"},{"time":1646857494,"uid":0,"pid":323,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"scsi_tmf_2"},{"time":1646857494,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.19,"nvcswch_s":0.0,"command":"kworker/u256:4"},{"time":1646857494,"uid":0,"pid":357,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.63,"nvcswch_s":0.0,"command":"irq/16-vmwgfx"},{"time":1646857494,"uid":0,"pid":359,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"ttm_swap"},{"time":1646857494,"uid":0,"pid":431,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kdmflush"},{"time":1646857494,"uid":0,"pid":432,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":441,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kdmflush"},{"time":1646857494,"uid":0,"pid":442,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":455,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"bioset"},{"time":1646857494,"uid":0,"pid":456,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfsalloc"},{"time":1646857494,"uid":0,"pid":457,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs_mru_cache"},{"time":1646857494,"uid":0,"pid":458,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-buf/dm-0"},{"time":1646857494,"uid":0,"pid":459,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-data/dm-0"},{"time":1646857494,"uid":0,"pid":460,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-conv/dm-0"},{"time":1646857494,"uid":0,"pid":461,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-cil/dm-0"},{"time":1646857494,"uid":0,"pid":462,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-reclaim/dm-"},{"time":1646857494,"uid":0,"pid":463,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-log/dm-0"},{"time":1646857494,"uid":0,"pid":464,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-eofblocks/d"},{"time":1646857494,"uid":0,"pid":465,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":3.04,"nvcswch_s":0.0,"command":"xfsaild/dm-0"},{"time":1646857494,"uid":0,"pid":466,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.2,"nvcswch_s":0.0,"command":"kworker/0:1H"},{"time":1646857494,"uid":0,"pid":698,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"kworker/u257:0"},{"time":1646857494,"uid":0,"pid":700,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"hci0"},{"time":1646857494,"uid":0,"pid":701,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"hci0"},{"time":1646857494,"uid":0,"pid":702,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.01,"nvcswch_s":0.0,"command":"kworker/u257:1"},{"time":1646857494,"uid":0,"pid":725,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-buf/sda1"},{"time":1646857494,"uid":0,"pid":726,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-data/sda1"},{"time":1646857494,"uid":0,"pid":727,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-conv/sda1"},{"time":1646857494,"uid":0,"pid":728,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-cil/sda1"},{"time":1646857494,"uid":0,"pid":729,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-reclaim/sda"},{"time":1646857494,"uid":0,"pid":730,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-log/sda1"},{"time":1646857494,"uid":0,"pid":731,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfs-eofblocks/s"},{"time":1646857494,"uid":0,"pid":732,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"xfsaild/sda1"},{"time":1646857494,"uid":1000,"pid":1852,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.25,"majflt_s":0.0,"vsz":115708,"rss":2324,"percent_mem":0.06,"stksize":264,"stkref":264,"kb_rd_s":0.08,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.01,"nvcswch_s":0.0,"command":"-bash"},{"time":1646857494,"uid":1000,"pid":1877,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.33,"majflt_s":0.0,"vsz":115580,"rss":2216,"percent_mem":0.06,"stksize":132,"stkref":24,"kb_rd_s":0.64,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.03,"nvcswch_s":0.0,"command":"-bash"},{"time":1646857494,"uid":0,"pid":2062,"percent_usr":0.0,"percent_system":0.03,"percent_guest":0.0,"percent_cpu":0.03,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.95,"nvcswch_s":0.0,"command":"kworker/0:0"},{"time":1646857494,"uid":0,"pid":2153,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.03,"nvcswch_s":0.0,"command":"kworker/0:1"},{"time":1646857494,"uid":0,"pid":2199,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.0,"majflt_s":0.0,"vsz":0,"rss":0,"percent_mem":0.0,"stksize":0,"stkref":0,"kb_rd_s":-1.0,"kb_wr_s":-1.0,"kb_ccwr_s":-1.0,"cswch_s":0.03,"nvcswch_s":0.0,"command":"kworker/0:2"},{"time":1646857494,"uid":1000,"pid":2201,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"minflt_s":0.09,"majflt_s":0.0,"vsz":108328,"rss":1040,"percent_mem":0.03,"stksize":132,"stkref":20,"kb_rd_s":0.0,"kb_wr_s":0.0,"kb_ccwr_s":0.0,"cswch_s":0.0,"nvcswch_s":0.0,"command":"pidstat -dlrsuwh"}] diff --git a/tests/fixtures/centos-7.7/pidstat-hl-streaming.json b/tests/fixtures/centos-7.7/pidstat-hl-streaming.json new file mode 100644 index 00000000..5502f9f2 --- /dev/null +++ b/tests/fixtures/centos-7.7/pidstat-hl-streaming.json @@ -0,0 +1 @@ +[{"time":1646859134,"uid":0,"pid":1,"percent_usr":0.0,"percent_system":0.03,"percent_guest":0.0,"percent_cpu":0.03,"cpu":0,"command":"/usr/lib/systemd/systemd --switched-root --system --deserialize 22"},{"time":1646859134,"uid":0,"pid":6,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"ksoftirqd/0"},{"time":1646859134,"uid":0,"pid":9,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"rcu_sched"},{"time":1646859134,"uid":0,"pid":11,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"watchdog/0"},{"time":1646859134,"uid":0,"pid":32,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"khugepaged"},{"time":1646859134,"uid":0,"pid":308,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"scsi_eh_1"},{"time":1646859134,"uid":0,"pid":309,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"kworker/u256:2"},{"time":1646859134,"uid":0,"pid":319,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"scsi_eh_2"},{"time":1646859134,"uid":0,"pid":356,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"kworker/u256:4"},{"time":1646859134,"uid":0,"pid":357,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"irq/16-vmwgfx"},{"time":1646859134,"uid":0,"pid":465,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"xfsaild/dm-0"},{"time":1646859134,"uid":0,"pid":466,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"kworker/0:1H"},{"time":1646859134,"uid":0,"pid":543,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/lib/systemd/systemd-journald"},{"time":1646859134,"uid":0,"pid":564,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/sbin/lvmetad -f"},{"time":1646859134,"uid":0,"pid":577,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/lib/systemd/systemd-udevd"},{"time":1646859134,"uid":0,"pid":752,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/sbin/auditd"},{"time":1646859134,"uid":0,"pid":779,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/libexec/bluetooth/bluetoothd"},{"time":1646859134,"uid":999,"pid":780,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/lib/polkit-1/polkitd --no-debug"},{"time":1646859134,"uid":0,"pid":782,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/sbin/smartd -n -q never"},{"time":1646859134,"uid":0,"pid":784,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/lib/systemd/systemd-logind"},{"time":1646859134,"uid":81,"pid":787,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation"},{"time":1646859134,"uid":998,"pid":790,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/sbin/chronyd"},{"time":1646859134,"uid":0,"pid":834,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"/usr/sbin/crond -n"},{"time":1646859134,"uid":0,"pid":847,"percent_usr":0.01,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"/usr/bin/python2 -Es /usr/sbin/firewalld --nofork --nopid"},{"time":1646859134,"uid":0,"pid":849,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/sbin/agetty --keep-baud 115200,38400,9600 ttyS0 vt220"},{"time":1646859134,"uid":0,"pid":852,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"login -- kbrazil"},{"time":1646859134,"uid":0,"pid":882,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/sbin/NetworkManager --no-daemon"},{"time":1646859134,"uid":0,"pid":1031,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/sbin/dhclient -d -q -sf /usr/libexec/nm-dhcp-helper -pf /var/run/dhclient-ens33.pid -lf /var/lib/NetworkManager/dhclient-d92ec"},{"time":1646859134,"uid":0,"pid":1220,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/sbin/sshd -D"},{"time":1646859134,"uid":0,"pid":1221,"percent_usr":0.06,"percent_system":0.03,"percent_guest":0.0,"percent_cpu":0.09,"cpu":0,"command":"/usr/bin/dockerd-current --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current --default-runtime=docker-runc --exec"},{"time":1646859134,"uid":0,"pid":1222,"percent_usr":0.01,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"/usr/bin/python2 -Es /usr/sbin/tuned -l -P"},{"time":1646859134,"uid":0,"pid":1225,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"/usr/sbin/rsyslogd -n"},{"time":1646859134,"uid":0,"pid":1293,"percent_usr":0.04,"percent_system":0.02,"percent_guest":0.0,"percent_cpu":0.05,"cpu":0,"command":"/usr/bin/docker-containerd-current -l unix:///var/run/docker/libcontainerd/docker-containerd.sock --metrics-interval=0 --start-"},{"time":1646859134,"uid":0,"pid":1496,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"/usr/libexec/postfix/master -w"},{"time":1646859134,"uid":89,"pid":1512,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"qmgr -l -t unix -u"},{"time":1646859134,"uid":1000,"pid":1852,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"-bash"},{"time":1646859134,"uid":0,"pid":1872,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"sshd: kbrazil [priv]"},{"time":1646859134,"uid":1000,"pid":1876,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"sshd: kbrazil@pts/0"},{"time":1646859134,"uid":1000,"pid":1877,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"-bash"},{"time":1646859134,"uid":0,"pid":2232,"percent_usr":0.0,"percent_system":0.01,"percent_guest":0.0,"percent_cpu":0.01,"cpu":0,"command":"kworker/0:1"},{"time":1646859134,"uid":0,"pid":2239,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"kworker/0:3"},{"time":1646859134,"uid":89,"pid":2240,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"pickup -l -t unix -u"},{"time":1646859134,"uid":0,"pid":2263,"percent_usr":0.0,"percent_system":0.0,"percent_guest":0.0,"percent_cpu":0.0,"cpu":0,"command":"kworker/0:0"}] diff --git a/tests/test_pidstat_s.py b/tests/test_pidstat_s.py new file mode 100644 index 00000000..7af3c8e4 --- /dev/null +++ b/tests/test_pidstat_s.py @@ -0,0 +1,71 @@ +import os +import json +import unittest +import jc.parsers.pidstat_s + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +# To create streaming output use: +# $ cat pidstat.out | jc --pidstat-s | jello -c > pidstat-streaming.json + + +class MyTests(unittest.TestCase): + + def setUp(self): + pass + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hl.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hl = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hdlrsuw = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hdlrsuw_2_5 = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hl-streaming.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hl_streaming_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-streaming.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hdlrsuw_streaming_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5-streaming.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_pidstat_hdlrsuw_2_5_streaming_json = json.loads(f.read()) + + def test_pidstat_s_nodata(self): + """ + Test 'pidstat' with no data + """ + self.assertEqual(list(jc.parsers.pidstat_s.parse([], quiet=True)), []) + + def test_pidstat_s_centos_7_7(self): + """ + Test 'pidstat' on Centos 7.7. Should be no output since only -h is supported + """ + self.assertEqual(list(jc.parsers.pidstat_s.parse(self.centos_7_7_pidstat.splitlines(), quiet=True)), []) + + def test_pidstat_s_hl_centos_7_7(self): + """ + Test 'pidstat -hl' on Centos 7.7 + """ + self.assertEqual(list(jc.parsers.pidstat_s.parse(self.centos_7_7_pidstat_hl.splitlines(), quiet=True)), self.centos_7_7_pidstat_hl_streaming_json) + + def test_pidstat_s_hdlrsuw_centos_7_7(self): + """ + Test 'pidstat -hdlrsuw' on Centos 7.7 + """ + self.assertEqual(list(jc.parsers.pidstat_s.parse(self.centos_7_7_pidstat_hdlrsuw.splitlines(), quiet=True)), self.centos_7_7_pidstat_hdlrsuw_streaming_json) + + def test_pidstat_s_hdlrsuw_2_5_centos_7_7(self): + """ + Test 'pidstat -hdlrsuw 2 5' on Centos 7.7 + """ + self.assertEqual(list(jc.parsers.pidstat_s.parse(self.centos_7_7_pidstat_hdlrsuw_2_5.splitlines(), quiet=True)), self.centos_7_7_pidstat_hdlrsuw_2_5_streaming_json) + + +if __name__ == '__main__': + unittest.main() From a5fb8fbf94b3b80bef82f078ad6582260b13e374 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 10:10:57 -0800 Subject: [PATCH 21/87] streaming doc update --- docs/parsers/csv_s.md | 12 ++++----- docs/parsers/iostat_s.md | 14 +++++------ docs/parsers/ls_s.md | 14 +++++------ docs/parsers/pidstat_s.md | 51 ++++++++++++++++++++++++++++----------- docs/parsers/ping_s.md | 14 +++++------ docs/parsers/rsync_s.md | 14 +++++------ docs/parsers/stat_s.md | 14 +++++------ docs/parsers/vmstat_s.md | 14 +++++------ jc/parsers/csv_s.py | 12 ++++----- jc/parsers/foo_s.py | 14 +++++------ jc/parsers/iostat_s.py | 14 +++++------ jc/parsers/ls_s.py | 14 +++++------ jc/parsers/pidstat_s.py | 51 ++++++++++++++++++++++++++++----------- jc/parsers/ping_s.py | 14 +++++------ jc/parsers/rsync_s.py | 14 +++++------ jc/parsers/stat_s.py | 14 +++++------ jc/parsers/vmstat_s.py | 14 +++++------ 17 files changed, 162 insertions(+), 146 deletions(-) diff --git a/docs/parsers/csv_s.md b/docs/parsers/csv_s.md index 4515a77b..bcbbaab2 100644 --- a/docs/parsers/csv_s.md +++ b/docs/parsers/csv_s.md @@ -44,13 +44,11 @@ Schema: "column_name2": string, # below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } Examples: diff --git a/docs/parsers/iostat_s.md b/docs/parsers/iostat_s.md index 8e598bbc..6bdafbf0 100644 --- a/docs/parsers/iostat_s.md +++ b/docs/parsers/iostat_s.md @@ -83,14 +83,12 @@ Schema: "percent_rrqm": float, "percent_wrqm": float, - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } Examples: diff --git a/docs/parsers/ls_s.md b/docs/parsers/ls_s.md index 20bc6a90..fe378ba3 100644 --- a/docs/parsers/ls_s.md +++ b/docs/parsers/ls_s.md @@ -54,14 +54,12 @@ Schema: "epoch": integer, # [0] "epoch_utc": integer, # [1] - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } [0] naive timestamp if date field exists and can be converted. diff --git a/docs/parsers/pidstat_s.md b/docs/parsers/pidstat_s.md index cc049b34..49a52d1e 100644 --- a/docs/parsers/pidstat_s.md +++ b/docs/parsers/pidstat_s.md @@ -7,7 +7,8 @@ jc - JSON Convert `pidstat` command output streaming parser > This streaming parser outputs JSON Lines -<> +Must use the `-h` option in `pidstat`. All other `pidstat` options are +supported in combination with `-h`. Usage (cli): @@ -32,26 +33,48 @@ Usage (module): Schema: { - "pidstat": string, + "time": integer, + "uid": integer, + "pid": integer, + "percent_usr": float, + "percent_system": float, + "percent_guest": float, + "percent_cpu": float, + "cpu": integer, + "minflt_s": float, + "majflt_s": float, + "vsz": integer, + "rss": integer, + "percent_mem": float, + "stksize": integer, + "stkref": integer, + "kb_rd_s": float, + "kb_wr_s": float, + "kb_ccwr_s": float, + "cswch_s": float, + "nvcswch_s": float, + "command": string, - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } Examples: - $ pidstat | jc --pidstat-s - {example output} + $ pidstat -hl | jc --pidstat-s + {"time":1646859134,"uid":0,"pid":1,"percent_usr":0.0,"percent_system...} + {"time":1646859134,"uid":0,"pid":6,"percent_usr":0.0,"percent_system...} + {"time":1646859134,"uid":0,"pid":9,"percent_usr":0.0,"percent_system...} ... - $ pidstat | jc --pidstat-s -r - {example output} + $ pidstat -hl | jc --pidstat-s -r + {"time":"1646859134","uid":"0","pid":"1","percent_usr":"0.00","perc...} + {"time":"1646859134","uid":"0","pid":"6","percent_usr":"0.00","perc...} + {"time":"1646859134","uid":"0","pid":"9","percent_usr":"0.00","perc...} ... diff --git a/docs/parsers/ping_s.md b/docs/parsers/ping_s.md index f678bbca..a9ed227d 100644 --- a/docs/parsers/ping_s.md +++ b/docs/parsers/ping_s.md @@ -61,14 +61,12 @@ Schema: "round_trip_ms_max": float, "round_trip_ms_stddev": float, - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } [0] 'reply', 'timeout', 'summary', etc. See `_error_type.type_map` diff --git a/docs/parsers/rsync_s.md b/docs/parsers/rsync_s.md index b3e41c30..7c8252e7 100644 --- a/docs/parsers/rsync_s.md +++ b/docs/parsers/rsync_s.md @@ -68,14 +68,12 @@ Schema: "extended_attribute_different": bool/null, "epoch": integer, [2] - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } [0] 'file sent', 'file received', 'local change or creation', diff --git a/docs/parsers/stat_s.md b/docs/parsers/stat_s.md index 0813541f..0cb079b8 100644 --- a/docs/parsers/stat_s.md +++ b/docs/parsers/stat_s.md @@ -68,14 +68,12 @@ Schema: "block_size": integer, "unix_flags": string, - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } Examples: diff --git a/docs/parsers/vmstat_s.md b/docs/parsers/vmstat_s.md index c2d35e39..2a148ce0 100644 --- a/docs/parsers/vmstat_s.md +++ b/docs/parsers/vmstat_s.md @@ -80,14 +80,12 @@ Schema: "epoch": integer, # [0] "epoch_utc": integer # [1] - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # [2] - "error": string, # [3] - "line": string # [3] - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # [2] + "error": string, # [3] + "line": string # [3] + } } [0] naive timestamp if -t flag is used diff --git a/jc/parsers/csv_s.py b/jc/parsers/csv_s.py index 86108659..16d0cf66 100644 --- a/jc/parsers/csv_s.py +++ b/jc/parsers/csv_s.py @@ -39,13 +39,11 @@ Schema: "column_name2": string, # below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } Examples: diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index 1113202d..0215e1f2 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -29,14 +29,12 @@ Schema: { "foo": string, - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } Examples: diff --git a/jc/parsers/iostat_s.py b/jc/parsers/iostat_s.py index 980e7908..6c52ea08 100644 --- a/jc/parsers/iostat_s.py +++ b/jc/parsers/iostat_s.py @@ -78,14 +78,12 @@ Schema: "percent_rrqm": float, "percent_wrqm": float, - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } Examples: diff --git a/jc/parsers/ls_s.py b/jc/parsers/ls_s.py index f59c1bd6..149ad683 100644 --- a/jc/parsers/ls_s.py +++ b/jc/parsers/ls_s.py @@ -49,14 +49,12 @@ Schema: "epoch": integer, # [0] "epoch_utc": integer, # [1] - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } [0] naive timestamp if date field exists and can be converted. diff --git a/jc/parsers/pidstat_s.py b/jc/parsers/pidstat_s.py index 281f82aa..8cbd6470 100644 --- a/jc/parsers/pidstat_s.py +++ b/jc/parsers/pidstat_s.py @@ -2,7 +2,8 @@ > This streaming parser outputs JSON Lines -<> +Must use the `-h` option in `pidstat`. All other `pidstat` options are +supported in combination with `-h`. Usage (cli): @@ -27,26 +28,48 @@ Usage (module): Schema: { - "pidstat": string, + "time": integer, + "uid": integer, + "pid": integer, + "percent_usr": float, + "percent_system": float, + "percent_guest": float, + "percent_cpu": float, + "cpu": integer, + "minflt_s": float, + "majflt_s": float, + "vsz": integer, + "rss": integer, + "percent_mem": float, + "stksize": integer, + "stkref": integer, + "kb_rd_s": float, + "kb_wr_s": float, + "kb_ccwr_s": float, + "cswch_s": float, + "nvcswch_s": float, + "command": string, - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } Examples: - $ pidstat | jc --pidstat-s - {example output} + $ pidstat -hl | jc --pidstat-s + {"time":1646859134,"uid":0,"pid":1,"percent_usr":0.0,"percent_system...} + {"time":1646859134,"uid":0,"pid":6,"percent_usr":0.0,"percent_system...} + {"time":1646859134,"uid":0,"pid":9,"percent_usr":0.0,"percent_system...} ... - $ pidstat | jc --pidstat-s -r - {example output} + $ pidstat -hl | jc --pidstat-s -r + {"time":"1646859134","uid":"0","pid":"1","percent_usr":"0.00","perc...} + {"time":"1646859134","uid":"0","pid":"6","percent_usr":"0.00","perc...} + {"time":"1646859134","uid":"0","pid":"9","percent_usr":"0.00","perc...} ... """ from typing import Dict, Iterable, Union diff --git a/jc/parsers/ping_s.py b/jc/parsers/ping_s.py index 34ce7369..1c74cea2 100644 --- a/jc/parsers/ping_s.py +++ b/jc/parsers/ping_s.py @@ -56,14 +56,12 @@ Schema: "round_trip_ms_max": float, "round_trip_ms_stddev": float, - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } [0] 'reply', 'timeout', 'summary', etc. See `_error_type.type_map` diff --git a/jc/parsers/rsync_s.py b/jc/parsers/rsync_s.py index 94b874e3..c55cab4a 100644 --- a/jc/parsers/rsync_s.py +++ b/jc/parsers/rsync_s.py @@ -63,14 +63,12 @@ Schema: "extended_attribute_different": bool/null, "epoch": integer, [2] - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } [0] 'file sent', 'file received', 'local change or creation', diff --git a/jc/parsers/stat_s.py b/jc/parsers/stat_s.py index 3b302e38..aa790ef2 100644 --- a/jc/parsers/stat_s.py +++ b/jc/parsers/stat_s.py @@ -63,14 +63,12 @@ Schema: "block_size": integer, "unix_flags": string, - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # false if error parsing - "error": string, # exists if "success" is false - "line": string # exists if "success" is false - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } } Examples: diff --git a/jc/parsers/vmstat_s.py b/jc/parsers/vmstat_s.py index 5e2899ea..75e480d8 100644 --- a/jc/parsers/vmstat_s.py +++ b/jc/parsers/vmstat_s.py @@ -75,14 +75,12 @@ Schema: "epoch": integer, # [0] "epoch_utc": integer # [1] - # Below object only exists if using -qq or ignore_exceptions=True - - "_jc_meta": - { - "success": boolean, # [2] - "error": string, # [3] - "line": string # [3] - } + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # [2] + "error": string, # [3] + "line": string # [3] + } } [0] naive timestamp if -t flag is used From 8aee4517bb4c2eebdd9873f7eb0827cd33b85519 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 13:02:50 -0800 Subject: [PATCH 22/87] doc formatting --- docs/parsers/pidstat_s.md | 19 ++++++------------- jc/parsers/pidstat_s.py | 19 ++++++------------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/docs/parsers/pidstat_s.md b/docs/parsers/pidstat_s.md index 49a52d1e..5d185262 100644 --- a/docs/parsers/pidstat_s.md +++ b/docs/parsers/pidstat_s.md @@ -5,7 +5,8 @@ jc - JSON Convert `pidstat` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator +iterator of Dictionaries (module) Must use the `-h` option in `pidstat`. All other `pidstat` options are supported in combination with `-h`. @@ -17,19 +18,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('pidstat_s', pidstat_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.pidstat_s - # result is an iterable object (generator) - result = jc.parsers.pidstat_s.parse(pidstat_command_output.splitlines()) - for item in result: - # do something - Schema: { @@ -66,9 +59,9 @@ Schema: Examples: $ pidstat -hl | jc --pidstat-s - {"time":1646859134,"uid":0,"pid":1,"percent_usr":0.0,"percent_system...} - {"time":1646859134,"uid":0,"pid":6,"percent_usr":0.0,"percent_system...} - {"time":1646859134,"uid":0,"pid":9,"percent_usr":0.0,"percent_system...} + {"time":1646859134,"uid":0,"pid":1,"percent_usr":0.0,"percent_syste...} + {"time":1646859134,"uid":0,"pid":6,"percent_usr":0.0,"percent_syste...} + {"time":1646859134,"uid":0,"pid":9,"percent_usr":0.0,"percent_syste...} ... $ pidstat -hl | jc --pidstat-s -r diff --git a/jc/parsers/pidstat_s.py b/jc/parsers/pidstat_s.py index 8cbd6470..19977598 100644 --- a/jc/parsers/pidstat_s.py +++ b/jc/parsers/pidstat_s.py @@ -1,6 +1,7 @@ """jc - JSON Convert `pidstat` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator +iterator of Dictionaries (module) Must use the `-h` option in `pidstat`. All other `pidstat` options are supported in combination with `-h`. @@ -12,19 +13,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('pidstat_s', pidstat_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.pidstat_s - # result is an iterable object (generator) - result = jc.parsers.pidstat_s.parse(pidstat_command_output.splitlines()) - for item in result: - # do something - Schema: { @@ -61,9 +54,9 @@ Schema: Examples: $ pidstat -hl | jc --pidstat-s - {"time":1646859134,"uid":0,"pid":1,"percent_usr":0.0,"percent_system...} - {"time":1646859134,"uid":0,"pid":6,"percent_usr":0.0,"percent_system...} - {"time":1646859134,"uid":0,"pid":9,"percent_usr":0.0,"percent_system...} + {"time":1646859134,"uid":0,"pid":1,"percent_usr":0.0,"percent_syste...} + {"time":1646859134,"uid":0,"pid":6,"percent_usr":0.0,"percent_syste...} + {"time":1646859134,"uid":0,"pid":9,"percent_usr":0.0,"percent_syste...} ... $ pidstat -hl | jc --pidstat-s -r From b784db404d0569a4c630eb124b37b0b1d0846453 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 13:32:26 -0800 Subject: [PATCH 23/87] streaming doc formatting update --- docs/parsers/csv_s.md | 13 +++---------- docs/parsers/iostat_s.md | 20 ++++++++++---------- docs/parsers/ls_s.md | 16 ++++------------ docs/parsers/pidstat_s.md | 9 ++++++++- docs/parsers/ping_s.md | 13 +++---------- docs/parsers/rsync_s.md | 13 +++---------- docs/parsers/stat_s.md | 13 +++---------- docs/parsers/vmstat_s.md | 23 ++++++++--------------- jc/parsers/csv_s.py | 13 +++---------- jc/parsers/foo_s.py | 13 +++---------- jc/parsers/iostat_s.py | 20 ++++++++++---------- jc/parsers/ls_s.py | 16 ++++------------ jc/parsers/pidstat_s.py | 9 ++++++++- jc/parsers/ping_s.py | 13 +++---------- jc/parsers/rsync_s.py | 13 +++---------- jc/parsers/stat_s.py | 13 +++---------- jc/parsers/vmstat_s.py | 23 ++++++++--------------- 17 files changed, 87 insertions(+), 166 deletions(-) diff --git a/docs/parsers/csv_s.md b/docs/parsers/csv_s.md index bcbbaab2..6b096ee2 100644 --- a/docs/parsers/csv_s.md +++ b/docs/parsers/csv_s.md @@ -5,7 +5,8 @@ jc - JSON Convert `csv` file streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) The `csv` streaming parser will attempt to automatically detect the delimiter character. If the delimiter cannot be detected it will default @@ -21,19 +22,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('csv_s', csv_output.splitlines()) for item in result: # do something - or - - import jc.parsers.csv_s - # result is an iterable object (generator) - result = jc.parsers.csv_s.parse(csv_output.splitlines()) - for item in result: - # do something - Schema: csv file converted to a Dictionary: diff --git a/docs/parsers/iostat_s.md b/docs/parsers/iostat_s.md index 6bdafbf0..698d15d8 100644 --- a/docs/parsers/iostat_s.md +++ b/docs/parsers/iostat_s.md @@ -5,7 +5,8 @@ jc - JSON Convert `iostat` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) Note: `iostat` version 11 and higher include a JSON output option @@ -13,22 +14,21 @@ Usage (cli): $ iostat | jc --iostat-s +> Note: When piping `jc` converted `iostat` output to other processes it may + appear the output is hanging due to the OS pipe buffers. This is because + `iostat` output is too small to quickly fill up the buffer. Use the `-u` + option to unbuffer the `jc` output if you would like immediate output. See + the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) + for more information. + Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('iostat_s', iostat_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.iostat_s - # result is an iterable object (generator) - result = jc.parsers.iostat_s.parse(iostat_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/docs/parsers/ls_s.md b/docs/parsers/ls_s.md index fe378ba3..02097f98 100644 --- a/docs/parsers/ls_s.md +++ b/docs/parsers/ls_s.md @@ -3,10 +3,10 @@ # jc.parsers.ls\_s -jc - JSON Convert `ls` and `vdir` command output streaming -parser +jc - JSON Convert `ls` and `vdir` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) Requires the `-l` option to be used on `ls`. If there are newline characters in the filename, then make sure to use the `-b` option on `ls`. @@ -27,19 +27,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('ls_s', ls_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.ls_s - # result is an iterable object (generator) - result = jc.parsers.ls_s.parse(ls_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/docs/parsers/pidstat_s.md b/docs/parsers/pidstat_s.md index 5d185262..d94feebb 100644 --- a/docs/parsers/pidstat_s.md +++ b/docs/parsers/pidstat_s.md @@ -6,7 +6,7 @@ jc - JSON Convert `pidstat` command output streaming parser > This streaming parser outputs JSON Lines (cli) or returns a Generator -iterator of Dictionaries (module) + iterator of Dictionaries (module) Must use the `-h` option in `pidstat`. All other `pidstat` options are supported in combination with `-h`. @@ -15,6 +15,13 @@ Usage (cli): $ pidstat | jc --pidstat-s +> Note: When piping `jc` converted `pidstat` output to other processes it + may appear the output is hanging due to the OS pipe buffers. This is + because `pidstat` output is too small to quickly fill up the buffer. Use + the `-u` option to unbuffer the `jc` output if you would like immediate + output. See the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) + for more information. + Usage (module): import jc diff --git a/docs/parsers/ping_s.md b/docs/parsers/ping_s.md index a9ed227d..86f22ba4 100644 --- a/docs/parsers/ping_s.md +++ b/docs/parsers/ping_s.md @@ -5,7 +5,8 @@ jc - JSON Convert `ping` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) Supports `ping` and `ping6` output. @@ -23,19 +24,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('ping_s', ping_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.ping_s - # result is an iterable object (generator) - result = jc.parsers.ping_s.parse(ping_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/docs/parsers/rsync_s.md b/docs/parsers/rsync_s.md index 7c8252e7..a77a2a2e 100644 --- a/docs/parsers/rsync_s.md +++ b/docs/parsers/rsync_s.md @@ -5,7 +5,8 @@ jc - JSON Convert `rsync` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) Supports the `-i` or `--itemize-changes` options with all levels of verbosity. This parser will process the STDOUT output or a log file @@ -22,19 +23,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('rsync_s', rsync_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.rsync_s - # result is an iterable object (generator) - result = jc.parsers.rsync_s.parse(rsync_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/docs/parsers/stat_s.md b/docs/parsers/stat_s.md index 0cb079b8..2ea8e03f 100644 --- a/docs/parsers/stat_s.md +++ b/docs/parsers/stat_s.md @@ -5,7 +5,8 @@ jc - JSON Convert `stat` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) The `xxx_epoch` calculated timestamp fields are naive. (i.e. based on the local time of the system the parser is run on). @@ -20,19 +21,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('stat_s', stat_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.stat_s - # result is an iterable object (generator) - result = jc.parsers.stat_s.parse(stat_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/docs/parsers/vmstat_s.md b/docs/parsers/vmstat_s.md index 2a148ce0..b92bd283 100644 --- a/docs/parsers/vmstat_s.md +++ b/docs/parsers/vmstat_s.md @@ -5,7 +5,8 @@ jc - JSON Convert `vmstat` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) Options supported: `-a`, `-w`, `-d`, `-t` @@ -20,28 +21,20 @@ Usage (cli): $ vmstat | jc --vmstat-s > Note: When piping `jc` converted `vmstat` output to other processes it may -appear the output is hanging due to the OS pipe buffers. This is because -`vmstat` output is too small to quickly fill up the buffer. Use the `-u` -option to unbuffer the `jc` output if you would like immediate output. See -the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) -for more information. + appear the output is hanging due to the OS pipe buffers. This is because + `vmstat` output is too small to quickly fill up the buffer. Use the `-u` + option to unbuffer the `jc` output if you would like immediate output. See + the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) + for more information. Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('vmstat_s', vmstat_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.vmstat_s - # result is an iterable object (generator) - result = jc.parsers.vmstat_s.parse(vmstat_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/jc/parsers/csv_s.py b/jc/parsers/csv_s.py index 16d0cf66..8434389d 100644 --- a/jc/parsers/csv_s.py +++ b/jc/parsers/csv_s.py @@ -1,6 +1,7 @@ """jc - JSON Convert `csv` file streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) The `csv` streaming parser will attempt to automatically detect the delimiter character. If the delimiter cannot be detected it will default @@ -16,19 +17,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('csv_s', csv_output.splitlines()) for item in result: # do something - or - - import jc.parsers.csv_s - # result is an iterable object (generator) - result = jc.parsers.csv_s.parse(csv_output.splitlines()) - for item in result: - # do something - Schema: csv file converted to a Dictionary: diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index 0215e1f2..229ac345 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -1,6 +1,7 @@ """jc - JSON Convert `foo` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) <> @@ -11,19 +12,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('foo_s', foo_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.foo_s - # result is an iterable object (generator) - result = jc.parsers.foo_s.parse(foo_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/jc/parsers/iostat_s.py b/jc/parsers/iostat_s.py index 6c52ea08..9df7e14c 100644 --- a/jc/parsers/iostat_s.py +++ b/jc/parsers/iostat_s.py @@ -1,6 +1,7 @@ """jc - JSON Convert `iostat` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) Note: `iostat` version 11 and higher include a JSON output option @@ -8,22 +9,21 @@ Usage (cli): $ iostat | jc --iostat-s +> Note: When piping `jc` converted `iostat` output to other processes it may + appear the output is hanging due to the OS pipe buffers. This is because + `iostat` output is too small to quickly fill up the buffer. Use the `-u` + option to unbuffer the `jc` output if you would like immediate output. See + the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) + for more information. + Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('iostat_s', iostat_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.iostat_s - # result is an iterable object (generator) - result = jc.parsers.iostat_s.parse(iostat_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/jc/parsers/ls_s.py b/jc/parsers/ls_s.py index 149ad683..003383b0 100644 --- a/jc/parsers/ls_s.py +++ b/jc/parsers/ls_s.py @@ -1,7 +1,7 @@ -"""jc - JSON Convert `ls` and `vdir` command output streaming -parser +"""jc - JSON Convert `ls` and `vdir` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) Requires the `-l` option to be used on `ls`. If there are newline characters in the filename, then make sure to use the `-b` option on `ls`. @@ -22,19 +22,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('ls_s', ls_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.ls_s - # result is an iterable object (generator) - result = jc.parsers.ls_s.parse(ls_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/jc/parsers/pidstat_s.py b/jc/parsers/pidstat_s.py index 19977598..3a7b7b62 100644 --- a/jc/parsers/pidstat_s.py +++ b/jc/parsers/pidstat_s.py @@ -1,7 +1,7 @@ """jc - JSON Convert `pidstat` command output streaming parser > This streaming parser outputs JSON Lines (cli) or returns a Generator -iterator of Dictionaries (module) + iterator of Dictionaries (module) Must use the `-h` option in `pidstat`. All other `pidstat` options are supported in combination with `-h`. @@ -10,6 +10,13 @@ Usage (cli): $ pidstat | jc --pidstat-s +> Note: When piping `jc` converted `pidstat` output to other processes it + may appear the output is hanging due to the OS pipe buffers. This is + because `pidstat` output is too small to quickly fill up the buffer. Use + the `-u` option to unbuffer the `jc` output if you would like immediate + output. See the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) + for more information. + Usage (module): import jc diff --git a/jc/parsers/ping_s.py b/jc/parsers/ping_s.py index 1c74cea2..2acea940 100644 --- a/jc/parsers/ping_s.py +++ b/jc/parsers/ping_s.py @@ -1,6 +1,7 @@ """jc - JSON Convert `ping` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) Supports `ping` and `ping6` output. @@ -18,19 +19,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('ping_s', ping_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.ping_s - # result is an iterable object (generator) - result = jc.parsers.ping_s.parse(ping_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/jc/parsers/rsync_s.py b/jc/parsers/rsync_s.py index c55cab4a..4ecfe9d2 100644 --- a/jc/parsers/rsync_s.py +++ b/jc/parsers/rsync_s.py @@ -1,6 +1,7 @@ """jc - JSON Convert `rsync` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) Supports the `-i` or `--itemize-changes` options with all levels of verbosity. This parser will process the STDOUT output or a log file @@ -17,19 +18,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('rsync_s', rsync_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.rsync_s - # result is an iterable object (generator) - result = jc.parsers.rsync_s.parse(rsync_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/jc/parsers/stat_s.py b/jc/parsers/stat_s.py index aa790ef2..3c3f9271 100644 --- a/jc/parsers/stat_s.py +++ b/jc/parsers/stat_s.py @@ -1,6 +1,7 @@ """jc - JSON Convert `stat` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) The `xxx_epoch` calculated timestamp fields are naive. (i.e. based on the local time of the system the parser is run on). @@ -15,19 +16,11 @@ Usage (cli): Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('stat_s', stat_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.stat_s - # result is an iterable object (generator) - result = jc.parsers.stat_s.parse(stat_command_output.splitlines()) - for item in result: - # do something - Schema: { diff --git a/jc/parsers/vmstat_s.py b/jc/parsers/vmstat_s.py index 75e480d8..cdb53c4a 100644 --- a/jc/parsers/vmstat_s.py +++ b/jc/parsers/vmstat_s.py @@ -1,6 +1,7 @@ """jc - JSON Convert `vmstat` command output streaming parser -> This streaming parser outputs JSON Lines +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) Options supported: `-a`, `-w`, `-d`, `-t` @@ -15,28 +16,20 @@ Usage (cli): $ vmstat | jc --vmstat-s > Note: When piping `jc` converted `vmstat` output to other processes it may -appear the output is hanging due to the OS pipe buffers. This is because -`vmstat` output is too small to quickly fill up the buffer. Use the `-u` -option to unbuffer the `jc` output if you would like immediate output. See -the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) -for more information. + appear the output is hanging due to the OS pipe buffers. This is because + `vmstat` output is too small to quickly fill up the buffer. Use the `-u` + option to unbuffer the `jc` output if you would like immediate output. See + the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) + for more information. Usage (module): import jc - # result is an iterable object (generator) + result = jc.parse('vmstat_s', vmstat_command_output.splitlines()) for item in result: # do something - or - - import jc.parsers.vmstat_s - # result is an iterable object (generator) - result = jc.parsers.vmstat_s.parse(vmstat_command_output.splitlines()) - for item in result: - # do something - Schema: { From 6986c74f6dceca907ea2e469a332deeb093a122c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 15:18:27 -0800 Subject: [PATCH 24/87] remove direct parser module example from doc --- docs/parsers/acpi.md | 5 ----- docs/parsers/airport.md | 5 ----- docs/parsers/airport_s.md | 5 ----- docs/parsers/arp.md | 5 ----- docs/parsers/blkid.md | 5 ----- docs/parsers/cksum.md | 5 ----- docs/parsers/crontab.md | 5 ----- docs/parsers/crontab_u.md | 5 ----- docs/parsers/csv.md | 5 ----- docs/parsers/date.md | 5 ----- docs/parsers/df.md | 5 ----- docs/parsers/dig.md | 5 ----- docs/parsers/dir.md | 5 ----- docs/parsers/dmidecode.md | 5 ----- docs/parsers/dpkg_l.md | 5 ----- docs/parsers/du.md | 5 ----- docs/parsers/env.md | 5 ----- docs/parsers/file.md | 5 ----- docs/parsers/finger.md | 5 ----- docs/parsers/free.md | 5 ----- docs/parsers/fstab.md | 5 ----- docs/parsers/group.md | 5 ----- docs/parsers/gshadow.md | 5 ----- docs/parsers/hash.md | 5 ----- docs/parsers/hashsum.md | 5 ----- docs/parsers/hciconfig.md | 5 ----- docs/parsers/history.md | 5 ----- docs/parsers/hosts.md | 5 ----- docs/parsers/id.md | 5 ----- docs/parsers/ifconfig.md | 5 ----- docs/parsers/ini.md | 5 ----- docs/parsers/iostat.md | 5 ----- docs/parsers/iptables.md | 5 ----- docs/parsers/iw_scan.md | 5 ----- docs/parsers/jar_manifest.md | 5 ----- docs/parsers/jobs.md | 5 ----- docs/parsers/kv.md | 5 ----- docs/parsers/last.md | 5 ----- docs/parsers/ls.md | 5 ----- docs/parsers/lsblk.md | 5 ----- docs/parsers/lsmod.md | 5 ----- docs/parsers/lsof.md | 5 ----- docs/parsers/lsusb.md | 5 ----- docs/parsers/mount.md | 5 ----- docs/parsers/netstat.md | 5 ----- docs/parsers/nmcli.md | 5 ----- docs/parsers/ntpq.md | 5 ----- docs/parsers/passwd.md | 5 ----- docs/parsers/pidstat.md | 5 ----- docs/parsers/ping.md | 5 ----- docs/parsers/pip_list.md | 5 ----- docs/parsers/pip_show.md | 5 ----- docs/parsers/ps.md | 5 ----- docs/parsers/route.md | 5 ----- docs/parsers/rpm_qi.md | 5 ----- docs/parsers/rsync.md | 5 ----- docs/parsers/sfdisk.md | 5 ----- docs/parsers/shadow.md | 5 ----- docs/parsers/ss.md | 5 ----- docs/parsers/stat.md | 5 ----- docs/parsers/sysctl.md | 5 ----- docs/parsers/systemctl.md | 5 ----- docs/parsers/systemctl_lj.md | 5 ----- docs/parsers/systemctl_ls.md | 5 ----- docs/parsers/systemctl_luf.md | 5 ----- docs/parsers/systeminfo.md | 5 ----- docs/parsers/time.md | 5 ----- docs/parsers/timedatectl.md | 5 ----- docs/parsers/tracepath.md | 5 ----- docs/parsers/traceroute.md | 5 ----- docs/parsers/ufw.md | 5 ----- docs/parsers/ufw_appinfo.md | 5 ----- docs/parsers/uname.md | 5 ----- docs/parsers/universal.md | 18 +++++++++++++++++- docs/parsers/upower.md | 5 ----- docs/parsers/uptime.md | 5 ----- docs/parsers/vmstat.md | 5 ----- docs/parsers/w.md | 5 ----- docs/parsers/wc.md | 5 ----- docs/parsers/who.md | 5 ----- docs/parsers/xml.md | 5 ----- docs/parsers/xrandr.md | 5 ----- docs/parsers/yaml.md | 5 ----- docs/parsers/zipinfo.md | 5 ----- jc/parsers/acpi.py | 5 ----- jc/parsers/airport.py | 5 ----- jc/parsers/airport_s.py | 5 ----- jc/parsers/arp.py | 5 ----- jc/parsers/blkid.py | 5 ----- jc/parsers/cksum.py | 5 ----- jc/parsers/crontab.py | 5 ----- jc/parsers/crontab_u.py | 5 ----- jc/parsers/csv.py | 5 ----- jc/parsers/date.py | 5 ----- jc/parsers/df.py | 5 ----- jc/parsers/dig.py | 5 ----- jc/parsers/dir.py | 5 ----- jc/parsers/dmidecode.py | 5 ----- jc/parsers/dpkg_l.py | 5 ----- jc/parsers/du.py | 5 ----- jc/parsers/env.py | 5 ----- jc/parsers/file.py | 5 ----- jc/parsers/finger.py | 5 ----- jc/parsers/foo.py | 5 ----- jc/parsers/free.py | 5 ----- jc/parsers/fstab.py | 5 ----- jc/parsers/group.py | 5 ----- jc/parsers/gshadow.py | 5 ----- jc/parsers/hash.py | 5 ----- jc/parsers/hashsum.py | 5 ----- jc/parsers/hciconfig.py | 5 ----- jc/parsers/history.py | 5 ----- jc/parsers/hosts.py | 5 ----- jc/parsers/id.py | 5 ----- jc/parsers/ifconfig.py | 5 ----- jc/parsers/ini.py | 5 ----- jc/parsers/iostat.py | 5 ----- jc/parsers/iptables.py | 5 ----- jc/parsers/iw_scan.py | 5 ----- jc/parsers/jar_manifest.py | 5 ----- jc/parsers/jobs.py | 5 ----- jc/parsers/kv.py | 5 ----- jc/parsers/last.py | 5 ----- jc/parsers/ls.py | 5 ----- jc/parsers/lsblk.py | 5 ----- jc/parsers/lsmod.py | 5 ----- jc/parsers/lsof.py | 5 ----- jc/parsers/lsusb.py | 5 ----- jc/parsers/mount.py | 5 ----- jc/parsers/netstat.py | 5 ----- jc/parsers/nmcli.py | 5 ----- jc/parsers/ntpq.py | 5 ----- jc/parsers/passwd.py | 5 ----- jc/parsers/pidstat.py | 5 ----- jc/parsers/ping.py | 5 ----- jc/parsers/pip_list.py | 5 ----- jc/parsers/pip_show.py | 5 ----- jc/parsers/ps.py | 5 ----- jc/parsers/route.py | 5 ----- jc/parsers/rpm_qi.py | 5 ----- jc/parsers/rsync.py | 5 ----- jc/parsers/sfdisk.py | 5 ----- jc/parsers/shadow.py | 5 ----- jc/parsers/ss.py | 5 ----- jc/parsers/stat.py | 5 ----- jc/parsers/sysctl.py | 5 ----- jc/parsers/systemctl.py | 5 ----- jc/parsers/systemctl_lj.py | 5 ----- jc/parsers/systemctl_ls.py | 5 ----- jc/parsers/systemctl_luf.py | 5 ----- jc/parsers/systeminfo.py | 5 ----- jc/parsers/time.py | 5 ----- jc/parsers/timedatectl.py | 5 ----- jc/parsers/tracepath.py | 5 ----- jc/parsers/traceroute.py | 5 ----- jc/parsers/ufw.py | 5 ----- jc/parsers/ufw_appinfo.py | 5 ----- jc/parsers/uname.py | 5 ----- jc/parsers/universal.py | 18 +++++++++++++++++- jc/parsers/upower.py | 5 ----- jc/parsers/uptime.py | 5 ----- jc/parsers/vmstat.py | 5 ----- jc/parsers/w.py | 5 ----- jc/parsers/wc.py | 5 ----- jc/parsers/who.py | 5 ----- jc/parsers/xml.py | 5 ----- jc/parsers/xrandr.py | 5 ----- jc/parsers/yaml.py | 5 ----- jc/parsers/zipinfo.py | 5 ----- 169 files changed, 34 insertions(+), 837 deletions(-) diff --git a/docs/parsers/acpi.md b/docs/parsers/acpi.md index 98202760..b93d3fe7 100644 --- a/docs/parsers/acpi.md +++ b/docs/parsers/acpi.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('acpi', acpi_command_output) - or - - import jc.parsers.acpi - result = jc.parsers.acpi.parse(acpi_command_output) - Schema: [ diff --git a/docs/parsers/airport.md b/docs/parsers/airport.md index 4ec9bad3..bb9aef39 100644 --- a/docs/parsers/airport.md +++ b/docs/parsers/airport.md @@ -20,11 +20,6 @@ Usage (module): import jc result = jc.parse('airport', airport_command_output) - or - - import jc.parsers.airport - result = jc.parsers.airport.parse(airport_command_output) - Schema: { diff --git a/docs/parsers/airport_s.md b/docs/parsers/airport_s.md index b4354c45..6f217e9d 100644 --- a/docs/parsers/airport_s.md +++ b/docs/parsers/airport_s.md @@ -20,11 +20,6 @@ Usage (module): import jc result = jc.parse('airport_s', airport_s_command_output) - or - - import jc.parsers.airport_s - result = jc.parsers.airport_s.parse(airport_s_command_output) - Schema: [ diff --git a/docs/parsers/arp.md b/docs/parsers/arp.md index a6c9b966..29b60625 100644 --- a/docs/parsers/arp.md +++ b/docs/parsers/arp.md @@ -20,11 +20,6 @@ Usage (module): import jc result = jc.parse('arp', arp_command_output) - or - - import jc.parsers.arp - result = jc.parsers.arp.parse(arp_command_output) - Schema: [ diff --git a/docs/parsers/blkid.md b/docs/parsers/blkid.md index 09fb8abe..ceff074f 100644 --- a/docs/parsers/blkid.md +++ b/docs/parsers/blkid.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('blkid', blkid_command_output) - or - - import jc.parsers.blkid - result = jc.parsers.blkid.parse(blkid_command_output) - Schema: [ diff --git a/docs/parsers/cksum.md b/docs/parsers/cksum.md index 2ab0ecf6..242b5a2a 100644 --- a/docs/parsers/cksum.md +++ b/docs/parsers/cksum.md @@ -22,11 +22,6 @@ Usage (module): import jc result = jc.parse('cksum', cksum_command_output) - or - - import jc.parsers.cksum - result = jc.parsers.cksum.parse(cksum_command_output) - Schema: [ diff --git a/docs/parsers/crontab.md b/docs/parsers/crontab.md index dfd27bf6..74b91cdf 100644 --- a/docs/parsers/crontab.md +++ b/docs/parsers/crontab.md @@ -21,11 +21,6 @@ Usage (module): import jc result = jc.parse('crontab', crontab_output) - or - - import jc.parsers.crontab - result = jc.parsers.crontab.parse(crontab_output) - Schema: { diff --git a/docs/parsers/crontab_u.md b/docs/parsers/crontab_u.md index 0ae1817d..a0ba9ca9 100644 --- a/docs/parsers/crontab_u.md +++ b/docs/parsers/crontab_u.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('crontab_u', crontab_u_output) - or - - import jc.parsers.crontab_u - result = jc.parsers.crontab_u.parse(crontab_u_output) - Schema: { diff --git a/docs/parsers/csv.md b/docs/parsers/csv.md index 16c65905..0807a0b0 100644 --- a/docs/parsers/csv.md +++ b/docs/parsers/csv.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('csv', csv_output) - or - - import jc.parsers.csv - result = jc.parsers.csv.parse(csv_output) - Schema: csv file converted to a Dictionary: diff --git a/docs/parsers/date.md b/docs/parsers/date.md index 82f8bd87..f2d7d3f9 100644 --- a/docs/parsers/date.md +++ b/docs/parsers/date.md @@ -24,11 +24,6 @@ Usage (module): import jc result = jc.parse('date', date_command_output) - or - - import jc.parsers.date - result = jc.parsers.date.parse(date_command_output) - Schema: { diff --git a/docs/parsers/df.md b/docs/parsers/df.md index 717a9637..90fbb415 100644 --- a/docs/parsers/df.md +++ b/docs/parsers/df.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('df', df_command_output) - or - - import jc.parsers.df - result = jc.parsers.df.parse(df_command_output) - Schema: [ diff --git a/docs/parsers/dig.md b/docs/parsers/dig.md index 4903fc82..6357038a 100644 --- a/docs/parsers/dig.md +++ b/docs/parsers/dig.md @@ -29,11 +29,6 @@ Usage (module): import jc result = jc.parse('dig', dig_command_output) - or - - import jc.parsers.dig - result = jc.parsers.dig.parse(dig_command_output) - Schema: [ diff --git a/docs/parsers/dir.md b/docs/parsers/dir.md index c04e4de6..47492b77 100644 --- a/docs/parsers/dir.md +++ b/docs/parsers/dir.md @@ -26,11 +26,6 @@ Usage (module): import jc result = jc.parse('dir', dir_command_output) - or - - import jc.parsers.dir - result = jc.parsers.dir.parse(dir_command_output) - Schema: [ diff --git a/docs/parsers/dmidecode.md b/docs/parsers/dmidecode.md index 8c7072ac..fb283c2d 100644 --- a/docs/parsers/dmidecode.md +++ b/docs/parsers/dmidecode.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('dmidecode', dmidecode_command_output) - or - - import jc.parsers.dmidecode - result = jc.parsers.dmidecode.parse(dmidecode_command_output) - Schema: [ diff --git a/docs/parsers/dpkg_l.md b/docs/parsers/dpkg_l.md index 97ed8209..3f5ef6e1 100644 --- a/docs/parsers/dpkg_l.md +++ b/docs/parsers/dpkg_l.md @@ -23,11 +23,6 @@ Usage (module): import jc result = jc.parse('dpkg_l', dpkg_command_output) - or - - import jc.parsers.dpkg_l - result = jc.parsers.dpkg_l.parse(dpkg_command_output) - Schema: [ diff --git a/docs/parsers/du.md b/docs/parsers/du.md index fe2f5511..0c4751a4 100644 --- a/docs/parsers/du.md +++ b/docs/parsers/du.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('du', du_command_output) - or - - import jc.parsers.du - result = jc.parsers.du.parse(du_command_output) - Schema: [ diff --git a/docs/parsers/env.md b/docs/parsers/env.md index 1b951549..89d10ada 100644 --- a/docs/parsers/env.md +++ b/docs/parsers/env.md @@ -23,11 +23,6 @@ Usage (module): import jc result = jc.parse('env', env_command_output) - or - - import jc.parsers.env - result = jc.parsers.env.parse(env_command_output) - Schema: [ diff --git a/docs/parsers/file.md b/docs/parsers/file.md index 25895fe7..71ab2b8e 100644 --- a/docs/parsers/file.md +++ b/docs/parsers/file.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('file', file_command_output) - or - - import jc.parsers.file - result = jc.parsers.file.parse(file_command_output) - Schema: [ diff --git a/docs/parsers/finger.md b/docs/parsers/finger.md index b9445add..438bba05 100644 --- a/docs/parsers/finger.md +++ b/docs/parsers/finger.md @@ -20,11 +20,6 @@ Usage (module): import jc result = jc.parse('finger', finger_command_output) - or - - import jc.parsers.finger - result = jc.parsers.finger.parse(finger_command_output) - Schema: [ diff --git a/docs/parsers/free.md b/docs/parsers/free.md index 5f540ace..588846b5 100644 --- a/docs/parsers/free.md +++ b/docs/parsers/free.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('free', free_command_output) - or - - import jc.parsers.free - result = jc.parsers.free.parse(free_command_output) - Schema: [ diff --git a/docs/parsers/fstab.md b/docs/parsers/fstab.md index 60c6555e..16f649e4 100644 --- a/docs/parsers/fstab.md +++ b/docs/parsers/fstab.md @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('fstab', fstab_command_output) - or - - import jc.parsers.fstab - result = jc.parsers.fstab.parse(fstab_command_output) - Schema: [ diff --git a/docs/parsers/group.md b/docs/parsers/group.md index 33ca0cdf..e0ea9ea9 100644 --- a/docs/parsers/group.md +++ b/docs/parsers/group.md @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('group', group_file_output) - or - - import jc.parsers.group - result = jc.parsers.group.parse(group_file_output) - Schema: [ diff --git a/docs/parsers/gshadow.md b/docs/parsers/gshadow.md index e48012f9..1f258ae8 100644 --- a/docs/parsers/gshadow.md +++ b/docs/parsers/gshadow.md @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('gshadow', gshadow_file_output) - or - - import jc.parsers.gshadow - result = jc.parsers.gshadow.parse(gshadow_file_output) - Schema: [ diff --git a/docs/parsers/hash.md b/docs/parsers/hash.md index 74cde8f5..c5071968 100644 --- a/docs/parsers/hash.md +++ b/docs/parsers/hash.md @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('hash', hash_command_output) - or - - import jc.parsers.hash - result = jc.parsers.hash.parse(hash_command_output) - Schema: [ diff --git a/docs/parsers/hashsum.md b/docs/parsers/hashsum.md index b8b76634..e139e98a 100644 --- a/docs/parsers/hashsum.md +++ b/docs/parsers/hashsum.md @@ -28,11 +28,6 @@ Usage (module): import jc result = jc.parse('hashsum', md5sum_command_output) - or - - import jc.parsers.hashsum - result = jc.parsers.hashsum.parse(md5sum_command_output) - Schema: [ diff --git a/docs/parsers/hciconfig.md b/docs/parsers/hciconfig.md index 240cc50c..6c4d5c19 100644 --- a/docs/parsers/hciconfig.md +++ b/docs/parsers/hciconfig.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('hciconfig', hciconfig_command_output) - or - - import jc.parsers.hciconfig - result = jc.parsers.hciconfig.parse(hciconfig_command_output) - Schema: [ diff --git a/docs/parsers/history.md b/docs/parsers/history.md index 5edd0ce6..2151e0ff 100644 --- a/docs/parsers/history.md +++ b/docs/parsers/history.md @@ -22,11 +22,6 @@ Usage (module): import jc result = jc.parse('history', history_command_output) - or - - import jc.parsers.history - result = jc.parsers.history.parse(history_command_output) - Schema: [ diff --git a/docs/parsers/hosts.md b/docs/parsers/hosts.md index dd77a886..a993d2bf 100644 --- a/docs/parsers/hosts.md +++ b/docs/parsers/hosts.md @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('hosts', hosts_file_output) - or - - import jc.parsers.hosts - result = jc.parsers.hosts.parse(hosts_file_output) - Schema: [ diff --git a/docs/parsers/id.md b/docs/parsers/id.md index b37a97d7..df76f94a 100644 --- a/docs/parsers/id.md +++ b/docs/parsers/id.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('id', id_command_output) - or - - import jc.parsers.id - result = jc.parsers.id.parse(id_command_output) - Schema: { diff --git a/docs/parsers/ifconfig.md b/docs/parsers/ifconfig.md index de99c8db..0fba185c 100644 --- a/docs/parsers/ifconfig.md +++ b/docs/parsers/ifconfig.md @@ -20,11 +20,6 @@ Usage (module): import jc result = jc.parse('ifconfig', ifconfig_command_output) - or - - import jc.parsers.ifconfig - result = jc.parsers.ifconfig.parse(ifconfig_command_output) - Schema: [ diff --git a/docs/parsers/ini.md b/docs/parsers/ini.md index 5e2dc174..cb5b555b 100644 --- a/docs/parsers/ini.md +++ b/docs/parsers/ini.md @@ -22,11 +22,6 @@ Usage (module): import jc result = jc.parse('ini', ini_file_output) - or - - import jc.parsers.ini - result = jc.parsers.ini.parse(ini_file_output) - Schema: ini or key/value document converted to a dictionary - see the diff --git a/docs/parsers/iostat.md b/docs/parsers/iostat.md index b8e5fae5..fcd4df50 100644 --- a/docs/parsers/iostat.md +++ b/docs/parsers/iostat.md @@ -20,11 +20,6 @@ Usage (module): import jc result = jc.parse('iostat', iostat_command_output) - or - - import jc.parsers.iostat - result = jc.parsers.iostat.parse(iostat_command_output) - Schema: [ diff --git a/docs/parsers/iptables.md b/docs/parsers/iptables.md index c51d6d13..1bbcbb36 100644 --- a/docs/parsers/iptables.md +++ b/docs/parsers/iptables.md @@ -20,11 +20,6 @@ Usage (module): import jc result = jc.parse('iptables', iptables_command_output) - or - - import jc.parsers.iptables - result = jc.parsers.iptables.parse(iptables_command_output) - Schema: [ diff --git a/docs/parsers/iw_scan.md b/docs/parsers/iw_scan.md index b39de09e..48d566af 100644 --- a/docs/parsers/iw_scan.md +++ b/docs/parsers/iw_scan.md @@ -21,11 +21,6 @@ Usage (module): import jc result = jc.parse('iw_scan', iw_scan_command_output) - or - - import jc.parsers.iw_scan - result = jc.parsers.iw_scan.parse(iw_scan_command_output) - Schema: [ diff --git a/docs/parsers/jar_manifest.md b/docs/parsers/jar_manifest.md index 91964220..45170037 100644 --- a/docs/parsers/jar_manifest.md +++ b/docs/parsers/jar_manifest.md @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('jar_manifest', jar_manifest_file_output) - or - - import jc.parsers.jar_manifest - result = jc.parsers.jar_manifest.parse(jar_manifest_file_output) - Schema: [ diff --git a/docs/parsers/jobs.md b/docs/parsers/jobs.md index ab7da0dd..a1c20f1c 100644 --- a/docs/parsers/jobs.md +++ b/docs/parsers/jobs.md @@ -19,11 +19,6 @@ Usage (module): import jc result = jc.parse('jobs', jobs_command_output) - or - - import jc.parsers.jobs - result = jc.parsers.jobs.parse(jobs_command_output) - Schema: [ diff --git a/docs/parsers/kv.md b/docs/parsers/kv.md index 9959671a..dd03c5e9 100644 --- a/docs/parsers/kv.md +++ b/docs/parsers/kv.md @@ -22,11 +22,6 @@ Usage (module): import jc result = jc.parse('kv', kv_file_output) - or - - import jc.parsers.kv - result = jc.parsers.kv.parse(kv_file_output) - Schema: key/value document converted to a dictionary - see the diff --git a/docs/parsers/last.md b/docs/parsers/last.md index 910e405f..012946d9 100644 --- a/docs/parsers/last.md +++ b/docs/parsers/last.md @@ -24,11 +24,6 @@ Usage (module): import jc result = jc.parse('last', last_command_output) - or - - import jc.parsers.last - result = jc.parsers.last.parse(last_command_output) - Schema: [ diff --git a/docs/parsers/ls.md b/docs/parsers/ls.md index 578b3dfb..8c913534 100644 --- a/docs/parsers/ls.md +++ b/docs/parsers/ls.md @@ -34,11 +34,6 @@ Usage (module): import jc result = jc.parse('ls', ls_command_output) - or - - import jc.parsers.ls - result = jc.parsers.ls.parse(ls_command_output) - Schema: [ diff --git a/docs/parsers/lsblk.md b/docs/parsers/lsblk.md index 0e8ba14e..edf03ae3 100644 --- a/docs/parsers/lsblk.md +++ b/docs/parsers/lsblk.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('lsblk', lsblk_command_output) - or - - import jc.parsers.lsblk - result = jc.parsers.lsblk.parse(lsblk_command_output) - Schema: [ diff --git a/docs/parsers/lsmod.md b/docs/parsers/lsmod.md index 705f45ff..ec17b876 100644 --- a/docs/parsers/lsmod.md +++ b/docs/parsers/lsmod.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('lsmod', lsmod_command_output) - or - - import jc.parsers.lsmod - result = jc.parsers.lsmod.parse(lsmod_command_output) - Schema: [ diff --git a/docs/parsers/lsof.md b/docs/parsers/lsof.md index bff7b40c..732f9e90 100644 --- a/docs/parsers/lsof.md +++ b/docs/parsers/lsof.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('lsof', lsof_command_output) - or - - import jc.parsers.lsof - result = jc.parsers.lsof.parse(lsof_command_output) - Schema: [ diff --git a/docs/parsers/lsusb.md b/docs/parsers/lsusb.md index fa27c836..04f40f96 100644 --- a/docs/parsers/lsusb.md +++ b/docs/parsers/lsusb.md @@ -20,11 +20,6 @@ Usage (module): import jc result = jc.parse('lsusb', lsusb_command_output) - or - - import jc.parsers.lsusb - result = jc.parsers.lsusb.parse(lsusb_command_output) - Schema: Note: object keynames are assigned directly from the lsusb diff --git a/docs/parsers/mount.md b/docs/parsers/mount.md index e5bbea98..39dcc1e7 100644 --- a/docs/parsers/mount.md +++ b/docs/parsers/mount.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('mount', mount_command_output) - or - - import jc.parsers.mount - result = jc.parsers.mount.parse(mount_command_output) - Schema: [ diff --git a/docs/parsers/netstat.md b/docs/parsers/netstat.md index 01d0782b..0d67133f 100644 --- a/docs/parsers/netstat.md +++ b/docs/parsers/netstat.md @@ -23,11 +23,6 @@ Usage (module): import jc result = jc.parse('netstat', netstat_command_output) - or - - import jc.parsers.netstat - result = jc.parsers.netstat.parse(netstat_command_output) - Schema: [ diff --git a/docs/parsers/nmcli.md b/docs/parsers/nmcli.md index 3df6a3de..6c4b3793 100644 --- a/docs/parsers/nmcli.md +++ b/docs/parsers/nmcli.md @@ -27,11 +27,6 @@ Usage (module): import jc result = jc.parse('nmcli', nmcli_command_output) - or - - import jc.parsers.nmcli - result = jc.parsers.nmcli.parse(nmcli_command_output) - Schema: Because there are so many options, the schema is not strictly defined. diff --git a/docs/parsers/ntpq.md b/docs/parsers/ntpq.md index ff043328..fd5b1fce 100644 --- a/docs/parsers/ntpq.md +++ b/docs/parsers/ntpq.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('ntpq', ntpq_command_output) - or - - import jc.parsers.ntpq - result = jc.parsers.ntpq.parse(ntpq_command_output) - Schema: [ diff --git a/docs/parsers/passwd.md b/docs/parsers/passwd.md index 498c1cfb..e7e820f4 100644 --- a/docs/parsers/passwd.md +++ b/docs/parsers/passwd.md @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('passwd', passwd_file_output) - or - - import jc.parsers.passwd - result = jc.parsers.passwd.parse(passwd_file_output) - Schema: [ diff --git a/docs/parsers/pidstat.md b/docs/parsers/pidstat.md index 8921b870..f8d1937e 100644 --- a/docs/parsers/pidstat.md +++ b/docs/parsers/pidstat.md @@ -21,11 +21,6 @@ Usage (module): import jc result = jc.parse('pidstat', pidstat_command_output) - or - - import jc.parsers.pidstat - result = jc.parsers.pidstat.parse(pidstat_command_output) - Schema: [ diff --git a/docs/parsers/ping.md b/docs/parsers/ping.md index a5e1f2cd..74eae286 100644 --- a/docs/parsers/ping.md +++ b/docs/parsers/ping.md @@ -23,11 +23,6 @@ Usage (module): import jc result = jc.parse('ping', ping_command_output) - or - - import jc.parsers.ping - result = jc.parsers.ping.parse(ping_command_output) - Schema: { diff --git a/docs/parsers/pip_list.md b/docs/parsers/pip_list.md index 33906cdf..eb395240 100644 --- a/docs/parsers/pip_list.md +++ b/docs/parsers/pip_list.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('pip_list', pip_list_command_output) - or - - import jc.parsers.pip_list - result = jc.parsers.pip_list.parse(pip_list_command_output) - Schema: [ diff --git a/docs/parsers/pip_show.md b/docs/parsers/pip_show.md index 661bc526..a43394cb 100644 --- a/docs/parsers/pip_show.md +++ b/docs/parsers/pip_show.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('pip_show', pip_show_command_output) - or - - import jc.parsers.pip_show - result = jc.parsers.pip_show.parse(pip_show_command_output) - Schema: [ diff --git a/docs/parsers/ps.md b/docs/parsers/ps.md index ef02011a..6c64942c 100644 --- a/docs/parsers/ps.md +++ b/docs/parsers/ps.md @@ -22,11 +22,6 @@ Usage (module): import jc result = jc.parse('ps', ps_command_output) - or - - import jc.parsers.ps - result = jc.parsers.ps.parse(ps_command_output) - Schema: [ diff --git a/docs/parsers/route.md b/docs/parsers/route.md index 022676a7..0012fd84 100644 --- a/docs/parsers/route.md +++ b/docs/parsers/route.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('route', route_command_output) - or - - import jc.parsers.route - result = jc.parsers.route.parse(route_command_output) - Schema: [ diff --git a/docs/parsers/rpm_qi.md b/docs/parsers/rpm_qi.md index b0238a17..b398edfa 100644 --- a/docs/parsers/rpm_qi.md +++ b/docs/parsers/rpm_qi.md @@ -26,11 +26,6 @@ Usage (module): import jc result = jc.parse('rpm_qi', rpm_qi_command_output) - or - - import jc.parsers.rpm_qi - result = jc.parsers.rpm_qi.parse(rpm_qi_command_output) - Schema: [ diff --git a/docs/parsers/rsync.md b/docs/parsers/rsync.md index dd799fdf..30828667 100644 --- a/docs/parsers/rsync.md +++ b/docs/parsers/rsync.md @@ -26,11 +26,6 @@ Usage (module): import jc result = jc.parse('rsync', rsync_command_output) - or - - import jc.parsers.rsync - result = jc.parsers.rsync.parse(rsync_command_output) - Schema: [ diff --git a/docs/parsers/sfdisk.md b/docs/parsers/sfdisk.md index 4358b074..15e3f0a9 100644 --- a/docs/parsers/sfdisk.md +++ b/docs/parsers/sfdisk.md @@ -27,11 +27,6 @@ Usage (module): import jc result = jc.parse('sfdisk', sfdisk_command_output) - or - - import jc.parsers.sfdisk - result = jc.parsers.sfdisk.parse(sfdisk_command_output) - Schema: [ diff --git a/docs/parsers/shadow.md b/docs/parsers/shadow.md index ef615c26..224a3f9f 100644 --- a/docs/parsers/shadow.md +++ b/docs/parsers/shadow.md @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('shadow', shadow_file_output) - or - - import jc.parsers.shadow - result = jc.parsers.shadow.parse(shadow_file_output) - Schema: [ diff --git a/docs/parsers/ss.md b/docs/parsers/ss.md index bd0ed975..22257d6d 100644 --- a/docs/parsers/ss.md +++ b/docs/parsers/ss.md @@ -21,11 +21,6 @@ Usage (module): import jc result = jc.parse('ss', ss_command_output) - or - - import jc.parsers.ss - result = jc.parsers.ss.parse(ss_command_output) - Schema: Information from https://www.cyberciti.biz/files/ss.html used to define diff --git a/docs/parsers/stat.md b/docs/parsers/stat.md index 9240b290..f221b7d0 100644 --- a/docs/parsers/stat.md +++ b/docs/parsers/stat.md @@ -24,11 +24,6 @@ Usage (module): import jc result = jc.parse('stat', stat_command_output) - or - - import jc.parsers.stat - result = jc.parsers.stat.parse(stat_command_output) - Schema: [ diff --git a/docs/parsers/sysctl.md b/docs/parsers/sysctl.md index e0ac3892..d72754ab 100644 --- a/docs/parsers/sysctl.md +++ b/docs/parsers/sysctl.md @@ -23,11 +23,6 @@ Usage (module): import jc result = jc.parse('sysctl', sysctl_command_output) - or - - import jc.parsers.sysctl - result = jc.parsers.sysctl.parse(sysctl_command_output) - Schema: { diff --git a/docs/parsers/systemctl.md b/docs/parsers/systemctl.md index e6d03231..b9e2eb4d 100644 --- a/docs/parsers/systemctl.md +++ b/docs/parsers/systemctl.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('systemctl', systemctl_command_output) - or - - import jc.parsers.systemctl - result = jc.parsers.systemctl.parse(systemctl_command_output) - Schema: [ diff --git a/docs/parsers/systemctl_lj.md b/docs/parsers/systemctl_lj.md index 68bfed5d..0423c083 100644 --- a/docs/parsers/systemctl_lj.md +++ b/docs/parsers/systemctl_lj.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('systemctl_lj', systemctl_lj_command_output) - or - - import jc.parsers.systemctl_lj - result = jc.parsers.systemctl_lj.parse(systemctl_lj_command_output) - Schema: [ diff --git a/docs/parsers/systemctl_ls.md b/docs/parsers/systemctl_ls.md index cd13a9c0..8de6787e 100644 --- a/docs/parsers/systemctl_ls.md +++ b/docs/parsers/systemctl_ls.md @@ -19,11 +19,6 @@ Usage (module): import jc result = jc.parse('systemctl_ls', systemctl_ls_command_output) - or - - import jc.parsers.systemctl_ls - result = jc.parsers.systemctl_ls.parse(systemctl_ls_command_output) - Schema: [ diff --git a/docs/parsers/systemctl_luf.md b/docs/parsers/systemctl_luf.md index ab3d2983..c7505516 100644 --- a/docs/parsers/systemctl_luf.md +++ b/docs/parsers/systemctl_luf.md @@ -19,11 +19,6 @@ Usage (module): import jc result = jc.parse('systemctl_luf', systemctl_luf_command_output) - or - - import jc.parsers.systemctl_luf - result = jc.parsers.systemctl_luf.parse(systemctl_luf_command_output) - Schema: [ diff --git a/docs/parsers/systeminfo.md b/docs/parsers/systeminfo.md index 0b442e5f..9fd3e71d 100644 --- a/docs/parsers/systeminfo.md +++ b/docs/parsers/systeminfo.md @@ -24,11 +24,6 @@ Usage (module): import jc result = jc.parse('systeminfo', systeminfo_command_output) - or - - import jc.parsers.systeminfo - result = jc.parsers.systeminfo.parse(systeminfo_command_output) - Schema: { diff --git a/docs/parsers/time.md b/docs/parsers/time.md index bdf3f1a4..6ad20f07 100644 --- a/docs/parsers/time.md +++ b/docs/parsers/time.md @@ -24,11 +24,6 @@ Usage (module): import jc result = jc.parse('time', time_command_output) - or - - import jc.parsers.time - result = jc.parsers.time.parse(time_command_output) - Schema: Source: https://www.freebsd.org/cgi/man.cgi?query=getrusage diff --git a/docs/parsers/timedatectl.md b/docs/parsers/timedatectl.md index ddcd4c73..679483c7 100644 --- a/docs/parsers/timedatectl.md +++ b/docs/parsers/timedatectl.md @@ -21,11 +21,6 @@ Usage (module): import jc result = jc.parse('timedatectl', timedatectl_command_output) - or - - import jc.parsers.timedatectl - result = jc.parsers.timedatectl.parse(timedatectl_command_output) - Schema: { diff --git a/docs/parsers/tracepath.md b/docs/parsers/tracepath.md index 9a8256c5..3b81a903 100644 --- a/docs/parsers/tracepath.md +++ b/docs/parsers/tracepath.md @@ -20,11 +20,6 @@ Usage (module): import jc result = jc.parse('tracepath', tracepath_command_output) - or - - import jc.parsers.tracepath - result = jc.parsers.tracepath.parse(tracepath_command_output) - Schema: { diff --git a/docs/parsers/traceroute.md b/docs/parsers/traceroute.md index f98e5ba3..3bf314c8 100644 --- a/docs/parsers/traceroute.md +++ b/docs/parsers/traceroute.md @@ -27,11 +27,6 @@ Usage (module): import jc result = jc.parse('traceroute', traceroute_command_output) - or - - import jc.parsers.traceroute - result = jc.parsers.traceroute.parse(traceroute_command_output) - Schema: { diff --git a/docs/parsers/ufw.md b/docs/parsers/ufw.md index 86e625ee..df84d73d 100644 --- a/docs/parsers/ufw.md +++ b/docs/parsers/ufw.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('ufw', ufw_command_output) - or - - import jc.parsers.ufw - result = jc.parsers.ufw.parse(ufw_command_output) - Schema: { diff --git a/docs/parsers/ufw_appinfo.md b/docs/parsers/ufw_appinfo.md index cb15ac4d..d966f570 100644 --- a/docs/parsers/ufw_appinfo.md +++ b/docs/parsers/ufw_appinfo.md @@ -26,11 +26,6 @@ Usage (module): import jc result = jc.parse('ufw_appinfo', ufw_appinfo_command_output) - or - - import jc.parsers.ufw_appinfo - result = jc.parsers.ufw_appinfo.parse(ufw_appinfo_command_output) - Schema: [ diff --git a/docs/parsers/uname.md b/docs/parsers/uname.md index d583146c..fba8c10a 100644 --- a/docs/parsers/uname.md +++ b/docs/parsers/uname.md @@ -20,11 +20,6 @@ Usage (module): import jc result = jc.parse('uname', uname_command_output) - or - - import jc.parsers.uname - result = jc.parsers.uname.parse(uname_command_output) - Schema: { diff --git a/docs/parsers/universal.md b/docs/parsers/universal.md index 5327ed83..934249c1 100644 --- a/docs/parsers/universal.md +++ b/docs/parsers/universal.md @@ -18,7 +18,15 @@ jc - JSON Convert universal parsers def simple_table_parse(data: List[str]) -> List[Dict] ``` -Parse simple tables. The last column may contain data with spaces. +Parse simple tables. There should be no blank cells. The last column +may contain data with spaces. + +Example Table: + + col1 col2 col3 col4 col5 + apple orange pear banana my favorite fruits + carrot squash celery spinach my favorite veggies + chicken beef pork eggs my favorite proteins Parameters: @@ -44,6 +52,14 @@ def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict] ``` Parse tables with missing column data or with spaces in column data. +Data elements must line up within column boundaries. + +Example Table: + + col1 col2 col3 col4 col5 + orange pear fuzzy peach my favorite fruits + green beans celery spinach my favorite veggies + chicken beef pork brown eggs my favorite proteins Parameters: diff --git a/docs/parsers/upower.md b/docs/parsers/upower.md index f8d513a4..ee3ebd91 100644 --- a/docs/parsers/upower.md +++ b/docs/parsers/upower.md @@ -24,11 +24,6 @@ Usage (module): import jc result = jc.parse('upower', upower_command_output) - or - - import jc.parsers.upower - result = jc.parsers.upower.parse(upower_command_output) - Schema: [ diff --git a/docs/parsers/uptime.md b/docs/parsers/uptime.md index d2c5ccea..a4640801 100644 --- a/docs/parsers/uptime.md +++ b/docs/parsers/uptime.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('uptime', uptime_command_output) - or - - import jc.parsers.uptime - result = jc.parsers.uptime.parse(uptime_command_output) - Schema: { diff --git a/docs/parsers/vmstat.md b/docs/parsers/vmstat.md index fea78a6a..29b520ad 100644 --- a/docs/parsers/vmstat.md +++ b/docs/parsers/vmstat.md @@ -26,11 +26,6 @@ Usage (module): import jc result = jc.parse('vmstat', vmstat_command_output) - or - - import jc.parsers.vmstat - result = jc.parsers.vmstat.parse(vmstat_command_output) - Schema: [ diff --git a/docs/parsers/w.md b/docs/parsers/w.md index 6ff07141..f15e665f 100644 --- a/docs/parsers/w.md +++ b/docs/parsers/w.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('w', w_command_output) - or - - import jc.parsers.w - result = jc.parsers.w.parse(w_command_output) - Schema: [ diff --git a/docs/parsers/wc.md b/docs/parsers/wc.md index 2969bf12..79e31164 100644 --- a/docs/parsers/wc.md +++ b/docs/parsers/wc.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('wc', wc_command_output) - or - - import jc.parsers.wc - result = jc.parsers.wc.parse(wc_command_output) - Schema: [ diff --git a/docs/parsers/who.md b/docs/parsers/who.md index 7d7eed62..157889bb 100644 --- a/docs/parsers/who.md +++ b/docs/parsers/who.md @@ -23,11 +23,6 @@ Usage (module): import jc result = jc.parse('who', who_command_output) - or - - import jc.parsers.who - result = jc.parsers.who.parse(who_command_output) - Schema: [ diff --git a/docs/parsers/xml.md b/docs/parsers/xml.md index 6e822260..e864eaa6 100644 --- a/docs/parsers/xml.md +++ b/docs/parsers/xml.md @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('xml', xml_file_output) - or - - import jc.parsers.xml - result = jc.parsers.xml.parse(xml_file_output) - Schema: XML Document converted to a Dictionary diff --git a/docs/parsers/xrandr.md b/docs/parsers/xrandr.md index 76d926b7..d5db0551 100644 --- a/docs/parsers/xrandr.md +++ b/docs/parsers/xrandr.md @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('xrandr', xrandr_command_output) - or - - import jc.parsers.xrandr - result = jc.parsers.xrandr.parse(xrandr_command_output) - Schema: { diff --git a/docs/parsers/yaml.md b/docs/parsers/yaml.md index 6e774c7c..bd4c6c70 100644 --- a/docs/parsers/yaml.md +++ b/docs/parsers/yaml.md @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('yaml', yaml_file_output) - or - - import jc.parsers.yaml - result = jc.parsers.yaml.parse(yaml_file_output) - Schema: YAML Document converted to a Dictionary diff --git a/docs/parsers/zipinfo.md b/docs/parsers/zipinfo.md index e2ca96f7..921caf49 100644 --- a/docs/parsers/zipinfo.md +++ b/docs/parsers/zipinfo.md @@ -23,11 +23,6 @@ Usage (module): import jc result = jc.parse('zipinfo', zipinfo_command_output) - or - - import jc.parsers.zipinfo - result = jc.parsers.zipinfo.parse(zipinfo_command_output) - Schema: [ diff --git a/jc/parsers/acpi.py b/jc/parsers/acpi.py index 46182eed..32769e57 100644 --- a/jc/parsers/acpi.py +++ b/jc/parsers/acpi.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('acpi', acpi_command_output) - or - - import jc.parsers.acpi - result = jc.parsers.acpi.parse(acpi_command_output) - Schema: [ diff --git a/jc/parsers/airport.py b/jc/parsers/airport.py index ae1d4291..045802b8 100644 --- a/jc/parsers/airport.py +++ b/jc/parsers/airport.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('airport', airport_command_output) - or - - import jc.parsers.airport - result = jc.parsers.airport.parse(airport_command_output) - Schema: { diff --git a/jc/parsers/airport_s.py b/jc/parsers/airport_s.py index bf478efa..b635acaa 100644 --- a/jc/parsers/airport_s.py +++ b/jc/parsers/airport_s.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('airport_s', airport_s_command_output) - or - - import jc.parsers.airport_s - result = jc.parsers.airport_s.parse(airport_s_command_output) - Schema: [ diff --git a/jc/parsers/arp.py b/jc/parsers/arp.py index 26203490..f0bc43a3 100644 --- a/jc/parsers/arp.py +++ b/jc/parsers/arp.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('arp', arp_command_output) - or - - import jc.parsers.arp - result = jc.parsers.arp.parse(arp_command_output) - Schema: [ diff --git a/jc/parsers/blkid.py b/jc/parsers/blkid.py index 2e94b63f..808b903e 100644 --- a/jc/parsers/blkid.py +++ b/jc/parsers/blkid.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('blkid', blkid_command_output) - or - - import jc.parsers.blkid - result = jc.parsers.blkid.parse(blkid_command_output) - Schema: [ diff --git a/jc/parsers/cksum.py b/jc/parsers/cksum.py index ea50103c..f0fd52ed 100644 --- a/jc/parsers/cksum.py +++ b/jc/parsers/cksum.py @@ -17,11 +17,6 @@ Usage (module): import jc result = jc.parse('cksum', cksum_command_output) - or - - import jc.parsers.cksum - result = jc.parsers.cksum.parse(cksum_command_output) - Schema: [ diff --git a/jc/parsers/crontab.py b/jc/parsers/crontab.py index 59c0fe81..314fa87c 100644 --- a/jc/parsers/crontab.py +++ b/jc/parsers/crontab.py @@ -16,11 +16,6 @@ Usage (module): import jc result = jc.parse('crontab', crontab_output) - or - - import jc.parsers.crontab - result = jc.parsers.crontab.parse(crontab_output) - Schema: { diff --git a/jc/parsers/crontab_u.py b/jc/parsers/crontab_u.py index 8ec1f13f..aa4f77be 100644 --- a/jc/parsers/crontab_u.py +++ b/jc/parsers/crontab_u.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('crontab_u', crontab_u_output) - or - - import jc.parsers.crontab_u - result = jc.parsers.crontab_u.parse(crontab_u_output) - Schema: { diff --git a/jc/parsers/csv.py b/jc/parsers/csv.py index 0df902ec..34b63437 100644 --- a/jc/parsers/csv.py +++ b/jc/parsers/csv.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('csv', csv_output) - or - - import jc.parsers.csv - result = jc.parsers.csv.parse(csv_output) - Schema: csv file converted to a Dictionary: diff --git a/jc/parsers/date.py b/jc/parsers/date.py index b24c510a..61180f57 100644 --- a/jc/parsers/date.py +++ b/jc/parsers/date.py @@ -19,11 +19,6 @@ Usage (module): import jc result = jc.parse('date', date_command_output) - or - - import jc.parsers.date - result = jc.parsers.date.parse(date_command_output) - Schema: { diff --git a/jc/parsers/df.py b/jc/parsers/df.py index cb1edb3e..a6b1fe55 100644 --- a/jc/parsers/df.py +++ b/jc/parsers/df.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('df', df_command_output) - or - - import jc.parsers.df - result = jc.parsers.df.parse(df_command_output) - Schema: [ diff --git a/jc/parsers/dig.py b/jc/parsers/dig.py index d5757b09..7119889f 100644 --- a/jc/parsers/dig.py +++ b/jc/parsers/dig.py @@ -24,11 +24,6 @@ Usage (module): import jc result = jc.parse('dig', dig_command_output) - or - - import jc.parsers.dig - result = jc.parsers.dig.parse(dig_command_output) - Schema: [ diff --git a/jc/parsers/dir.py b/jc/parsers/dir.py index a7665628..92fe9298 100644 --- a/jc/parsers/dir.py +++ b/jc/parsers/dir.py @@ -21,11 +21,6 @@ Usage (module): import jc result = jc.parse('dir', dir_command_output) - or - - import jc.parsers.dir - result = jc.parsers.dir.parse(dir_command_output) - Schema: [ diff --git a/jc/parsers/dmidecode.py b/jc/parsers/dmidecode.py index 0ec59c03..23b7f392 100644 --- a/jc/parsers/dmidecode.py +++ b/jc/parsers/dmidecode.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('dmidecode', dmidecode_command_output) - or - - import jc.parsers.dmidecode - result = jc.parsers.dmidecode.parse(dmidecode_command_output) - Schema: [ diff --git a/jc/parsers/dpkg_l.py b/jc/parsers/dpkg_l.py index 46f192d0..4e8e0e4c 100644 --- a/jc/parsers/dpkg_l.py +++ b/jc/parsers/dpkg_l.py @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('dpkg_l', dpkg_command_output) - or - - import jc.parsers.dpkg_l - result = jc.parsers.dpkg_l.parse(dpkg_command_output) - Schema: [ diff --git a/jc/parsers/du.py b/jc/parsers/du.py index c2fa2f0d..60403345 100644 --- a/jc/parsers/du.py +++ b/jc/parsers/du.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('du', du_command_output) - or - - import jc.parsers.du - result = jc.parsers.du.parse(du_command_output) - Schema: [ diff --git a/jc/parsers/env.py b/jc/parsers/env.py index 50cfaed4..2dae8164 100644 --- a/jc/parsers/env.py +++ b/jc/parsers/env.py @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('env', env_command_output) - or - - import jc.parsers.env - result = jc.parsers.env.parse(env_command_output) - Schema: [ diff --git a/jc/parsers/file.py b/jc/parsers/file.py index 1fbb3c67..4a8adc0d 100644 --- a/jc/parsers/file.py +++ b/jc/parsers/file.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('file', file_command_output) - or - - import jc.parsers.file - result = jc.parsers.file.parse(file_command_output) - Schema: [ diff --git a/jc/parsers/finger.py b/jc/parsers/finger.py index 5da95946..69c95c06 100644 --- a/jc/parsers/finger.py +++ b/jc/parsers/finger.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('finger', finger_command_output) - or - - import jc.parsers.finger - result = jc.parsers.finger.parse(finger_command_output) - Schema: [ diff --git a/jc/parsers/foo.py b/jc/parsers/foo.py index c3a19357..02677549 100644 --- a/jc/parsers/foo.py +++ b/jc/parsers/foo.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('foo', foo_command_output) - or - - import jc.parsers.foo - result = jc.parsers.foo.parse(foo_command_output) - Schema: [ diff --git a/jc/parsers/free.py b/jc/parsers/free.py index 21a4dc45..c87d294e 100644 --- a/jc/parsers/free.py +++ b/jc/parsers/free.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('free', free_command_output) - or - - import jc.parsers.free - result = jc.parsers.free.parse(free_command_output) - Schema: [ diff --git a/jc/parsers/fstab.py b/jc/parsers/fstab.py index 3608253b..3d136de2 100644 --- a/jc/parsers/fstab.py +++ b/jc/parsers/fstab.py @@ -9,11 +9,6 @@ Usage (module): import jc result = jc.parse('fstab', fstab_command_output) - or - - import jc.parsers.fstab - result = jc.parsers.fstab.parse(fstab_command_output) - Schema: [ diff --git a/jc/parsers/group.py b/jc/parsers/group.py index 7818ada2..72a433ef 100644 --- a/jc/parsers/group.py +++ b/jc/parsers/group.py @@ -9,11 +9,6 @@ Usage (module): import jc result = jc.parse('group', group_file_output) - or - - import jc.parsers.group - result = jc.parsers.group.parse(group_file_output) - Schema: [ diff --git a/jc/parsers/gshadow.py b/jc/parsers/gshadow.py index 581c1e26..34422498 100644 --- a/jc/parsers/gshadow.py +++ b/jc/parsers/gshadow.py @@ -9,11 +9,6 @@ Usage (module): import jc result = jc.parse('gshadow', gshadow_file_output) - or - - import jc.parsers.gshadow - result = jc.parsers.gshadow.parse(gshadow_file_output) - Schema: [ diff --git a/jc/parsers/hash.py b/jc/parsers/hash.py index b5f87c4c..2c25ba81 100644 --- a/jc/parsers/hash.py +++ b/jc/parsers/hash.py @@ -9,11 +9,6 @@ Usage (module): import jc result = jc.parse('hash', hash_command_output) - or - - import jc.parsers.hash - result = jc.parsers.hash.parse(hash_command_output) - Schema: [ diff --git a/jc/parsers/hashsum.py b/jc/parsers/hashsum.py index 0d963ca7..34bb43fe 100644 --- a/jc/parsers/hashsum.py +++ b/jc/parsers/hashsum.py @@ -23,11 +23,6 @@ Usage (module): import jc result = jc.parse('hashsum', md5sum_command_output) - or - - import jc.parsers.hashsum - result = jc.parsers.hashsum.parse(md5sum_command_output) - Schema: [ diff --git a/jc/parsers/hciconfig.py b/jc/parsers/hciconfig.py index 5e9551bb..88d84d32 100644 --- a/jc/parsers/hciconfig.py +++ b/jc/parsers/hciconfig.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('hciconfig', hciconfig_command_output) - or - - import jc.parsers.hciconfig - result = jc.parsers.hciconfig.parse(hciconfig_command_output) - Schema: [ diff --git a/jc/parsers/history.py b/jc/parsers/history.py index 9ee422ff..adcc7aa9 100644 --- a/jc/parsers/history.py +++ b/jc/parsers/history.py @@ -17,11 +17,6 @@ Usage (module): import jc result = jc.parse('history', history_command_output) - or - - import jc.parsers.history - result = jc.parsers.history.parse(history_command_output) - Schema: [ diff --git a/jc/parsers/hosts.py b/jc/parsers/hosts.py index e6f8d679..6164f61c 100644 --- a/jc/parsers/hosts.py +++ b/jc/parsers/hosts.py @@ -9,11 +9,6 @@ Usage (module): import jc result = jc.parse('hosts', hosts_file_output) - or - - import jc.parsers.hosts - result = jc.parsers.hosts.parse(hosts_file_output) - Schema: [ diff --git a/jc/parsers/id.py b/jc/parsers/id.py index 046a28f9..82a06e6c 100644 --- a/jc/parsers/id.py +++ b/jc/parsers/id.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('id', id_command_output) - or - - import jc.parsers.id - result = jc.parsers.id.parse(id_command_output) - Schema: { diff --git a/jc/parsers/ifconfig.py b/jc/parsers/ifconfig.py index 3c6abc88..7a125a18 100644 --- a/jc/parsers/ifconfig.py +++ b/jc/parsers/ifconfig.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('ifconfig', ifconfig_command_output) - or - - import jc.parsers.ifconfig - result = jc.parsers.ifconfig.parse(ifconfig_command_output) - Schema: [ diff --git a/jc/parsers/ini.py b/jc/parsers/ini.py index 192e3292..b476937e 100644 --- a/jc/parsers/ini.py +++ b/jc/parsers/ini.py @@ -17,11 +17,6 @@ Usage (module): import jc result = jc.parse('ini', ini_file_output) - or - - import jc.parsers.ini - result = jc.parsers.ini.parse(ini_file_output) - Schema: ini or key/value document converted to a dictionary - see the diff --git a/jc/parsers/iostat.py b/jc/parsers/iostat.py index 59a43a6d..4eeb2add 100644 --- a/jc/parsers/iostat.py +++ b/jc/parsers/iostat.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('iostat', iostat_command_output) - or - - import jc.parsers.iostat - result = jc.parsers.iostat.parse(iostat_command_output) - Schema: [ diff --git a/jc/parsers/iptables.py b/jc/parsers/iptables.py index b7de3886..8ec2ca9e 100644 --- a/jc/parsers/iptables.py +++ b/jc/parsers/iptables.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('iptables', iptables_command_output) - or - - import jc.parsers.iptables - result = jc.parsers.iptables.parse(iptables_command_output) - Schema: [ diff --git a/jc/parsers/iw_scan.py b/jc/parsers/iw_scan.py index 883c3c76..eea984e8 100644 --- a/jc/parsers/iw_scan.py +++ b/jc/parsers/iw_scan.py @@ -16,11 +16,6 @@ Usage (module): import jc result = jc.parse('iw_scan', iw_scan_command_output) - or - - import jc.parsers.iw_scan - result = jc.parsers.iw_scan.parse(iw_scan_command_output) - Schema: [ diff --git a/jc/parsers/jar_manifest.py b/jc/parsers/jar_manifest.py index 3a68b9cc..0b7a320e 100644 --- a/jc/parsers/jar_manifest.py +++ b/jc/parsers/jar_manifest.py @@ -9,11 +9,6 @@ Usage (module): import jc result = jc.parse('jar_manifest', jar_manifest_file_output) - or - - import jc.parsers.jar_manifest - result = jc.parsers.jar_manifest.parse(jar_manifest_file_output) - Schema: [ diff --git a/jc/parsers/jobs.py b/jc/parsers/jobs.py index b10ae7ae..7e39e7fb 100644 --- a/jc/parsers/jobs.py +++ b/jc/parsers/jobs.py @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('jobs', jobs_command_output) - or - - import jc.parsers.jobs - result = jc.parsers.jobs.parse(jobs_command_output) - Schema: [ diff --git a/jc/parsers/kv.py b/jc/parsers/kv.py index db2d1544..2d5feee5 100644 --- a/jc/parsers/kv.py +++ b/jc/parsers/kv.py @@ -17,11 +17,6 @@ Usage (module): import jc result = jc.parse('kv', kv_file_output) - or - - import jc.parsers.kv - result = jc.parsers.kv.parse(kv_file_output) - Schema: key/value document converted to a dictionary - see the diff --git a/jc/parsers/last.py b/jc/parsers/last.py index 2598e37c..74948a36 100644 --- a/jc/parsers/last.py +++ b/jc/parsers/last.py @@ -19,11 +19,6 @@ Usage (module): import jc result = jc.parse('last', last_command_output) - or - - import jc.parsers.last - result = jc.parsers.last.parse(last_command_output) - Schema: [ diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index 226060de..981f8d5b 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -29,11 +29,6 @@ Usage (module): import jc result = jc.parse('ls', ls_command_output) - or - - import jc.parsers.ls - result = jc.parsers.ls.parse(ls_command_output) - Schema: [ diff --git a/jc/parsers/lsblk.py b/jc/parsers/lsblk.py index 4ee44d69..64326f92 100644 --- a/jc/parsers/lsblk.py +++ b/jc/parsers/lsblk.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('lsblk', lsblk_command_output) - or - - import jc.parsers.lsblk - result = jc.parsers.lsblk.parse(lsblk_command_output) - Schema: [ diff --git a/jc/parsers/lsmod.py b/jc/parsers/lsmod.py index 1342d5cd..524d6960 100644 --- a/jc/parsers/lsmod.py +++ b/jc/parsers/lsmod.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('lsmod', lsmod_command_output) - or - - import jc.parsers.lsmod - result = jc.parsers.lsmod.parse(lsmod_command_output) - Schema: [ diff --git a/jc/parsers/lsof.py b/jc/parsers/lsof.py index 08804dd0..e0865d99 100644 --- a/jc/parsers/lsof.py +++ b/jc/parsers/lsof.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('lsof', lsof_command_output) - or - - import jc.parsers.lsof - result = jc.parsers.lsof.parse(lsof_command_output) - Schema: [ diff --git a/jc/parsers/lsusb.py b/jc/parsers/lsusb.py index 4650d60d..397f934e 100644 --- a/jc/parsers/lsusb.py +++ b/jc/parsers/lsusb.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('lsusb', lsusb_command_output) - or - - import jc.parsers.lsusb - result = jc.parsers.lsusb.parse(lsusb_command_output) - Schema: Note: object keynames are assigned directly from the lsusb diff --git a/jc/parsers/mount.py b/jc/parsers/mount.py index 6b7fc095..048ab9b1 100644 --- a/jc/parsers/mount.py +++ b/jc/parsers/mount.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('mount', mount_command_output) - or - - import jc.parsers.mount - result = jc.parsers.mount.parse(mount_command_output) - Schema: [ diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index f417c241..9b0a48f0 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('netstat', netstat_command_output) - or - - import jc.parsers.netstat - result = jc.parsers.netstat.parse(netstat_command_output) - Schema: [ diff --git a/jc/parsers/nmcli.py b/jc/parsers/nmcli.py index e89516cd..cacbbad2 100644 --- a/jc/parsers/nmcli.py +++ b/jc/parsers/nmcli.py @@ -22,11 +22,6 @@ Usage (module): import jc result = jc.parse('nmcli', nmcli_command_output) - or - - import jc.parsers.nmcli - result = jc.parsers.nmcli.parse(nmcli_command_output) - Schema: Because there are so many options, the schema is not strictly defined. diff --git a/jc/parsers/ntpq.py b/jc/parsers/ntpq.py index e8e474a6..1c24731f 100644 --- a/jc/parsers/ntpq.py +++ b/jc/parsers/ntpq.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('ntpq', ntpq_command_output) - or - - import jc.parsers.ntpq - result = jc.parsers.ntpq.parse(ntpq_command_output) - Schema: [ diff --git a/jc/parsers/passwd.py b/jc/parsers/passwd.py index 6281fc5d..5a5e20a4 100644 --- a/jc/parsers/passwd.py +++ b/jc/parsers/passwd.py @@ -9,11 +9,6 @@ Usage (module): import jc result = jc.parse('passwd', passwd_file_output) - or - - import jc.parsers.passwd - result = jc.parsers.passwd.parse(passwd_file_output) - Schema: [ diff --git a/jc/parsers/pidstat.py b/jc/parsers/pidstat.py index b789cf54..7d8efc47 100644 --- a/jc/parsers/pidstat.py +++ b/jc/parsers/pidstat.py @@ -16,11 +16,6 @@ Usage (module): import jc result = jc.parse('pidstat', pidstat_command_output) - or - - import jc.parsers.pidstat - result = jc.parsers.pidstat.parse(pidstat_command_output) - Schema: [ diff --git a/jc/parsers/ping.py b/jc/parsers/ping.py index 0758045b..eaee59b6 100644 --- a/jc/parsers/ping.py +++ b/jc/parsers/ping.py @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('ping', ping_command_output) - or - - import jc.parsers.ping - result = jc.parsers.ping.parse(ping_command_output) - Schema: { diff --git a/jc/parsers/pip_list.py b/jc/parsers/pip_list.py index 00f5d310..416c7fb0 100644 --- a/jc/parsers/pip_list.py +++ b/jc/parsers/pip_list.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('pip_list', pip_list_command_output) - or - - import jc.parsers.pip_list - result = jc.parsers.pip_list.parse(pip_list_command_output) - Schema: [ diff --git a/jc/parsers/pip_show.py b/jc/parsers/pip_show.py index f465c26a..5d2cdb37 100644 --- a/jc/parsers/pip_show.py +++ b/jc/parsers/pip_show.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('pip_show', pip_show_command_output) - or - - import jc.parsers.pip_show - result = jc.parsers.pip_show.parse(pip_show_command_output) - Schema: [ diff --git a/jc/parsers/ps.py b/jc/parsers/ps.py index 06e7c7fa..3b529442 100644 --- a/jc/parsers/ps.py +++ b/jc/parsers/ps.py @@ -17,11 +17,6 @@ Usage (module): import jc result = jc.parse('ps', ps_command_output) - or - - import jc.parsers.ps - result = jc.parsers.ps.parse(ps_command_output) - Schema: [ diff --git a/jc/parsers/route.py b/jc/parsers/route.py index 7eb07872..169aca62 100644 --- a/jc/parsers/route.py +++ b/jc/parsers/route.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('route', route_command_output) - or - - import jc.parsers.route - result = jc.parsers.route.parse(route_command_output) - Schema: [ diff --git a/jc/parsers/rpm_qi.py b/jc/parsers/rpm_qi.py index 52e42d74..fa28f092 100644 --- a/jc/parsers/rpm_qi.py +++ b/jc/parsers/rpm_qi.py @@ -21,11 +21,6 @@ Usage (module): import jc result = jc.parse('rpm_qi', rpm_qi_command_output) - or - - import jc.parsers.rpm_qi - result = jc.parsers.rpm_qi.parse(rpm_qi_command_output) - Schema: [ diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py index eadfa495..8664e0dc 100644 --- a/jc/parsers/rsync.py +++ b/jc/parsers/rsync.py @@ -21,11 +21,6 @@ Usage (module): import jc result = jc.parse('rsync', rsync_command_output) - or - - import jc.parsers.rsync - result = jc.parsers.rsync.parse(rsync_command_output) - Schema: [ diff --git a/jc/parsers/sfdisk.py b/jc/parsers/sfdisk.py index 9900b60b..9553de51 100644 --- a/jc/parsers/sfdisk.py +++ b/jc/parsers/sfdisk.py @@ -22,11 +22,6 @@ Usage (module): import jc result = jc.parse('sfdisk', sfdisk_command_output) - or - - import jc.parsers.sfdisk - result = jc.parsers.sfdisk.parse(sfdisk_command_output) - Schema: [ diff --git a/jc/parsers/shadow.py b/jc/parsers/shadow.py index ad1ae570..184c49bc 100644 --- a/jc/parsers/shadow.py +++ b/jc/parsers/shadow.py @@ -9,11 +9,6 @@ Usage (module): import jc result = jc.parse('shadow', shadow_file_output) - or - - import jc.parsers.shadow - result = jc.parsers.shadow.parse(shadow_file_output) - Schema: [ diff --git a/jc/parsers/ss.py b/jc/parsers/ss.py index 10bd69dc..5d922c12 100644 --- a/jc/parsers/ss.py +++ b/jc/parsers/ss.py @@ -16,11 +16,6 @@ Usage (module): import jc result = jc.parse('ss', ss_command_output) - or - - import jc.parsers.ss - result = jc.parsers.ss.parse(ss_command_output) - Schema: Information from https://www.cyberciti.biz/files/ss.html used to define diff --git a/jc/parsers/stat.py b/jc/parsers/stat.py index ca58789d..cf20f504 100644 --- a/jc/parsers/stat.py +++ b/jc/parsers/stat.py @@ -19,11 +19,6 @@ Usage (module): import jc result = jc.parse('stat', stat_command_output) - or - - import jc.parsers.stat - result = jc.parsers.stat.parse(stat_command_output) - Schema: [ diff --git a/jc/parsers/sysctl.py b/jc/parsers/sysctl.py index 022546b1..abb85fd1 100644 --- a/jc/parsers/sysctl.py +++ b/jc/parsers/sysctl.py @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('sysctl', sysctl_command_output) - or - - import jc.parsers.sysctl - result = jc.parsers.sysctl.parse(sysctl_command_output) - Schema: { diff --git a/jc/parsers/systemctl.py b/jc/parsers/systemctl.py index 47136e15..71825192 100644 --- a/jc/parsers/systemctl.py +++ b/jc/parsers/systemctl.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('systemctl', systemctl_command_output) - or - - import jc.parsers.systemctl - result = jc.parsers.systemctl.parse(systemctl_command_output) - Schema: [ diff --git a/jc/parsers/systemctl_lj.py b/jc/parsers/systemctl_lj.py index a6ccd5fd..671e81e0 100644 --- a/jc/parsers/systemctl_lj.py +++ b/jc/parsers/systemctl_lj.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('systemctl_lj', systemctl_lj_command_output) - or - - import jc.parsers.systemctl_lj - result = jc.parsers.systemctl_lj.parse(systemctl_lj_command_output) - Schema: [ diff --git a/jc/parsers/systemctl_ls.py b/jc/parsers/systemctl_ls.py index 670986d5..9a8a9527 100644 --- a/jc/parsers/systemctl_ls.py +++ b/jc/parsers/systemctl_ls.py @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('systemctl_ls', systemctl_ls_command_output) - or - - import jc.parsers.systemctl_ls - result = jc.parsers.systemctl_ls.parse(systemctl_ls_command_output) - Schema: [ diff --git a/jc/parsers/systemctl_luf.py b/jc/parsers/systemctl_luf.py index 8fd540e8..5ed9526f 100644 --- a/jc/parsers/systemctl_luf.py +++ b/jc/parsers/systemctl_luf.py @@ -14,11 +14,6 @@ Usage (module): import jc result = jc.parse('systemctl_luf', systemctl_luf_command_output) - or - - import jc.parsers.systemctl_luf - result = jc.parsers.systemctl_luf.parse(systemctl_luf_command_output) - Schema: [ diff --git a/jc/parsers/systeminfo.py b/jc/parsers/systeminfo.py index 85222662..fde6ec94 100644 --- a/jc/parsers/systeminfo.py +++ b/jc/parsers/systeminfo.py @@ -19,11 +19,6 @@ Usage (module): import jc result = jc.parse('systeminfo', systeminfo_command_output) - or - - import jc.parsers.systeminfo - result = jc.parsers.systeminfo.parse(systeminfo_command_output) - Schema: { diff --git a/jc/parsers/time.py b/jc/parsers/time.py index 74b9ce56..a0ad598e 100644 --- a/jc/parsers/time.py +++ b/jc/parsers/time.py @@ -19,11 +19,6 @@ Usage (module): import jc result = jc.parse('time', time_command_output) - or - - import jc.parsers.time - result = jc.parsers.time.parse(time_command_output) - Schema: Source: https://www.freebsd.org/cgi/man.cgi?query=getrusage diff --git a/jc/parsers/timedatectl.py b/jc/parsers/timedatectl.py index e4957c69..79752a2b 100644 --- a/jc/parsers/timedatectl.py +++ b/jc/parsers/timedatectl.py @@ -16,11 +16,6 @@ Usage (module): import jc result = jc.parse('timedatectl', timedatectl_command_output) - or - - import jc.parsers.timedatectl - result = jc.parsers.timedatectl.parse(timedatectl_command_output) - Schema: { diff --git a/jc/parsers/tracepath.py b/jc/parsers/tracepath.py index 2e0396a0..63ebe2c5 100644 --- a/jc/parsers/tracepath.py +++ b/jc/parsers/tracepath.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('tracepath', tracepath_command_output) - or - - import jc.parsers.tracepath - result = jc.parsers.tracepath.parse(tracepath_command_output) - Schema: { diff --git a/jc/parsers/traceroute.py b/jc/parsers/traceroute.py index aa11b5a1..913aa34b 100644 --- a/jc/parsers/traceroute.py +++ b/jc/parsers/traceroute.py @@ -22,11 +22,6 @@ Usage (module): import jc result = jc.parse('traceroute', traceroute_command_output) - or - - import jc.parsers.traceroute - result = jc.parsers.traceroute.parse(traceroute_command_output) - Schema: { diff --git a/jc/parsers/ufw.py b/jc/parsers/ufw.py index cb0a8f39..35f7843e 100644 --- a/jc/parsers/ufw.py +++ b/jc/parsers/ufw.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('ufw', ufw_command_output) - or - - import jc.parsers.ufw - result = jc.parsers.ufw.parse(ufw_command_output) - Schema: { diff --git a/jc/parsers/ufw_appinfo.py b/jc/parsers/ufw_appinfo.py index 04524653..3327c3f5 100644 --- a/jc/parsers/ufw_appinfo.py +++ b/jc/parsers/ufw_appinfo.py @@ -21,11 +21,6 @@ Usage (module): import jc result = jc.parse('ufw_appinfo', ufw_appinfo_command_output) - or - - import jc.parsers.ufw_appinfo - result = jc.parsers.ufw_appinfo.parse(ufw_appinfo_command_output) - Schema: [ diff --git a/jc/parsers/uname.py b/jc/parsers/uname.py index 7260803f..e6d709a6 100644 --- a/jc/parsers/uname.py +++ b/jc/parsers/uname.py @@ -15,11 +15,6 @@ Usage (module): import jc result = jc.parse('uname', uname_command_output) - or - - import jc.parsers.uname - result = jc.parsers.uname.parse(uname_command_output) - Schema: { diff --git a/jc/parsers/universal.py b/jc/parsers/universal.py index 5fe253a4..ac1ed969 100644 --- a/jc/parsers/universal.py +++ b/jc/parsers/universal.py @@ -7,7 +7,15 @@ from typing import List, Dict def simple_table_parse(data: List[str]) -> List[Dict]: """ - Parse simple tables. The last column may contain data with spaces. + Parse simple tables. There should be no blank cells. The last column + may contain data with spaces. + + Example Table: + + col1 col2 col3 col4 col5 + apple orange pear banana my favorite fruits + carrot squash celery spinach my favorite veggies + chicken beef pork eggs my favorite proteins Parameters: @@ -36,6 +44,14 @@ def simple_table_parse(data: List[str]) -> List[Dict]: def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]: """ Parse tables with missing column data or with spaces in column data. + Data elements must line up within column boundaries. + + Example Table: + + col1 col2 col3 col4 col5 + orange pear fuzzy peach my favorite fruits + green beans celery spinach my favorite veggies + chicken beef pork brown eggs my favorite proteins Parameters: diff --git a/jc/parsers/upower.py b/jc/parsers/upower.py index 7dd511a3..e0621b7b 100644 --- a/jc/parsers/upower.py +++ b/jc/parsers/upower.py @@ -19,11 +19,6 @@ Usage (module): import jc result = jc.parse('upower', upower_command_output) - or - - import jc.parsers.upower - result = jc.parsers.upower.parse(upower_command_output) - Schema: [ diff --git a/jc/parsers/uptime.py b/jc/parsers/uptime.py index 486f8799..99e5f705 100644 --- a/jc/parsers/uptime.py +++ b/jc/parsers/uptime.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('uptime', uptime_command_output) - or - - import jc.parsers.uptime - result = jc.parsers.uptime.parse(uptime_command_output) - Schema: { diff --git a/jc/parsers/vmstat.py b/jc/parsers/vmstat.py index 8b31acdd..6b44c7eb 100644 --- a/jc/parsers/vmstat.py +++ b/jc/parsers/vmstat.py @@ -21,11 +21,6 @@ Usage (module): import jc result = jc.parse('vmstat', vmstat_command_output) - or - - import jc.parsers.vmstat - result = jc.parsers.vmstat.parse(vmstat_command_output) - Schema: [ diff --git a/jc/parsers/w.py b/jc/parsers/w.py index 7ce640c8..a40659d8 100644 --- a/jc/parsers/w.py +++ b/jc/parsers/w.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('w', w_command_output) - or - - import jc.parsers.w - result = jc.parsers.w.parse(w_command_output) - Schema: [ diff --git a/jc/parsers/wc.py b/jc/parsers/wc.py index eb1def49..9638a5b0 100644 --- a/jc/parsers/wc.py +++ b/jc/parsers/wc.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('wc', wc_command_output) - or - - import jc.parsers.wc - result = jc.parsers.wc.parse(wc_command_output) - Schema: [ diff --git a/jc/parsers/who.py b/jc/parsers/who.py index c3655bfc..199c6498 100644 --- a/jc/parsers/who.py +++ b/jc/parsers/who.py @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('who', who_command_output) - or - - import jc.parsers.who - result = jc.parsers.who.parse(who_command_output) - Schema: [ diff --git a/jc/parsers/xml.py b/jc/parsers/xml.py index 9eda8d53..e24ae5c9 100644 --- a/jc/parsers/xml.py +++ b/jc/parsers/xml.py @@ -9,11 +9,6 @@ Usage (module): import jc result = jc.parse('xml', xml_file_output) - or - - import jc.parsers.xml - result = jc.parsers.xml.parse(xml_file_output) - Schema: XML Document converted to a Dictionary diff --git a/jc/parsers/xrandr.py b/jc/parsers/xrandr.py index c2f5ed66..f9c0853e 100644 --- a/jc/parsers/xrandr.py +++ b/jc/parsers/xrandr.py @@ -13,11 +13,6 @@ Usage (module): import jc result = jc.parse('xrandr', xrandr_command_output) - or - - import jc.parsers.xrandr - result = jc.parsers.xrandr.parse(xrandr_command_output) - Schema: { diff --git a/jc/parsers/yaml.py b/jc/parsers/yaml.py index a8e815fd..f55b2acb 100644 --- a/jc/parsers/yaml.py +++ b/jc/parsers/yaml.py @@ -9,11 +9,6 @@ Usage (module): import jc result = jc.parse('yaml', yaml_file_output) - or - - import jc.parsers.yaml - result = jc.parsers.yaml.parse(yaml_file_output) - Schema: YAML Document converted to a Dictionary diff --git a/jc/parsers/zipinfo.py b/jc/parsers/zipinfo.py index d06ae365..f52e7baa 100644 --- a/jc/parsers/zipinfo.py +++ b/jc/parsers/zipinfo.py @@ -18,11 +18,6 @@ Usage (module): import jc result = jc.parse('zipinfo', zipinfo_command_output) - or - - import jc.parsers.zipinfo - result = jc.parsers.zipinfo.parse(zipinfo_command_output) - Schema: [ From def7aa5764e69cd0cf2c5b70aa62c62023f76c9c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 15:36:11 -0800 Subject: [PATCH 25/87] formatting --- docs/parsers/universal.md | 16 ++++++++-------- jc/parsers/universal.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/parsers/universal.md b/docs/parsers/universal.md index 934249c1..934b203c 100644 --- a/docs/parsers/universal.md +++ b/docs/parsers/universal.md @@ -23,10 +23,10 @@ may contain data with spaces. Example Table: - col1 col2 col3 col4 col5 - apple orange pear banana my favorite fruits - carrot squash celery spinach my favorite veggies - chicken beef pork eggs my favorite proteins + col1 col2 col3 col4 col5 + apple orange pear banana my favorite fruits + carrot squash celery spinach my favorite veggies + chicken beef pork eggs my favorite proteins Parameters: @@ -56,10 +56,10 @@ Data elements must line up within column boundaries. Example Table: - col1 col2 col3 col4 col5 - orange pear fuzzy peach my favorite fruits - green beans celery spinach my favorite veggies - chicken beef pork brown eggs my favorite proteins + col1 col2 col3 col4 col5 + apple orange fuzzy peach my favorite fruits + green beans celery spinach my favorite veggies + chicken beef brown eggs my favorite proteins Parameters: diff --git a/jc/parsers/universal.py b/jc/parsers/universal.py index ac1ed969..1be0812f 100644 --- a/jc/parsers/universal.py +++ b/jc/parsers/universal.py @@ -12,10 +12,10 @@ def simple_table_parse(data: List[str]) -> List[Dict]: Example Table: - col1 col2 col3 col4 col5 - apple orange pear banana my favorite fruits - carrot squash celery spinach my favorite veggies - chicken beef pork eggs my favorite proteins + col1 col2 col3 col4 col5 + apple orange pear banana my favorite fruits + carrot squash celery spinach my favorite veggies + chicken beef pork eggs my favorite proteins Parameters: @@ -48,10 +48,10 @@ def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]: Example Table: - col1 col2 col3 col4 col5 - orange pear fuzzy peach my favorite fruits - green beans celery spinach my favorite veggies - chicken beef pork brown eggs my favorite proteins + col1 col2 col3 col4 col5 + apple orange fuzzy peach my favorite fruits + green beans celery spinach my favorite veggies + chicken beef brown eggs my favorite proteins Parameters: From 54e8f58145f6c4332fbd375d112259ea7abb2d72 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 10 Mar 2022 16:50:55 -0800 Subject: [PATCH 26/87] add table result examples --- docs/parsers/universal.md | 15 ++++++++++++++- jc/parsers/universal.py | 15 ++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/parsers/universal.md b/docs/parsers/universal.md index 934b203c..38235594 100644 --- a/docs/parsers/universal.md +++ b/docs/parsers/universal.md @@ -28,6 +28,12 @@ Example Table: carrot squash celery spinach my favorite veggies chicken beef pork eggs my favorite proteins + [{'col1': 'apple', 'col2': 'orange', 'col3': 'pear', 'col4': + 'banana', 'col5': 'my favorite fruits'}, {'col1': 'carrot', 'col2': + 'squash', 'col3': 'celery', 'col4': 'spinach', 'col5': + 'my favorite veggies'}, {'col1': 'chicken', 'col2': 'beef', 'col3': + 'pork', 'col4': 'eggs', 'col5': 'my favorite proteins'}] + Parameters: data: (list) Text data to parse that has been split into lines @@ -52,7 +58,8 @@ def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict] ``` Parse tables with missing column data or with spaces in column data. -Data elements must line up within column boundaries. +Blank cells are converted to None in the resulting dictionary. Data +elements must line up within column boundaries. Example Table: @@ -61,6 +68,12 @@ Example Table: green beans celery spinach my favorite veggies chicken beef brown eggs my favorite proteins + [{'col1': 'apple', 'col2': 'orange', 'col3': None, 'col4': + 'fuzzy peach', 'col5': 'my favorite fruits'}, {'col1': + 'green beans', 'col2': None, 'col3': 'celery', 'col4': 'spinach', + 'col5': 'my favorite veggies'}, {'col1': 'chicken', 'col2': 'beef', + 'col3': None, 'col4': 'brown eggs', 'col5': 'my favorite proteins'}] + Parameters: data: (list) Text data to parse that has been split into lines diff --git a/jc/parsers/universal.py b/jc/parsers/universal.py index 1be0812f..1ee3e257 100644 --- a/jc/parsers/universal.py +++ b/jc/parsers/universal.py @@ -17,6 +17,12 @@ def simple_table_parse(data: List[str]) -> List[Dict]: carrot squash celery spinach my favorite veggies chicken beef pork eggs my favorite proteins + [{'col1': 'apple', 'col2': 'orange', 'col3': 'pear', 'col4': + 'banana', 'col5': 'my favorite fruits'}, {'col1': 'carrot', 'col2': + 'squash', 'col3': 'celery', 'col4': 'spinach', 'col5': + 'my favorite veggies'}, {'col1': 'chicken', 'col2': 'beef', 'col3': + 'pork', 'col4': 'eggs', 'col5': 'my favorite proteins'}] + Parameters: data: (list) Text data to parse that has been split into lines @@ -44,7 +50,8 @@ def simple_table_parse(data: List[str]) -> List[Dict]: def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]: """ Parse tables with missing column data or with spaces in column data. - Data elements must line up within column boundaries. + Blank cells are converted to None in the resulting dictionary. Data + elements must line up within column boundaries. Example Table: @@ -53,6 +60,12 @@ def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]: green beans celery spinach my favorite veggies chicken beef brown eggs my favorite proteins + [{'col1': 'apple', 'col2': 'orange', 'col3': None, 'col4': + 'fuzzy peach', 'col5': 'my favorite fruits'}, {'col1': + 'green beans', 'col2': None, 'col3': 'celery', 'col4': 'spinach', + 'col5': 'my favorite veggies'}, {'col1': 'chicken', 'col2': 'beef', + 'col3': None, 'col4': 'brown eggs', 'col5': 'my favorite proteins'}] + Parameters: data: (list) Text data to parse that has been split into lines From dd052e01469e9e3328f7326d3591365a8e7322eb Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 11 Mar 2022 10:25:24 -0800 Subject: [PATCH 27/87] add underscores to column examples --- docs/parsers/universal.md | 25 +++++++++++++------------ jc/parsers/universal.py | 25 +++++++++++++------------ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/docs/parsers/universal.md b/docs/parsers/universal.md index 38235594..2a01569f 100644 --- a/docs/parsers/universal.md +++ b/docs/parsers/universal.md @@ -23,16 +23,16 @@ may contain data with spaces. Example Table: - col1 col2 col3 col4 col5 + col_1 col_2 col_3 col_4 col_5 apple orange pear banana my favorite fruits carrot squash celery spinach my favorite veggies chicken beef pork eggs my favorite proteins - [{'col1': 'apple', 'col2': 'orange', 'col3': 'pear', 'col4': - 'banana', 'col5': 'my favorite fruits'}, {'col1': 'carrot', 'col2': - 'squash', 'col3': 'celery', 'col4': 'spinach', 'col5': - 'my favorite veggies'}, {'col1': 'chicken', 'col2': 'beef', 'col3': - 'pork', 'col4': 'eggs', 'col5': 'my favorite proteins'}] + [{'col_1': 'apple', 'col_2': 'orange', 'col_3': 'pear', 'col_4': + 'banana', 'col_5': 'my favorite fruits'}, {'col_1': 'carrot', + 'col_2': 'squash', 'col_3': 'celery', 'col_4': 'spinach', 'col_5': + 'my favorite veggies'}, {'col_1': 'chicken', 'col_2': 'beef', + 'col_3': 'pork', 'col_4': 'eggs', 'col_5': 'my favorite proteins'}] Parameters: @@ -63,16 +63,17 @@ elements must line up within column boundaries. Example Table: - col1 col2 col3 col4 col5 + col_1 col_2 col_3 col_4 col_5 apple orange fuzzy peach my favorite fruits green beans celery spinach my favorite veggies chicken beef brown eggs my favorite proteins - [{'col1': 'apple', 'col2': 'orange', 'col3': None, 'col4': - 'fuzzy peach', 'col5': 'my favorite fruits'}, {'col1': - 'green beans', 'col2': None, 'col3': 'celery', 'col4': 'spinach', - 'col5': 'my favorite veggies'}, {'col1': 'chicken', 'col2': 'beef', - 'col3': None, 'col4': 'brown eggs', 'col5': 'my favorite proteins'}] + [{'col_1': 'apple', 'col_2': 'orange', 'col_3': None, 'col_4': + 'fuzzy peach', 'col_5': 'my favorite fruits'}, {'col_1': + 'green beans', 'col_2': None, 'col_3': 'celery', 'col_4': 'spinach', + 'col_5': 'my favorite veggies'}, {'col_1': 'chicken', 'col_2': + 'beef', 'col_3': None, 'col_4': 'brown eggs', 'col_5': + 'my favorite proteins'}] Parameters: diff --git a/jc/parsers/universal.py b/jc/parsers/universal.py index 1ee3e257..e3b9fa62 100644 --- a/jc/parsers/universal.py +++ b/jc/parsers/universal.py @@ -12,16 +12,16 @@ def simple_table_parse(data: List[str]) -> List[Dict]: Example Table: - col1 col2 col3 col4 col5 + col_1 col_2 col_3 col_4 col_5 apple orange pear banana my favorite fruits carrot squash celery spinach my favorite veggies chicken beef pork eggs my favorite proteins - [{'col1': 'apple', 'col2': 'orange', 'col3': 'pear', 'col4': - 'banana', 'col5': 'my favorite fruits'}, {'col1': 'carrot', 'col2': - 'squash', 'col3': 'celery', 'col4': 'spinach', 'col5': - 'my favorite veggies'}, {'col1': 'chicken', 'col2': 'beef', 'col3': - 'pork', 'col4': 'eggs', 'col5': 'my favorite proteins'}] + [{'col_1': 'apple', 'col_2': 'orange', 'col_3': 'pear', 'col_4': + 'banana', 'col_5': 'my favorite fruits'}, {'col_1': 'carrot', + 'col_2': 'squash', 'col_3': 'celery', 'col_4': 'spinach', 'col_5': + 'my favorite veggies'}, {'col_1': 'chicken', 'col_2': 'beef', + 'col_3': 'pork', 'col_4': 'eggs', 'col_5': 'my favorite proteins'}] Parameters: @@ -55,16 +55,17 @@ def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]: Example Table: - col1 col2 col3 col4 col5 + col_1 col_2 col_3 col_4 col_5 apple orange fuzzy peach my favorite fruits green beans celery spinach my favorite veggies chicken beef brown eggs my favorite proteins - [{'col1': 'apple', 'col2': 'orange', 'col3': None, 'col4': - 'fuzzy peach', 'col5': 'my favorite fruits'}, {'col1': - 'green beans', 'col2': None, 'col3': 'celery', 'col4': 'spinach', - 'col5': 'my favorite veggies'}, {'col1': 'chicken', 'col2': 'beef', - 'col3': None, 'col4': 'brown eggs', 'col5': 'my favorite proteins'}] + [{'col_1': 'apple', 'col_2': 'orange', 'col_3': None, 'col_4': + 'fuzzy peach', 'col_5': 'my favorite fruits'}, {'col_1': + 'green beans', 'col_2': None, 'col_3': 'celery', 'col_4': 'spinach', + 'col_5': 'my favorite veggies'}, {'col_1': 'chicken', 'col_2': + 'beef', 'col_3': None, 'col_4': 'brown eggs', 'col_5': + 'my favorite proteins'}] Parameters: From 6c3e0e2aa0e8cc375d9f7473930b18957d33f1aa Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 11 Mar 2022 12:37:01 -0800 Subject: [PATCH 28/87] formatting --- jc/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/utils.py b/jc/utils.py index 678f824a..8aa0a75d 100644 --- a/jc/utils.py +++ b/jc/utils.py @@ -108,7 +108,7 @@ def compatibility(mod_name: str, compatible: List, quiet: bool = False) -> None: mod = mod_name.split('.')[-1] compat_list = ', '.join(compatible) warning_message([ - f'{mod} parser not compatible with your OS ({sys.platform}).', + f'{mod} parser is not compatible with your OS ({sys.platform}).', f'Compatible platforms: {compat_list}' ]) From bc97052ed4ee3bf3f25127968139f2e89a659fe3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 11 Mar 2022 12:37:17 -0800 Subject: [PATCH 29/87] initial add mpstat parser --- CHANGELOG | 1 + jc/lib.py | 1 + jc/parsers/mpstat.py | 107 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 jc/parsers/mpstat.py diff --git a/CHANGELOG b/CHANGELOG index cd899fe9..52d219d1 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,7 @@ jc changelog 20220309 v1.18.6 (in progress) - Add pidstat command parser tested on linux - Add pidstat command streaming parser tested on linux +- Add mpstat command parser tested on linux 20220305 v1.18.5 - Fix date parser to ensure AM/PM period string is always uppercase diff --git a/jc/lib.py b/jc/lib.py index b61f39df..f9f03e3f 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -59,6 +59,7 @@ parsers = [ 'lsof', 'lsusb', 'mount', + 'mpstat', 'netstat', 'nmcli', 'ntpq', diff --git a/jc/parsers/mpstat.py b/jc/parsers/mpstat.py new file mode 100644 index 00000000..63e946ae --- /dev/null +++ b/jc/parsers/mpstat.py @@ -0,0 +1,107 @@ +"""jc - JSON Convert `mpstat` command output parser + +<> + +Usage (cli): + + $ mpstat | jc --mpstat + + or + + $ jc mpstat + +Usage (module): + + import jc + result = jc.parse('mpstat', mpstat_command_output) + +Schema: + + [ + { + "mpstat": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ mpstat | jc --mpstat -p + [] + + $ mpstat | jc --mpstat -p -r + [] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`mpstat` command parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + magic_commands = ['mpstat'] + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + + # process the data here + # rebuild output for added semantic information + # use helper functions in jc.utils for int, float, bool + # conversions and timestamps + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + # parse the content here + # check out helper functions in jc.utils + # and jc.parsers.universal + + pass + + return raw_output if raw else _process(raw_output) From ae1c331595dd857d8ae7f8cf93daee1b26be2c3e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 11 Mar 2022 13:15:39 -0800 Subject: [PATCH 30/87] initial working version --- jc/parsers/mpstat.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/jc/parsers/mpstat.py b/jc/parsers/mpstat.py index 63e946ae..db734d4a 100644 --- a/jc/parsers/mpstat.py +++ b/jc/parsers/mpstat.py @@ -35,6 +35,7 @@ Examples: """ from typing import List, Dict import jc.utils +from jc.parsers.universal import simple_table_parse class info(): @@ -93,15 +94,39 @@ def parse( jc.utils.input_type_check(data) raw_output: List = [] + output_line: Dict = {} + header_found = False + header_start: int = 0 + stat_type: str = '' # 'cpu' or 'interrupts' if jc.utils.has_data(data): for line in filter(None, data.splitlines()): - # parse the content here - # check out helper functions in jc.utils - # and jc.parsers.universal + # check for header, normalize it, and fix the time column + if ' CPU ' in line: + header_found = True + if '%usr' in line: + stat_type = 'cpu' + else: + stat_type = 'interrupts' - pass + header_text: str = line.replace('/', '_')\ + .replace('%', 'percent_')\ + .lower() + header_start = line.find('CPU ') + header_text = header_text[header_start:] + continue + + # data line - pull time from beginning and then parse as a table + if header_found: + output_line = simple_table_parse([header_text, line[header_start:]])[0] + output_line['type'] = stat_type + item_time = line[:header_start].strip() + if 'Average:' not in item_time: + output_line['time'] = line[:header_start].strip() + else: + output_line['average'] = True + raw_output.append(output_line) return raw_output if raw else _process(raw_output) From 2cddb1f0bb497be0be071cdcb56bf49d2dafc920 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 11 Mar 2022 14:00:19 -0800 Subject: [PATCH 31/87] working mpstat --- jc/parsers/mpstat.py | 109 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 94 insertions(+), 15 deletions(-) diff --git a/jc/parsers/mpstat.py b/jc/parsers/mpstat.py index db734d4a..e3f92a3e 100644 --- a/jc/parsers/mpstat.py +++ b/jc/parsers/mpstat.py @@ -1,7 +1,5 @@ """jc - JSON Convert `mpstat` command output parser -<> - Usage (cli): $ mpstat | jc --mpstat @@ -19,19 +17,93 @@ Schema: [ { - "mpstat": string, - "bar": boolean, - "baz": integer + "type": string, + "time": string, + "cpu": string, + "average": boolean, + "percent_usr": float, + "percent_nice": float, + "percent_sys": float, + "percent_iowait": float, + "percent_irq": float, + "percent_soft": float, + "percent_steal": float, + "percent_guest": float, + "percent_gnice": float, + "percent_idle": float, + "intr_s": float, + "_s": float, # is an integer + "nmi_s": float, + "loc_s": float, + "spu_s": float, + "pmi_s": float, + "iwi_s": float, + "rtr_s": float, + "res_s": float, + "cal_s": float, + "tlb_s": float, + "trm_s": float, + "thr_s": float, + "dfr_s": float, + "mce_s": float, + "mcp_s": float, + "err_s": float, + "mis_s": float, + "pin_s": float, + "npi_s": float, + "piw_s": float, + "hi_s": float, + "timer_s": float, + "net_tx_s": float, + "net_rx_s": float, + "block_s": float, + "block_iopoll_s": float, + "tasklet_s": float, + "sched_s": float, + "hrtimer_s": float, + "rcu_s": float } ] Examples: $ mpstat | jc --mpstat -p - [] + [ + { + "cpu": "all", + "percent_usr": 12.94, + "percent_nice": 0.0, + "percent_sys": 26.42, + "percent_iowait": 0.43, + "percent_irq": 0.0, + "percent_soft": 0.16, + "percent_steal": 0.0, + "percent_guest": 0.0, + "percent_gnice": 0.0, + "percent_idle": 60.05, + "type": "cpu", + "time": "01:58:14 PM" + } + ] $ mpstat | jc --mpstat -p -r - [] + [ + { + "cpu": "all", + "percent_usr": "12.94", + "percent_nice": "0.00", + "percent_sys": "26.42", + "percent_iowait": "0.43", + "percent_irq": "0.00", + "percent_soft": "0.16", + "percent_steal": "0.00", + "percent_guest": "0.00", + "percent_gnice": "0.00", + "percent_idle": "60.05", + "type": "cpu", + "time": "01:58:14 PM" + } + ] """ from typing import List, Dict import jc.utils @@ -63,11 +135,18 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - - # process the data here - # rebuild output for added semantic information - # use helper functions in jc.utils for int, float, bool - # conversions and timestamps + float_list = [ + "percent_usr", "percent_nice", "percent_sys", "percent_iowait", "percent_irq", + "percent_soft", "percent_steal", "percent_guest", "percent_gnice", "percent_idle", "intr_s", + "nmi_s", "loc_s", "spu_s", "pmi_s", "iwi_s", "rtr_s", "res_s", "cal_s", "tlb_s", "trm_s", + "thr_s", "dfr_s", "mce_s", "mcp_s", "err_s", "mis_s", "pin_s", "npi_s", "piw_s", "hi_s", + "timer_s", "net_tx_s", "net_rx_s", "block_s", "block_iopoll_s", "tasklet_s", "sched_s", + "hrtimer_s", "rcu_s" + ] + for entry in proc_data: + for key in entry: + if (key in float_list or (key[0].isdigit() and key.endswith('_s'))): + entry[key] = jc.utils.convert_to_float(entry[key]) return proc_data @@ -95,7 +174,7 @@ def parse( raw_output: List = [] output_line: Dict = {} - header_found = False + header_found: bool = False header_start: int = 0 stat_type: str = '' # 'cpu' or 'interrupts' @@ -112,8 +191,8 @@ def parse( stat_type = 'interrupts' header_text: str = line.replace('/', '_')\ - .replace('%', 'percent_')\ - .lower() + .replace('%', 'percent_')\ + .lower() header_start = line.find('CPU ') header_text = header_text[header_start:] continue From d7efd25d88a8f489564ce48223845a16e4ab1cc7 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 11 Mar 2022 14:54:06 -0800 Subject: [PATCH 32/87] add mpstat to docs --- EXAMPLES.md | 64 +++++++++++++++++++ README.md | 1 + docs/parsers/mpstat.md | 136 +++++++++++++++++++++++++++++++++++++++++ man/jc.1 | 7 ++- 4 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 docs/parsers/mpstat.md diff --git a/EXAMPLES.md b/EXAMPLES.md index 943933db..21debe32 100644 --- a/EXAMPLES.md +++ b/EXAMPLES.md @@ -2149,6 +2149,29 @@ mount | jc --mount -p # or: jc -p mount } ] ``` +### mpstat +```bash +mpstat | jc --mpstat -p # or jc -p mpstat +``` +```json +[ + { + "cpu": "all", + "percent_usr": 12.94, + "percent_nice": 0.0, + "percent_sys": 26.42, + "percent_iowait": 0.43, + "percent_irq": 0.0, + "percent_soft": 0.16, + "percent_steal": 0.0, + "percent_guest": 0.0, + "percent_gnice": 0.0, + "percent_idle": 60.05, + "type": "cpu", + "time": "01:58:14 PM" + } +] +``` ### netstat ```bash netstat -apee | jc --netstat -p # or: jc -p netstat -apee @@ -2497,6 +2520,47 @@ cat /etc/passwd | jc --passwd -p } ] ``` +### pidstat +```bash +pidstat -hl | jc --pidstat -p # or jc -p pidstat -hl +``` +```json +[ + { + "time": 1646859134, + "uid": 0, + "pid": 1, + "percent_usr": 0.0, + "percent_system": 0.03, + "percent_guest": 0.0, + "percent_cpu": 0.03, + "cpu": 0, + "command": "/usr/lib/systemd/systemd --switched-root --system..." + }, + { + "time": 1646859134, + "uid": 0, + "pid": 6, + "percent_usr": 0.0, + "percent_system": 0.0, + "percent_guest": 0.0, + "percent_cpu": 0.0, + "cpu": 0, + "command": "ksoftirqd/0" + }, + { + "time": 1646859134, + "uid": 0, + "pid": 2263, + "percent_usr": 0.0, + "percent_system": 0.0, + "percent_guest": 0.0, + "percent_cpu": 0.0, + "cpu": 0, + "command": "kworker/0:0" + } +] +``` ### ping ```bash ping 8.8.8.8 -c 3 | jc --ping -p # or: jc -p ping 8.8.8.8 -c 3 diff --git a/README.md b/README.md index 354e373d..21bf10c6 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,7 @@ option. - `--lsof` enables the `lsof` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/lsof)) - `--lsusb` enables the `lsusb` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/lsusb)) - `--mount` enables the `mount` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/mount)) +- `--mpstat` enables the `mpstat` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat)) - `--netstat` enables the `netstat` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/netstat)) - `--nmcli` enables the `nmcli` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/nmcli)) - `--ntpq` enables the `ntpq -p` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ntpq)) diff --git a/docs/parsers/mpstat.md b/docs/parsers/mpstat.md new file mode 100644 index 00000000..588b7612 --- /dev/null +++ b/docs/parsers/mpstat.md @@ -0,0 +1,136 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.mpstat + +jc - JSON Convert `mpstat` command output parser + +Usage (cli): + + $ mpstat | jc --mpstat + + or + + $ jc mpstat + +Usage (module): + + import jc + result = jc.parse('mpstat', mpstat_command_output) + +Schema: + + [ + { + "type": string, + "time": string, + "cpu": string, + "average": boolean, + "percent_usr": float, + "percent_nice": float, + "percent_sys": float, + "percent_iowait": float, + "percent_irq": float, + "percent_soft": float, + "percent_steal": float, + "percent_guest": float, + "percent_gnice": float, + "percent_idle": float, + "intr_s": float, + "_s": float, # is an integer + "nmi_s": float, + "loc_s": float, + "spu_s": float, + "pmi_s": float, + "iwi_s": float, + "rtr_s": float, + "res_s": float, + "cal_s": float, + "tlb_s": float, + "trm_s": float, + "thr_s": float, + "dfr_s": float, + "mce_s": float, + "mcp_s": float, + "err_s": float, + "mis_s": float, + "pin_s": float, + "npi_s": float, + "piw_s": float, + "hi_s": float, + "timer_s": float, + "net_tx_s": float, + "net_rx_s": float, + "block_s": float, + "block_iopoll_s": float, + "tasklet_s": float, + "sched_s": float, + "hrtimer_s": float, + "rcu_s": float + } + ] + +Examples: + + $ mpstat | jc --mpstat -p + [ + { + "cpu": "all", + "percent_usr": 12.94, + "percent_nice": 0.0, + "percent_sys": 26.42, + "percent_iowait": 0.43, + "percent_irq": 0.0, + "percent_soft": 0.16, + "percent_steal": 0.0, + "percent_guest": 0.0, + "percent_gnice": 0.0, + "percent_idle": 60.05, + "type": "cpu", + "time": "01:58:14 PM" + } + ] + + $ mpstat | jc --mpstat -p -r + [ + { + "cpu": "all", + "percent_usr": "12.94", + "percent_nice": "0.00", + "percent_sys": "26.42", + "percent_iowait": "0.43", + "percent_irq": "0.00", + "percent_soft": "0.16", + "percent_steal": "0.00", + "percent_guest": "0.00", + "percent_gnice": "0.00", + "percent_idle": "60.05", + "type": "cpu", + "time": "01:58:14 PM" + } + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/man/jc.1 b/man/jc.1 index 45711507..40ce7249 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-03-10 1.18.6 "JSON Convert" +.TH jc 1 2022-03-11 1.18.6 "JSON Convert" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -252,6 +252,11 @@ Key/Value file parser \fB--mount\fP `mount` command parser +.TP +.B +\fB--mpstat\fP +`mpstat` command parser + .TP .B \fB--netstat\fP From 6f98b27a05d6e1f7571a97316b193f096521b959 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 11 Mar 2022 15:33:31 -0800 Subject: [PATCH 33/87] add mpstat tests --- tests/fixtures/centos-7.7/mpstat-A-2-5.json | 1 + tests/fixtures/centos-7.7/mpstat-A-2-5.out | 86 +++++++++++++++++++++ tests/fixtures/centos-7.7/mpstat-A.json | 1 + tests/fixtures/centos-7.7/mpstat-A.out | 15 ++++ tests/fixtures/centos-7.7/mpstat.json | 1 + tests/fixtures/centos-7.7/mpstat.out | 5 ++ tests/test_mpstat.py | 59 ++++++++++++++ 7 files changed, 168 insertions(+) create mode 100644 tests/fixtures/centos-7.7/mpstat-A-2-5.json create mode 100644 tests/fixtures/centos-7.7/mpstat-A-2-5.out create mode 100644 tests/fixtures/centos-7.7/mpstat-A.json create mode 100644 tests/fixtures/centos-7.7/mpstat-A.out create mode 100644 tests/fixtures/centos-7.7/mpstat.json create mode 100644 tests/fixtures/centos-7.7/mpstat.out create mode 100644 tests/test_mpstat.py diff --git a/tests/fixtures/centos-7.7/mpstat-A-2-5.json b/tests/fixtures/centos-7.7/mpstat-A-2-5.json new file mode 100644 index 00000000..4fb6a882 --- /dev/null +++ b/tests/fixtures/centos-7.7/mpstat-A-2-5.json @@ -0,0 +1 @@ +[{"cpu":"all","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:17 PM"},{"cpu":"0","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:17 PM"},{"cpu":"all","intr_s":55.5,"type":"interrupts","time":"03:16:17 PM"},{"cpu":"0","intr_s":54.0,"type":"interrupts","time":"03:16:17 PM"},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":1.0,"16_s":4.5,"17_s":0.0,"18_s":0.0,"19_s":1.5,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":44.0,"spu_s":0.0,"pmi_s":0.0,"iwi_s":4.5,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:16:17 PM"},{"cpu":"0","hi_s":0.0,"timer_s":35.0,"net_tx_s":0.5,"net_rx_s":1.5,"block_s":0.5,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":16.5,"type":"interrupts","time":"03:16:17 PM"},{"cpu":"all","percent_usr":0.5,"percent_nice":0.0,"percent_sys":0.0,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:19 PM"},{"cpu":"0","percent_usr":0.5,"percent_nice":0.0,"percent_sys":0.0,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:19 PM"},{"cpu":"all","intr_s":56.5,"type":"interrupts","time":"03:16:19 PM"},{"cpu":"0","intr_s":51.5,"type":"interrupts","time":"03:16:19 PM"},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":1.0,"16_s":5.0,"17_s":0.0,"18_s":0.0,"19_s":1.5,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":43.5,"spu_s":0.0,"pmi_s":0.0,"iwi_s":5.5,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:16:19 PM"},{"cpu":"0","hi_s":0.0,"timer_s":33.5,"net_tx_s":0.0,"net_rx_s":1.5,"block_s":0.5,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":16.0,"type":"interrupts","time":"03:16:19 PM"},{"cpu":"all","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:21 PM"},{"cpu":"0","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:21 PM"},{"cpu":"all","intr_s":59.0,"type":"interrupts","time":"03:16:21 PM"},{"cpu":"0","intr_s":55.0,"type":"interrupts","time":"03:16:21 PM"},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":0.0,"16_s":5.0,"17_s":0.0,"18_s":0.0,"19_s":1.5,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":47.0,"spu_s":0.0,"pmi_s":0.0,"iwi_s":6.5,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:16:21 PM"},{"cpu":"0","hi_s":0.0,"timer_s":35.5,"net_tx_s":0.5,"net_rx_s":1.5,"block_s":0.0,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":17.5,"type":"interrupts","time":"03:16:21 PM"},{"cpu":"all","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:23 PM"},{"cpu":"0","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:23 PM"},{"cpu":"all","intr_s":56.5,"type":"interrupts","time":"03:16:23 PM"},{"cpu":"0","intr_s":53.5,"type":"interrupts","time":"03:16:23 PM"},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":1.0,"16_s":5.0,"17_s":0.0,"18_s":0.0,"19_s":1.0,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":43.5,"spu_s":0.0,"pmi_s":0.0,"iwi_s":6.0,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:16:23 PM"},{"cpu":"0","hi_s":0.0,"timer_s":34.5,"net_tx_s":0.0,"net_rx_s":1.0,"block_s":0.5,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":17.5,"type":"interrupts","time":"03:16:23 PM"},{"cpu":"all","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.0,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":100.0,"type":"cpu","time":"03:16:25 PM"},{"cpu":"0","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.0,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":100.0,"type":"cpu","time":"03:16:25 PM"},{"cpu":"all","intr_s":56.28,"type":"interrupts","time":"03:16:25 PM"},{"cpu":"0","intr_s":50.75,"type":"interrupts","time":"03:16:25 PM"},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":1.01,"16_s":4.52,"17_s":1.01,"18_s":0.0,"19_s":1.51,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":43.22,"spu_s":0.0,"pmi_s":0.0,"iwi_s":5.03,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:16:25 PM"},{"cpu":"0","hi_s":0.0,"timer_s":32.16,"net_tx_s":0.0,"net_rx_s":1.51,"block_s":1.51,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":15.58,"type":"interrupts","time":"03:16:25 PM"},{"cpu":"all","percent_usr":0.1,"percent_nice":0.0,"percent_sys":0.3,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.6,"type":"cpu","average":true},{"cpu":"0","percent_usr":0.1,"percent_nice":0.0,"percent_sys":0.3,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.6,"type":"cpu","average":true},{"cpu":"all","intr_s":56.76,"type":"interrupts","average":true},{"cpu":"0","intr_s":52.95,"type":"interrupts","average":true},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":0.8,"16_s":4.8,"17_s":0.2,"18_s":0.0,"19_s":1.4,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":44.24,"spu_s":0.0,"pmi_s":0.0,"iwi_s":5.51,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","average":true},{"cpu":"0","hi_s":0.0,"timer_s":34.13,"net_tx_s":0.2,"net_rx_s":1.4,"block_s":0.6,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":16.62,"type":"interrupts","average":true}] diff --git a/tests/fixtures/centos-7.7/mpstat-A-2-5.out b/tests/fixtures/centos-7.7/mpstat-A-2-5.out new file mode 100644 index 00000000..0b64b884 --- /dev/null +++ b/tests/fixtures/centos-7.7/mpstat-A-2-5.out @@ -0,0 +1,86 @@ +Linux 3.10.0-1062.1.2.el7.x86_64 (localhost) 03/11/2022 _x86_64_ (1 CPU) + +03:16:15 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +03:16:17 PM all 0.00 0.00 0.50 0.00 0.00 0.00 0.00 0.00 0.00 99.50 +03:16:17 PM 0 0.00 0.00 0.50 0.00 0.00 0.00 0.00 0.00 0.00 99.50 + +03:16:15 PM CPU intr/s +03:16:17 PM all 55.50 +03:16:17 PM 0 54.00 + +03:16:15 PM CPU 0/s 1/s 4/s 8/s 9/s 12/s 14/s 15/s 16/s 17/s 18/s 19/s 24/s 25/s 26/s 27/s 28/s 29/s 30/s 31/s 32/s 33/s 34/s 35/s 36/s 37/s 38/s 39/s 40/s 41/s 42/s 43/s 44/s 45/s 46/s 47/s 48/s 49/s 50/s 51/s 52/s 53/s 54/s 55/s 56/s 57/s NMI/s LOC/s SPU/s PMI/s IWI/s RTR/s RES/s CAL/s TLB/s TRM/s THR/s DFR/s MCE/s MCP/s ERR/s MIS/s PIN/s NPI/s PIW/s +03:16:17 PM 0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 1.00 4.50 0.00 0.00 1.50 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 44.00 0.00 0.00 4.50 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 + +03:16:15 PM CPU HI/s TIMER/s NET_TX/s NET_RX/s BLOCK/s BLOCK_IOPOLL/s TASKLET/s SCHED/s HRTIMER/s RCU/s +03:16:17 PM 0 0.00 35.00 0.50 1.50 0.50 0.00 0.00 0.00 0.00 16.50 + +03:16:17 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +03:16:19 PM all 0.50 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 99.50 +03:16:19 PM 0 0.50 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 99.50 + +03:16:17 PM CPU intr/s +03:16:19 PM all 56.50 +03:16:19 PM 0 51.50 + +03:16:17 PM CPU 0/s 1/s 4/s 8/s 9/s 12/s 14/s 15/s 16/s 17/s 18/s 19/s 24/s 25/s 26/s 27/s 28/s 29/s 30/s 31/s 32/s 33/s 34/s 35/s 36/s 37/s 38/s 39/s 40/s 41/s 42/s 43/s 44/s 45/s 46/s 47/s 48/s 49/s 50/s 51/s 52/s 53/s 54/s 55/s 56/s 57/s NMI/s LOC/s SPU/s PMI/s IWI/s RTR/s RES/s CAL/s TLB/s TRM/s THR/s DFR/s MCE/s MCP/s ERR/s MIS/s PIN/s NPI/s PIW/s +03:16:19 PM 0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 1.00 5.00 0.00 0.00 1.50 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 43.50 0.00 0.00 5.50 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 + +03:16:17 PM CPU HI/s TIMER/s NET_TX/s NET_RX/s BLOCK/s BLOCK_IOPOLL/s TASKLET/s SCHED/s HRTIMER/s RCU/s +03:16:19 PM 0 0.00 33.50 0.00 1.50 0.50 0.00 0.00 0.00 0.00 16.00 + +03:16:19 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +03:16:21 PM all 0.00 0.00 0.50 0.00 0.00 0.00 0.00 0.00 0.00 99.50 +03:16:21 PM 0 0.00 0.00 0.50 0.00 0.00 0.00 0.00 0.00 0.00 99.50 + +03:16:19 PM CPU intr/s +03:16:21 PM all 59.00 +03:16:21 PM 0 55.00 + +03:16:19 PM CPU 0/s 1/s 4/s 8/s 9/s 12/s 14/s 15/s 16/s 17/s 18/s 19/s 24/s 25/s 26/s 27/s 28/s 29/s 30/s 31/s 32/s 33/s 34/s 35/s 36/s 37/s 38/s 39/s 40/s 41/s 42/s 43/s 44/s 45/s 46/s 47/s 48/s 49/s 50/s 51/s 52/s 53/s 54/s 55/s 56/s 57/s NMI/s LOC/s SPU/s PMI/s IWI/s RTR/s RES/s CAL/s TLB/s TRM/s THR/s DFR/s MCE/s MCP/s ERR/s MIS/s PIN/s NPI/s PIW/s +03:16:21 PM 0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 5.00 0.00 0.00 1.50 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 47.00 0.00 0.00 6.50 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 + +03:16:19 PM CPU HI/s TIMER/s NET_TX/s NET_RX/s BLOCK/s BLOCK_IOPOLL/s TASKLET/s SCHED/s HRTIMER/s RCU/s +03:16:21 PM 0 0.00 35.50 0.50 1.50 0.00 0.00 0.00 0.00 0.00 17.50 + +03:16:21 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +03:16:23 PM all 0.00 0.00 0.50 0.00 0.00 0.00 0.00 0.00 0.00 99.50 +03:16:23 PM 0 0.00 0.00 0.50 0.00 0.00 0.00 0.00 0.00 0.00 99.50 + +03:16:21 PM CPU intr/s +03:16:23 PM all 56.50 +03:16:23 PM 0 53.50 + +03:16:21 PM CPU 0/s 1/s 4/s 8/s 9/s 12/s 14/s 15/s 16/s 17/s 18/s 19/s 24/s 25/s 26/s 27/s 28/s 29/s 30/s 31/s 32/s 33/s 34/s 35/s 36/s 37/s 38/s 39/s 40/s 41/s 42/s 43/s 44/s 45/s 46/s 47/s 48/s 49/s 50/s 51/s 52/s 53/s 54/s 55/s 56/s 57/s NMI/s LOC/s SPU/s PMI/s IWI/s RTR/s RES/s CAL/s TLB/s TRM/s THR/s DFR/s MCE/s MCP/s ERR/s MIS/s PIN/s NPI/s PIW/s +03:16:23 PM 0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 1.00 5.00 0.00 0.00 1.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 43.50 0.00 0.00 6.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 + +03:16:21 PM CPU HI/s TIMER/s NET_TX/s NET_RX/s BLOCK/s BLOCK_IOPOLL/s TASKLET/s SCHED/s HRTIMER/s RCU/s +03:16:23 PM 0 0.00 34.50 0.00 1.00 0.50 0.00 0.00 0.00 0.00 17.50 + +03:16:23 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +03:16:25 PM all 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 100.00 +03:16:25 PM 0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 100.00 + +03:16:23 PM CPU intr/s +03:16:25 PM all 56.28 +03:16:25 PM 0 50.75 + +03:16:23 PM CPU 0/s 1/s 4/s 8/s 9/s 12/s 14/s 15/s 16/s 17/s 18/s 19/s 24/s 25/s 26/s 27/s 28/s 29/s 30/s 31/s 32/s 33/s 34/s 35/s 36/s 37/s 38/s 39/s 40/s 41/s 42/s 43/s 44/s 45/s 46/s 47/s 48/s 49/s 50/s 51/s 52/s 53/s 54/s 55/s 56/s 57/s NMI/s LOC/s SPU/s PMI/s IWI/s RTR/s RES/s CAL/s TLB/s TRM/s THR/s DFR/s MCE/s MCP/s ERR/s MIS/s PIN/s NPI/s PIW/s +03:16:25 PM 0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 1.01 4.52 1.01 0.00 1.51 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 43.22 0.00 0.00 5.03 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 + +03:16:23 PM CPU HI/s TIMER/s NET_TX/s NET_RX/s BLOCK/s BLOCK_IOPOLL/s TASKLET/s SCHED/s HRTIMER/s RCU/s +03:16:25 PM 0 0.00 32.16 0.00 1.51 1.51 0.00 0.00 0.00 0.00 15.58 + +Average: CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +Average: all 0.10 0.00 0.30 0.00 0.00 0.00 0.00 0.00 0.00 99.60 +Average: 0 0.10 0.00 0.30 0.00 0.00 0.00 0.00 0.00 0.00 99.60 + +Average: CPU intr/s +Average: all 56.76 +Average: 0 52.95 + +Average: CPU 0/s 1/s 4/s 8/s 9/s 12/s 14/s 15/s 16/s 17/s 18/s 19/s 24/s 25/s 26/s 27/s 28/s 29/s 30/s 31/s 32/s 33/s 34/s 35/s 36/s 37/s 38/s 39/s 40/s 41/s 42/s 43/s 44/s 45/s 46/s 47/s 48/s 49/s 50/s 51/s 52/s 53/s 54/s 55/s 56/s 57/s NMI/s LOC/s SPU/s PMI/s IWI/s RTR/s RES/s CAL/s TLB/s TRM/s THR/s DFR/s MCE/s MCP/s ERR/s MIS/s PIN/s NPI/s PIW/s +Average: 0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.80 4.80 0.20 0.00 1.40 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 44.24 0.00 0.00 5.51 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 + +Average: CPU HI/s TIMER/s NET_TX/s NET_RX/s BLOCK/s BLOCK_IOPOLL/s TASKLET/s SCHED/s HRTIMER/s RCU/s +Average: 0 0.00 34.13 0.20 1.40 0.60 0.00 0.00 0.00 0.00 16.62 + diff --git a/tests/fixtures/centos-7.7/mpstat-A.json b/tests/fixtures/centos-7.7/mpstat-A.json new file mode 100644 index 00000000..c685ff0f --- /dev/null +++ b/tests/fixtures/centos-7.7/mpstat-A.json @@ -0,0 +1 @@ +[{"cpu":"all","percent_usr":0.22,"percent_nice":0.0,"percent_sys":0.37,"percent_iowait":0.01,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.4,"type":"cpu","time":"03:15:06 PM"},{"cpu":"0","percent_usr":0.22,"percent_nice":0.0,"percent_sys":0.37,"percent_iowait":0.01,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.4,"type":"cpu","time":"03:15:06 PM"},{"cpu":"all","intr_s":37.61,"type":"interrupts","time":"03:15:06 PM"},{"cpu":"0","intr_s":33.15,"type":"interrupts","time":"03:15:06 PM"},{"cpu":"0","0_s":0.02,"1_s":0.02,"4_s":0.01,"8_s":0.0,"9_s":0.0,"12_s":0.01,"14_s":0.0,"15_s":1.0,"16_s":0.75,"17_s":1.14,"18_s":0.03,"19_s":0.69,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":32.69,"spu_s":0.0,"pmi_s":0.0,"iwi_s":1.24,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:15:06 PM"},{"cpu":"0","hi_s":0.0,"timer_s":21.26,"net_tx_s":0.2,"net_rx_s":0.69,"block_s":1.58,"block_iopoll_s":0.0,"tasklet_s":0.03,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":9.39,"type":"interrupts","time":"03:15:06 PM"}] diff --git a/tests/fixtures/centos-7.7/mpstat-A.out b/tests/fixtures/centos-7.7/mpstat-A.out new file mode 100644 index 00000000..5499bee0 --- /dev/null +++ b/tests/fixtures/centos-7.7/mpstat-A.out @@ -0,0 +1,15 @@ +Linux 3.10.0-1062.1.2.el7.x86_64 (localhost) 03/11/2022 _x86_64_ (1 CPU) + +03:15:06 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +03:15:06 PM all 0.22 0.00 0.37 0.01 0.00 0.00 0.00 0.00 0.00 99.40 +03:15:06 PM 0 0.22 0.00 0.37 0.01 0.00 0.00 0.00 0.00 0.00 99.40 + +03:15:06 PM CPU intr/s +03:15:06 PM all 37.61 +03:15:06 PM 0 33.15 + +03:15:06 PM CPU 0/s 1/s 4/s 8/s 9/s 12/s 14/s 15/s 16/s 17/s 18/s 19/s 24/s 25/s 26/s 27/s 28/s 29/s 30/s 31/s 32/s 33/s 34/s 35/s 36/s 37/s 38/s 39/s 40/s 41/s 42/s 43/s 44/s 45/s 46/s 47/s 48/s 49/s 50/s 51/s 52/s 53/s 54/s 55/s 56/s 57/s NMI/s LOC/s SPU/s PMI/s IWI/s RTR/s RES/s CAL/s TLB/s TRM/s THR/s DFR/s MCE/s MCP/s ERR/s MIS/s PIN/s NPI/s PIW/s +03:15:06 PM 0 0.02 0.02 0.01 0.00 0.00 0.01 0.00 1.00 0.75 1.14 0.03 0.69 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 32.69 0.00 0.00 1.24 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 + +03:15:06 PM CPU HI/s TIMER/s NET_TX/s NET_RX/s BLOCK/s BLOCK_IOPOLL/s TASKLET/s SCHED/s HRTIMER/s RCU/s +03:15:06 PM 0 0.00 21.26 0.20 0.69 1.58 0.00 0.03 0.00 0.00 9.39 diff --git a/tests/fixtures/centos-7.7/mpstat.json b/tests/fixtures/centos-7.7/mpstat.json new file mode 100644 index 00000000..f89c71d3 --- /dev/null +++ b/tests/fixtures/centos-7.7/mpstat.json @@ -0,0 +1 @@ +[{"cpu":"all","percent_usr":0.23,"percent_nice":0.0,"percent_sys":0.37,"percent_iowait":0.01,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.39,"type":"cpu","time":"03:14:20 PM"}] diff --git a/tests/fixtures/centos-7.7/mpstat.out b/tests/fixtures/centos-7.7/mpstat.out new file mode 100644 index 00000000..fbdb59ff --- /dev/null +++ b/tests/fixtures/centos-7.7/mpstat.out @@ -0,0 +1,5 @@ +Linux 3.10.0-1062.1.2.el7.x86_64 (localhost) 03/11/2022 _x86_64_ (1 CPU) + +03:14:20 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +03:14:20 PM all 0.23 0.00 0.37 0.01 0.00 0.00 0.00 0.00 0.00 99.39 + diff --git a/tests/test_mpstat.py b/tests/test_mpstat.py new file mode 100644 index 00000000..de22950e --- /dev/null +++ b/tests/test_mpstat.py @@ -0,0 +1,59 @@ +import os +import unittest +import json +import jc.parsers.mpstat + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def setUp(self): + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat_A = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A-2-5.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat_A_2_5 = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat_A_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A-2-5.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat_A_2_5_json = json.loads(f.read()) + + + def test_mpstat_nodata(self): + """ + Test 'mpstat' with no data + """ + self.assertEqual(jc.parsers.mpstat.parse('', quiet=True), []) + + def test_mpstat_centos_7_7(self): + """ + Test 'mpstat' on Centos 7.7 + """ + self.assertEqual(jc.parsers.mpstat.parse(self.centos_7_7_mpstat, quiet=True), self.centos_7_7_mpstat_json) + + def test_mpstat_A_centos_7_7(self): + """ + Test 'mpstat -A' on Centos 7.7 + """ + self.assertEqual(jc.parsers.mpstat.parse(self.centos_7_7_mpstat_A, quiet=True), self.centos_7_7_mpstat_A_json) + + def test_mpstat_A_2_5_centos_7_7(self): + """ + Test 'mpstat -A 2 5' on Centos 7.7 + """ + self.assertEqual(jc.parsers.mpstat.parse(self.centos_7_7_mpstat_A_2_5, quiet=True), self.centos_7_7_mpstat_A_2_5_json) + + +if __name__ == '__main__': + unittest.main() From c693c868cacbcd81f44fd383719723dd7a137c87 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 11 Mar 2022 15:59:38 -0800 Subject: [PATCH 34/87] add mpstat streaming parser --- README.md | 1 + docs/parsers/mpstat.md | 2 + docs/parsers/mpstat_s.md | 130 ++++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/mpstat.py | 2 + jc/parsers/mpstat_s.py | 211 +++++++++++++++++++++++++++++++++++++++ man/jc.1 | 5 + 7 files changed, 352 insertions(+) create mode 100644 docs/parsers/mpstat_s.md create mode 100644 jc/parsers/mpstat_s.py diff --git a/README.md b/README.md index 21bf10c6..46316b3d 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,7 @@ option. - `--lsusb` enables the `lsusb` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/lsusb)) - `--mount` enables the `mount` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/mount)) - `--mpstat` enables the `mpstat` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat)) +- `--mpstat-s` enables the `mpstat` command streaming parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat_s)) - `--netstat` enables the `netstat` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/netstat)) - `--nmcli` enables the `nmcli` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/nmcli)) - `--ntpq` enables the `ntpq -p` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ntpq)) diff --git a/docs/parsers/mpstat.md b/docs/parsers/mpstat.md index 588b7612..a6653e40 100644 --- a/docs/parsers/mpstat.md +++ b/docs/parsers/mpstat.md @@ -5,6 +5,8 @@ jc - JSON Convert `mpstat` command output parser +Note: Latest versions of `mpstat` support JSON output (v11.5.1+) + Usage (cli): $ mpstat | jc --mpstat diff --git a/docs/parsers/mpstat_s.md b/docs/parsers/mpstat_s.md new file mode 100644 index 00000000..25fe3b03 --- /dev/null +++ b/docs/parsers/mpstat_s.md @@ -0,0 +1,130 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.mpstat\_s + +jc - JSON Convert `mpstat` command output streaming parser + +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) + +Note: Latest versions of `mpstat` support JSON output (v11.5.1+) + +Usage (cli): + + $ mpstat | jc --mpstat-s + +Usage (module): + + import jc + + result = jc.parse('mpstat_s', mpstat_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "type": string, + "time": string, + "cpu": string, + "average": boolean, + "percent_usr": float, + "percent_nice": float, + "percent_sys": float, + "percent_iowait": float, + "percent_irq": float, + "percent_soft": float, + "percent_steal": float, + "percent_guest": float, + "percent_gnice": float, + "percent_idle": float, + "intr_s": float, + "_s": float, # is an integer + "nmi_s": float, + "loc_s": float, + "spu_s": float, + "pmi_s": float, + "iwi_s": float, + "rtr_s": float, + "res_s": float, + "cal_s": float, + "tlb_s": float, + "trm_s": float, + "thr_s": float, + "dfr_s": float, + "mce_s": float, + "mcp_s": float, + "err_s": float, + "mis_s": float, + "pin_s": float, + "npi_s": float, + "piw_s": float, + "hi_s": float, + "timer_s": float, + "net_tx_s": float, + "net_rx_s": float, + "block_s": float, + "block_iopoll_s": float, + "tasklet_s": float, + "sched_s": float, + "hrtimer_s": float, + "rcu_s": float, + + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + +Examples: + + $ mpstat -A | jc --mpstat-s + {"cpu":"all","percent_usr":0.22,"percent_nice":0.0,"percent_sys":...} + {"cpu":"0","percent_usr":0.22,"percent_nice":0.0,"percent_sys":0....} + {"cpu":"all","intr_s":37.61,"type":"interrupts","time":"03:15:06 PM"} + ... + + $ mpstat | jc --mpstat-s -r + {"cpu":"all","percent_usr":"0.22","percent_nice":"0.00","percent_...} + {"cpu":"0","percent_usr":"0.22","percent_nice":"0.00","percent_sy...} + {"cpu":"all","intr_s":"37.61","type":"interrupts","time":"03:15:06 PM"} + ... + + + +### parse + +```python +@add_jc_meta +def parse(data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] +``` + +Main text parsing generator function. Returns an iterator object. + +Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + +Yields: + + Dictionary. Raw or processed structured data. + +Returns: + + Iterator object (generator) + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index f9f03e3f..e2944fa4 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -60,6 +60,7 @@ parsers = [ 'lsusb', 'mount', 'mpstat', + 'mpstat-s', 'netstat', 'nmcli', 'ntpq', diff --git a/jc/parsers/mpstat.py b/jc/parsers/mpstat.py index e3f92a3e..93764713 100644 --- a/jc/parsers/mpstat.py +++ b/jc/parsers/mpstat.py @@ -1,5 +1,7 @@ """jc - JSON Convert `mpstat` command output parser +Note: Latest versions of `mpstat` support JSON output (v11.5.1+) + Usage (cli): $ mpstat | jc --mpstat diff --git a/jc/parsers/mpstat_s.py b/jc/parsers/mpstat_s.py new file mode 100644 index 00000000..9c5d7c0c --- /dev/null +++ b/jc/parsers/mpstat_s.py @@ -0,0 +1,211 @@ +"""jc - JSON Convert `mpstat` command output streaming parser + +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) + +Note: Latest versions of `mpstat` support JSON output (v11.5.1+) + +Usage (cli): + + $ mpstat | jc --mpstat-s + +Usage (module): + + import jc + + result = jc.parse('mpstat_s', mpstat_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "type": string, + "time": string, + "cpu": string, + "average": boolean, + "percent_usr": float, + "percent_nice": float, + "percent_sys": float, + "percent_iowait": float, + "percent_irq": float, + "percent_soft": float, + "percent_steal": float, + "percent_guest": float, + "percent_gnice": float, + "percent_idle": float, + "intr_s": float, + "_s": float, # is an integer + "nmi_s": float, + "loc_s": float, + "spu_s": float, + "pmi_s": float, + "iwi_s": float, + "rtr_s": float, + "res_s": float, + "cal_s": float, + "tlb_s": float, + "trm_s": float, + "thr_s": float, + "dfr_s": float, + "mce_s": float, + "mcp_s": float, + "err_s": float, + "mis_s": float, + "pin_s": float, + "npi_s": float, + "piw_s": float, + "hi_s": float, + "timer_s": float, + "net_tx_s": float, + "net_rx_s": float, + "block_s": float, + "block_iopoll_s": float, + "tasklet_s": float, + "sched_s": float, + "hrtimer_s": float, + "rcu_s": float, + + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + +Examples: + + $ mpstat -A | jc --mpstat-s + {"cpu":"all","percent_usr":0.22,"percent_nice":0.0,"percent_sys":...} + {"cpu":"0","percent_usr":0.22,"percent_nice":0.0,"percent_sys":0....} + {"cpu":"all","intr_s":37.61,"type":"interrupts","time":"03:15:06 PM"} + ... + + $ mpstat | jc --mpstat-s -r + {"cpu":"all","percent_usr":"0.22","percent_nice":"0.00","percent_...} + {"cpu":"0","percent_usr":"0.22","percent_nice":"0.00","percent_sy...} + {"cpu":"all","intr_s":"37.61","type":"interrupts","time":"03:15:06 PM"} + ... +""" +from typing import Dict, Iterable, Union +import jc.utils +from jc.parsers.universal import simple_table_parse +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) +from jc.exceptions import ParseError + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`mpstat` command streaming parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + streaming = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured data to conform to the schema. + """ + float_list = [ + "percent_usr", "percent_nice", "percent_sys", "percent_iowait", "percent_irq", + "percent_soft", "percent_steal", "percent_guest", "percent_gnice", "percent_idle", "intr_s", + "nmi_s", "loc_s", "spu_s", "pmi_s", "iwi_s", "rtr_s", "res_s", "cal_s", "tlb_s", "trm_s", + "thr_s", "dfr_s", "mce_s", "mcp_s", "err_s", "mis_s", "pin_s", "npi_s", "piw_s", "hi_s", + "timer_s", "net_tx_s", "net_rx_s", "block_s", "block_iopoll_s", "tasklet_s", "sched_s", + "hrtimer_s", "rcu_s" + ] + for key in proc_data: + if (key in float_list or (key[0].isdigit() and key.endswith('_s'))): + proc_data[key] = jc.utils.convert_to_float(proc_data[key]) + + return proc_data + + +@add_jc_meta +def parse( + data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False +) -> Union[Iterable[Dict], tuple]: + """ + Main text parsing generator function. Returns an iterator object. + + Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + + Yields: + + Dictionary. Raw or processed structured data. + + Returns: + + Iterator object (generator) + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + streaming_input_type_check(data) + + header_found: bool = False + + for line in data: + try: + streaming_line_input_type_check(line) + + # skip blank lines + if not line.strip(): + continue + + output_line: Dict = {} + + # check for header, normalize it, and fix the time column + if ' CPU ' in line: + header_found = True + if '%usr' in line: + stat_type = 'cpu' + else: + stat_type = 'interrupts' + + header_text: str = line.replace('/', '_')\ + .replace('%', 'percent_')\ + .lower() + header_start = line.find('CPU ') + header_text = header_text[header_start:] + continue + + # data line - pull time from beginning and then parse as a table + if header_found: + output_line = simple_table_parse([header_text, line[header_start:]])[0] + output_line['type'] = stat_type + item_time = line[:header_start].strip() + if 'Average:' not in item_time: + output_line['time'] = line[:header_start].strip() + else: + output_line['average'] = True + + if output_line: + yield output_line if raw else _process(output_line) + + except Exception as e: + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/man/jc.1 b/man/jc.1 index 40ce7249..820be574 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -257,6 +257,11 @@ Key/Value file parser \fB--mpstat\fP `mpstat` command parser +.TP +.B +\fB--mpstat-s\fP +`mpstat` command streaming parser + .TP .B \fB--netstat\fP From b257ce8c2fe6825fe435b4d04b4184091030db10 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 13 Mar 2022 12:25:50 -0700 Subject: [PATCH 35/87] add mpstat_s tests --- .../centos-7.7/mpstat-A-2-5-streaming.json | 1 + .../centos-7.7/mpstat-A-streaming.json | 1 + .../fixtures/centos-7.7/mpstat-streaming.json | 1 + tests/test_mpstat_s.py | 62 +++++++++++++++++++ 4 files changed, 65 insertions(+) create mode 100644 tests/fixtures/centos-7.7/mpstat-A-2-5-streaming.json create mode 100644 tests/fixtures/centos-7.7/mpstat-A-streaming.json create mode 100644 tests/fixtures/centos-7.7/mpstat-streaming.json create mode 100644 tests/test_mpstat_s.py diff --git a/tests/fixtures/centos-7.7/mpstat-A-2-5-streaming.json b/tests/fixtures/centos-7.7/mpstat-A-2-5-streaming.json new file mode 100644 index 00000000..4fb6a882 --- /dev/null +++ b/tests/fixtures/centos-7.7/mpstat-A-2-5-streaming.json @@ -0,0 +1 @@ +[{"cpu":"all","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:17 PM"},{"cpu":"0","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:17 PM"},{"cpu":"all","intr_s":55.5,"type":"interrupts","time":"03:16:17 PM"},{"cpu":"0","intr_s":54.0,"type":"interrupts","time":"03:16:17 PM"},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":1.0,"16_s":4.5,"17_s":0.0,"18_s":0.0,"19_s":1.5,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":44.0,"spu_s":0.0,"pmi_s":0.0,"iwi_s":4.5,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:16:17 PM"},{"cpu":"0","hi_s":0.0,"timer_s":35.0,"net_tx_s":0.5,"net_rx_s":1.5,"block_s":0.5,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":16.5,"type":"interrupts","time":"03:16:17 PM"},{"cpu":"all","percent_usr":0.5,"percent_nice":0.0,"percent_sys":0.0,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:19 PM"},{"cpu":"0","percent_usr":0.5,"percent_nice":0.0,"percent_sys":0.0,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:19 PM"},{"cpu":"all","intr_s":56.5,"type":"interrupts","time":"03:16:19 PM"},{"cpu":"0","intr_s":51.5,"type":"interrupts","time":"03:16:19 PM"},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":1.0,"16_s":5.0,"17_s":0.0,"18_s":0.0,"19_s":1.5,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":43.5,"spu_s":0.0,"pmi_s":0.0,"iwi_s":5.5,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:16:19 PM"},{"cpu":"0","hi_s":0.0,"timer_s":33.5,"net_tx_s":0.0,"net_rx_s":1.5,"block_s":0.5,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":16.0,"type":"interrupts","time":"03:16:19 PM"},{"cpu":"all","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:21 PM"},{"cpu":"0","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:21 PM"},{"cpu":"all","intr_s":59.0,"type":"interrupts","time":"03:16:21 PM"},{"cpu":"0","intr_s":55.0,"type":"interrupts","time":"03:16:21 PM"},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":0.0,"16_s":5.0,"17_s":0.0,"18_s":0.0,"19_s":1.5,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":47.0,"spu_s":0.0,"pmi_s":0.0,"iwi_s":6.5,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:16:21 PM"},{"cpu":"0","hi_s":0.0,"timer_s":35.5,"net_tx_s":0.5,"net_rx_s":1.5,"block_s":0.0,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":17.5,"type":"interrupts","time":"03:16:21 PM"},{"cpu":"all","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:23 PM"},{"cpu":"0","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.5,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.5,"type":"cpu","time":"03:16:23 PM"},{"cpu":"all","intr_s":56.5,"type":"interrupts","time":"03:16:23 PM"},{"cpu":"0","intr_s":53.5,"type":"interrupts","time":"03:16:23 PM"},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":1.0,"16_s":5.0,"17_s":0.0,"18_s":0.0,"19_s":1.0,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":43.5,"spu_s":0.0,"pmi_s":0.0,"iwi_s":6.0,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:16:23 PM"},{"cpu":"0","hi_s":0.0,"timer_s":34.5,"net_tx_s":0.0,"net_rx_s":1.0,"block_s":0.5,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":17.5,"type":"interrupts","time":"03:16:23 PM"},{"cpu":"all","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.0,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":100.0,"type":"cpu","time":"03:16:25 PM"},{"cpu":"0","percent_usr":0.0,"percent_nice":0.0,"percent_sys":0.0,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":100.0,"type":"cpu","time":"03:16:25 PM"},{"cpu":"all","intr_s":56.28,"type":"interrupts","time":"03:16:25 PM"},{"cpu":"0","intr_s":50.75,"type":"interrupts","time":"03:16:25 PM"},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":1.01,"16_s":4.52,"17_s":1.01,"18_s":0.0,"19_s":1.51,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":43.22,"spu_s":0.0,"pmi_s":0.0,"iwi_s":5.03,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:16:25 PM"},{"cpu":"0","hi_s":0.0,"timer_s":32.16,"net_tx_s":0.0,"net_rx_s":1.51,"block_s":1.51,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":15.58,"type":"interrupts","time":"03:16:25 PM"},{"cpu":"all","percent_usr":0.1,"percent_nice":0.0,"percent_sys":0.3,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.6,"type":"cpu","average":true},{"cpu":"0","percent_usr":0.1,"percent_nice":0.0,"percent_sys":0.3,"percent_iowait":0.0,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.6,"type":"cpu","average":true},{"cpu":"all","intr_s":56.76,"type":"interrupts","average":true},{"cpu":"0","intr_s":52.95,"type":"interrupts","average":true},{"cpu":"0","0_s":0.0,"1_s":0.0,"4_s":0.0,"8_s":0.0,"9_s":0.0,"12_s":0.0,"14_s":0.0,"15_s":0.8,"16_s":4.8,"17_s":0.2,"18_s":0.0,"19_s":1.4,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":44.24,"spu_s":0.0,"pmi_s":0.0,"iwi_s":5.51,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","average":true},{"cpu":"0","hi_s":0.0,"timer_s":34.13,"net_tx_s":0.2,"net_rx_s":1.4,"block_s":0.6,"block_iopoll_s":0.0,"tasklet_s":0.0,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":16.62,"type":"interrupts","average":true}] diff --git a/tests/fixtures/centos-7.7/mpstat-A-streaming.json b/tests/fixtures/centos-7.7/mpstat-A-streaming.json new file mode 100644 index 00000000..c685ff0f --- /dev/null +++ b/tests/fixtures/centos-7.7/mpstat-A-streaming.json @@ -0,0 +1 @@ +[{"cpu":"all","percent_usr":0.22,"percent_nice":0.0,"percent_sys":0.37,"percent_iowait":0.01,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.4,"type":"cpu","time":"03:15:06 PM"},{"cpu":"0","percent_usr":0.22,"percent_nice":0.0,"percent_sys":0.37,"percent_iowait":0.01,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.4,"type":"cpu","time":"03:15:06 PM"},{"cpu":"all","intr_s":37.61,"type":"interrupts","time":"03:15:06 PM"},{"cpu":"0","intr_s":33.15,"type":"interrupts","time":"03:15:06 PM"},{"cpu":"0","0_s":0.02,"1_s":0.02,"4_s":0.01,"8_s":0.0,"9_s":0.0,"12_s":0.01,"14_s":0.0,"15_s":1.0,"16_s":0.75,"17_s":1.14,"18_s":0.03,"19_s":0.69,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":0.0,"57_s":0.0,"nmi_s":0.0,"loc_s":32.69,"spu_s":0.0,"pmi_s":0.0,"iwi_s":1.24,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.0,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"03:15:06 PM"},{"cpu":"0","hi_s":0.0,"timer_s":21.26,"net_tx_s":0.2,"net_rx_s":0.69,"block_s":1.58,"block_iopoll_s":0.0,"tasklet_s":0.03,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":9.39,"type":"interrupts","time":"03:15:06 PM"}] diff --git a/tests/fixtures/centos-7.7/mpstat-streaming.json b/tests/fixtures/centos-7.7/mpstat-streaming.json new file mode 100644 index 00000000..f89c71d3 --- /dev/null +++ b/tests/fixtures/centos-7.7/mpstat-streaming.json @@ -0,0 +1 @@ +[{"cpu":"all","percent_usr":0.23,"percent_nice":0.0,"percent_sys":0.37,"percent_iowait":0.01,"percent_irq":0.0,"percent_soft":0.0,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":99.39,"type":"cpu","time":"03:14:20 PM"}] diff --git a/tests/test_mpstat_s.py b/tests/test_mpstat_s.py new file mode 100644 index 00000000..39de208d --- /dev/null +++ b/tests/test_mpstat_s.py @@ -0,0 +1,62 @@ +import os +import json +import unittest +import jc.parsers.mpstat_s + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +# To create streaming output use: +# $ cat mpstat.out | jc --mpstat-s | jello -c > mpstat-streaming.json + + +class MyTests(unittest.TestCase): + + def setUp(self): + pass + # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat_A = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A-2-5.out'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat_A_2_5 = f.read() + + # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-streaming.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat_streaming_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A-streaming.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat_A_streaming_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A-2-5-streaming.json'), 'r', encoding='utf-8') as f: + self.centos_7_7_mpstat_A_2_5_streaming_json = json.loads(f.read()) + + def test_mpstat_s_nodata(self): + """ + Test 'mpstat' with no data + """ + self.assertEqual(list(jc.parsers.mpstat_s.parse([], quiet=True)), []) + + def test_mpstat_s_centos_7_7(self): + """ + Test 'mpstat' on Centos 7.7 + """ + self.assertEqual(list(jc.parsers.mpstat_s.parse(self.centos_7_7_mpstat.splitlines(), quiet=True)), self.centos_7_7_mpstat_streaming_json) + + def test_mpstat_s_A_centos_7_7(self): + """ + Test 'mpstat -A' on Centos 7.7 + """ + self.assertEqual(list(jc.parsers.mpstat_s.parse(self.centos_7_7_mpstat_A.splitlines(), quiet=True)), self.centos_7_7_mpstat_A_streaming_json) + + def test_mpstat_s_A_2_5_centos_7_7(self): + """ + Test 'mpstat -A 2 5' on Centos 7.7 + """ + self.assertEqual(list(jc.parsers.mpstat_s.parse(self.centos_7_7_mpstat_A_2_5.splitlines(), quiet=True)), self.centos_7_7_mpstat_A_2_5_streaming_json) + + +if __name__ == '__main__': + unittest.main() From 77c667eec045daff5cbd062fe9d6e83f237727e5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 14 Mar 2022 09:43:18 -0700 Subject: [PATCH 36/87] ubuntu fixes --- docs/parsers/mpstat.md | 2 ++ docs/parsers/mpstat_s.md | 2 ++ jc/parsers/mpstat.py | 12 +++++++++--- jc/parsers/mpstat_s.py | 12 +++++++++--- .../ubuntu-18.04/mpstat-A-streaming.json | 1 + tests/fixtures/ubuntu-18.04/mpstat-A.json | 1 + tests/fixtures/ubuntu-18.04/mpstat-A.out | 19 +++++++++++++++++++ tests/test_mpstat.py | 12 ++++++++++++ tests/test_mpstat_s.py | 12 ++++++++++++ 9 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/ubuntu-18.04/mpstat-A-streaming.json create mode 100644 tests/fixtures/ubuntu-18.04/mpstat-A.json create mode 100644 tests/fixtures/ubuntu-18.04/mpstat-A.out diff --git a/docs/parsers/mpstat.md b/docs/parsers/mpstat.md index a6653e40..c16f5ab0 100644 --- a/docs/parsers/mpstat.md +++ b/docs/parsers/mpstat.md @@ -27,6 +27,7 @@ Schema: "type": string, "time": string, "cpu": string, + "node": string, "average": boolean, "percent_usr": float, "percent_nice": float, @@ -64,6 +65,7 @@ Schema: "net_tx_s": float, "net_rx_s": float, "block_s": float, + "irq_poll_s": float, "block_iopoll_s": float, "tasklet_s": float, "sched_s": float, diff --git a/docs/parsers/mpstat_s.md b/docs/parsers/mpstat_s.md index 25fe3b03..1762010e 100644 --- a/docs/parsers/mpstat_s.md +++ b/docs/parsers/mpstat_s.md @@ -28,6 +28,7 @@ Schema: "type": string, "time": string, "cpu": string, + "node": string, "average": boolean, "percent_usr": float, "percent_nice": float, @@ -65,6 +66,7 @@ Schema: "net_tx_s": float, "net_rx_s": float, "block_s": float, + "irq_poll_s": float, "block_iopoll_s": float, "tasklet_s": float, "sched_s": float, diff --git a/jc/parsers/mpstat.py b/jc/parsers/mpstat.py index 93764713..1e5116e3 100644 --- a/jc/parsers/mpstat.py +++ b/jc/parsers/mpstat.py @@ -22,6 +22,7 @@ Schema: "type": string, "time": string, "cpu": string, + "node": string, "average": boolean, "percent_usr": float, "percent_nice": float, @@ -59,6 +60,7 @@ Schema: "net_tx_s": float, "net_rx_s": float, "block_s": float, + "irq_poll_s": float, "block_iopoll_s": float, "tasklet_s": float, "sched_s": float, @@ -142,8 +144,8 @@ def _process(proc_data: List[Dict]) -> List[Dict]: "percent_soft", "percent_steal", "percent_guest", "percent_gnice", "percent_idle", "intr_s", "nmi_s", "loc_s", "spu_s", "pmi_s", "iwi_s", "rtr_s", "res_s", "cal_s", "tlb_s", "trm_s", "thr_s", "dfr_s", "mce_s", "mcp_s", "err_s", "mis_s", "pin_s", "npi_s", "piw_s", "hi_s", - "timer_s", "net_tx_s", "net_rx_s", "block_s", "block_iopoll_s", "tasklet_s", "sched_s", - "hrtimer_s", "rcu_s" + "timer_s", "net_tx_s", "net_rx_s", "block_s", "irq_poll_s", "block_iopoll_s", "tasklet_s", + "sched_s", "hrtimer_s", "rcu_s" ] for entry in proc_data: for key in entry: @@ -185,7 +187,7 @@ def parse( for line in filter(None, data.splitlines()): # check for header, normalize it, and fix the time column - if ' CPU ' in line: + if ' CPU ' in line or ' NODE ' in line: header_found = True if '%usr' in line: stat_type = 'cpu' @@ -196,6 +198,10 @@ def parse( .replace('%', 'percent_')\ .lower() header_start = line.find('CPU ') + + if header_start == -1: + header_start = line.find('NODE ') + header_text = header_text[header_start:] continue diff --git a/jc/parsers/mpstat_s.py b/jc/parsers/mpstat_s.py index 9c5d7c0c..6b192439 100644 --- a/jc/parsers/mpstat_s.py +++ b/jc/parsers/mpstat_s.py @@ -23,6 +23,7 @@ Schema: "type": string, "time": string, "cpu": string, + "node": string, "average": boolean, "percent_usr": float, "percent_nice": float, @@ -60,6 +61,7 @@ Schema: "net_tx_s": float, "net_rx_s": float, "block_s": float, + "irq_poll_s": float, "block_iopoll_s": float, "tasklet_s": float, "sched_s": float, @@ -127,8 +129,8 @@ def _process(proc_data: Dict) -> Dict: "percent_soft", "percent_steal", "percent_guest", "percent_gnice", "percent_idle", "intr_s", "nmi_s", "loc_s", "spu_s", "pmi_s", "iwi_s", "rtr_s", "res_s", "cal_s", "tlb_s", "trm_s", "thr_s", "dfr_s", "mce_s", "mcp_s", "err_s", "mis_s", "pin_s", "npi_s", "piw_s", "hi_s", - "timer_s", "net_tx_s", "net_rx_s", "block_s", "block_iopoll_s", "tasklet_s", "sched_s", - "hrtimer_s", "rcu_s" + "timer_s", "net_tx_s", "net_rx_s", "block_s", "irq_poll_s", "block_iopoll_s", "tasklet_s", + "sched_s", "hrtimer_s", "rcu_s" ] for key in proc_data: if (key in float_list or (key[0].isdigit() and key.endswith('_s'))): @@ -180,7 +182,7 @@ def parse( output_line: Dict = {} # check for header, normalize it, and fix the time column - if ' CPU ' in line: + if ' CPU ' in line or ' NODE ' in line: header_found = True if '%usr' in line: stat_type = 'cpu' @@ -191,6 +193,10 @@ def parse( .replace('%', 'percent_')\ .lower() header_start = line.find('CPU ') + + if header_start == -1: + header_start = line.find('NODE ') + header_text = header_text[header_start:] continue diff --git a/tests/fixtures/ubuntu-18.04/mpstat-A-streaming.json b/tests/fixtures/ubuntu-18.04/mpstat-A-streaming.json new file mode 100644 index 00000000..e781d158 --- /dev/null +++ b/tests/fixtures/ubuntu-18.04/mpstat-A-streaming.json @@ -0,0 +1 @@ +[{"cpu":"all","percent_usr":26.15,"percent_nice":8.92,"percent_sys":32.36,"percent_iowait":1.39,"percent_irq":0.0,"percent_soft":2.09,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":29.09,"type":"cpu","time":"08:50:55 AM"},{"cpu":"0","percent_usr":26.15,"percent_nice":8.92,"percent_sys":32.36,"percent_iowait":1.39,"percent_irq":0.0,"percent_soft":2.09,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":29.09,"type":"cpu","time":"08:50:55 AM"},{"node":"all","percent_usr":26.15,"percent_nice":8.92,"percent_sys":32.36,"percent_iowait":1.39,"percent_irq":0.0,"percent_soft":2.09,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":29.09,"type":"cpu","time":"08:50:55 AM"},{"node":"0","percent_usr":26.15,"percent_nice":8.92,"percent_sys":32.36,"percent_iowait":1.39,"percent_irq":0.0,"percent_soft":2.09,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":29.09,"type":"cpu","time":"08:50:55 AM"},{"cpu":"all","intr_s":1301.92,"type":"interrupts","time":"08:50:55 AM"},{"cpu":"0","intr_s":1973.0,"type":"interrupts","time":"08:50:55 AM"},{"cpu":"0","0_s":0.12,"1_s":0.69,"4_s":1.08,"6_s":0.06,"8_s":0.01,"9_s":0.0,"12_s":0.97,"14_s":0.0,"15_s":0.0,"16_s":0.0,"17_s":160.05,"18_s":2.56,"19_s":469.15,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":27.16,"57_s":0.47,"58_s":0.0,"nmi_s":0.0,"loc_s":639.59,"spu_s":0.0,"pmi_s":0.0,"iwi_s":0.0,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.01,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"08:50:55 AM"},{"cpu":"0","hi_s":0.01,"timer_s":189.73,"net_tx_s":0.23,"net_rx_s":470.58,"block_s":176.49,"irq_poll_s":0.0,"tasklet_s":1.73,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":1134.22,"type":"interrupts","time":"08:50:55 AM"}] diff --git a/tests/fixtures/ubuntu-18.04/mpstat-A.json b/tests/fixtures/ubuntu-18.04/mpstat-A.json new file mode 100644 index 00000000..e781d158 --- /dev/null +++ b/tests/fixtures/ubuntu-18.04/mpstat-A.json @@ -0,0 +1 @@ +[{"cpu":"all","percent_usr":26.15,"percent_nice":8.92,"percent_sys":32.36,"percent_iowait":1.39,"percent_irq":0.0,"percent_soft":2.09,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":29.09,"type":"cpu","time":"08:50:55 AM"},{"cpu":"0","percent_usr":26.15,"percent_nice":8.92,"percent_sys":32.36,"percent_iowait":1.39,"percent_irq":0.0,"percent_soft":2.09,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":29.09,"type":"cpu","time":"08:50:55 AM"},{"node":"all","percent_usr":26.15,"percent_nice":8.92,"percent_sys":32.36,"percent_iowait":1.39,"percent_irq":0.0,"percent_soft":2.09,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":29.09,"type":"cpu","time":"08:50:55 AM"},{"node":"0","percent_usr":26.15,"percent_nice":8.92,"percent_sys":32.36,"percent_iowait":1.39,"percent_irq":0.0,"percent_soft":2.09,"percent_steal":0.0,"percent_guest":0.0,"percent_gnice":0.0,"percent_idle":29.09,"type":"cpu","time":"08:50:55 AM"},{"cpu":"all","intr_s":1301.92,"type":"interrupts","time":"08:50:55 AM"},{"cpu":"0","intr_s":1973.0,"type":"interrupts","time":"08:50:55 AM"},{"cpu":"0","0_s":0.12,"1_s":0.69,"4_s":1.08,"6_s":0.06,"8_s":0.01,"9_s":0.0,"12_s":0.97,"14_s":0.0,"15_s":0.0,"16_s":0.0,"17_s":160.05,"18_s":2.56,"19_s":469.15,"24_s":0.0,"25_s":0.0,"26_s":0.0,"27_s":0.0,"28_s":0.0,"29_s":0.0,"30_s":0.0,"31_s":0.0,"32_s":0.0,"33_s":0.0,"34_s":0.0,"35_s":0.0,"36_s":0.0,"37_s":0.0,"38_s":0.0,"39_s":0.0,"40_s":0.0,"41_s":0.0,"42_s":0.0,"43_s":0.0,"44_s":0.0,"45_s":0.0,"46_s":0.0,"47_s":0.0,"48_s":0.0,"49_s":0.0,"50_s":0.0,"51_s":0.0,"52_s":0.0,"53_s":0.0,"54_s":0.0,"55_s":0.0,"56_s":27.16,"57_s":0.47,"58_s":0.0,"nmi_s":0.0,"loc_s":639.59,"spu_s":0.0,"pmi_s":0.0,"iwi_s":0.0,"rtr_s":0.0,"res_s":0.0,"cal_s":0.0,"tlb_s":0.0,"trm_s":0.0,"thr_s":0.0,"dfr_s":0.0,"mce_s":0.0,"mcp_s":0.01,"err_s":0.0,"mis_s":0.0,"pin_s":0.0,"npi_s":0.0,"piw_s":0.0,"type":"interrupts","time":"08:50:55 AM"},{"cpu":"0","hi_s":0.01,"timer_s":189.73,"net_tx_s":0.23,"net_rx_s":470.58,"block_s":176.49,"irq_poll_s":0.0,"tasklet_s":1.73,"sched_s":0.0,"hrtimer_s":0.0,"rcu_s":1134.22,"type":"interrupts","time":"08:50:55 AM"}] diff --git a/tests/fixtures/ubuntu-18.04/mpstat-A.out b/tests/fixtures/ubuntu-18.04/mpstat-A.out new file mode 100644 index 00000000..d8b36aa1 --- /dev/null +++ b/tests/fixtures/ubuntu-18.04/mpstat-A.out @@ -0,0 +1,19 @@ +Linux 4.15.0-169-generic (kbrazil-ubuntu) 03/14/2022 _x86_64_ (1 CPU) + +08:50:55 AM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +08:50:55 AM all 26.15 8.92 32.36 1.39 0.00 2.09 0.00 0.00 0.00 29.09 +08:50:55 AM 0 26.15 8.92 32.36 1.39 0.00 2.09 0.00 0.00 0.00 29.09 + +08:50:55 AM NODE %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +08:50:55 AM all 26.15 8.92 32.36 1.39 0.00 2.09 0.00 0.00 0.00 29.09 +08:50:55 AM 0 26.15 8.92 32.36 1.39 0.00 2.09 0.00 0.00 0.00 29.09 + +08:50:55 AM CPU intr/s +08:50:55 AM all 1301.92 +08:50:55 AM 0 1973.00 + +08:50:55 AM CPU 0/s 1/s 4/s 6/s 8/s 9/s 12/s 14/s 15/s 16/s 17/s 18/s 19/s 24/s 25/s 26/s 27/s 28/s 29/s 30/s 31/s 32/s 33/s 34/s 35/s 36/s 37/s 38/s 39/s 40/s 41/s 42/s 43/s 44/s 45/s 46/s 47/s 48/s 49/s 50/s 51/s 52/s 53/s 54/s 55/s 56/s 57/s 58/s NMI/s LOC/s SPU/s PMI/s IWI/s RTR/s RES/s CAL/s TLB/s TRM/s THR/s DFR/s MCE/s MCP/s ERR/s MIS/s PIN/s NPI/s PIW/s +08:50:55 AM 0 0.12 0.69 1.08 0.06 0.01 0.00 0.97 0.00 0.00 0.00 160.05 2.56 469.15 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 27.16 0.47 0.00 0.00 639.59 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.01 0.00 0.00 0.00 0.00 0.00 + +08:50:55 AM CPU HI/s TIMER/s NET_TX/s NET_RX/s BLOCK/s IRQ_POLL/s TASKLET/s SCHED/s HRTIMER/s RCU/s +08:50:55 AM 0 0.01 189.73 0.23 470.58 176.49 0.00 1.73 0.00 0.00 1134.22 diff --git a/tests/test_mpstat.py b/tests/test_mpstat.py index de22950e..389562e9 100644 --- a/tests/test_mpstat.py +++ b/tests/test_mpstat.py @@ -19,6 +19,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A-2-5.out'), 'r', encoding='utf-8') as f: self.centos_7_7_mpstat_A_2_5 = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/mpstat-A.out'), 'r', encoding='utf-8') as f: + self.ubuntu_18_4_mpstat_A = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat.json'), 'r', encoding='utf-8') as f: self.centos_7_7_mpstat_json = json.loads(f.read()) @@ -29,6 +32,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A-2-5.json'), 'r', encoding='utf-8') as f: self.centos_7_7_mpstat_A_2_5_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/mpstat-A.json'), 'r', encoding='utf-8') as f: + self.ubuntu_18_4_mpstat_A_json = json.loads(f.read()) + def test_mpstat_nodata(self): """ @@ -54,6 +60,12 @@ class MyTests(unittest.TestCase): """ self.assertEqual(jc.parsers.mpstat.parse(self.centos_7_7_mpstat_A_2_5, quiet=True), self.centos_7_7_mpstat_A_2_5_json) + def test_mpstat_A_ubuntu_18_4(self): + """ + Test 'mpstat -A' on Ubuntu 18.4 + """ + self.assertEqual(jc.parsers.mpstat.parse(self.ubuntu_18_4_mpstat_A, quiet=True), self.ubuntu_18_4_mpstat_A_json) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_mpstat_s.py b/tests/test_mpstat_s.py index 39de208d..8d097f39 100644 --- a/tests/test_mpstat_s.py +++ b/tests/test_mpstat_s.py @@ -23,6 +23,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A-2-5.out'), 'r', encoding='utf-8') as f: self.centos_7_7_mpstat_A_2_5 = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/mpstat-A.out'), 'r', encoding='utf-8') as f: + self.ubuntu_18_4_mpstat_A = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-streaming.json'), 'r', encoding='utf-8') as f: self.centos_7_7_mpstat_streaming_json = json.loads(f.read()) @@ -33,6 +36,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/mpstat-A-2-5-streaming.json'), 'r', encoding='utf-8') as f: self.centos_7_7_mpstat_A_2_5_streaming_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/mpstat-A-streaming.json'), 'r', encoding='utf-8') as f: + self.ubuntu_18_4_mpstat_A_streaming_json = json.loads(f.read()) + def test_mpstat_s_nodata(self): """ Test 'mpstat' with no data @@ -57,6 +63,12 @@ class MyTests(unittest.TestCase): """ self.assertEqual(list(jc.parsers.mpstat_s.parse(self.centos_7_7_mpstat_A_2_5.splitlines(), quiet=True)), self.centos_7_7_mpstat_A_2_5_streaming_json) + def test_mpstat_s_A_ubuntu_18_4(self): + """ + Test 'mpstat -A' on Ubuntu 18.4 + """ + self.assertEqual(list(jc.parsers.mpstat_s.parse(self.ubuntu_18_4_mpstat_A.splitlines(), quiet=True)), self.ubuntu_18_4_mpstat_A_streaming_json) + if __name__ == '__main__': unittest.main() From e5f5b2591d8ac3abeb9571688c7fb8de34d647f4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 14 Mar 2022 09:45:17 -0700 Subject: [PATCH 37/87] changelog update --- CHANGELOG | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 52d219d1..2c5f7e66 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,8 @@ jc changelog - Add pidstat command parser tested on linux - Add pidstat command streaming parser tested on linux - Add mpstat command parser tested on linux +- Add mpstat command streaming parser tested on linux +- Add ASCII table parser 20220305 v1.18.5 - Fix date parser to ensure AM/PM period string is always uppercase From 89e5919796f7da585285901dca461aae21336415 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 14 Mar 2022 09:56:50 -0700 Subject: [PATCH 38/87] add -A to second example --- docs/parsers/mpstat_s.md | 2 +- jc/parsers/mpstat_s.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parsers/mpstat_s.md b/docs/parsers/mpstat_s.md index 1762010e..66f3ef8b 100644 --- a/docs/parsers/mpstat_s.md +++ b/docs/parsers/mpstat_s.md @@ -89,7 +89,7 @@ Examples: {"cpu":"all","intr_s":37.61,"type":"interrupts","time":"03:15:06 PM"} ... - $ mpstat | jc --mpstat-s -r + $ mpstat -A | jc --mpstat-s -r {"cpu":"all","percent_usr":"0.22","percent_nice":"0.00","percent_...} {"cpu":"0","percent_usr":"0.22","percent_nice":"0.00","percent_sy...} {"cpu":"all","intr_s":"37.61","type":"interrupts","time":"03:15:06 PM"} diff --git a/jc/parsers/mpstat_s.py b/jc/parsers/mpstat_s.py index 6b192439..8eea4730 100644 --- a/jc/parsers/mpstat_s.py +++ b/jc/parsers/mpstat_s.py @@ -84,7 +84,7 @@ Examples: {"cpu":"all","intr_s":37.61,"type":"interrupts","time":"03:15:06 PM"} ... - $ mpstat | jc --mpstat-s -r + $ mpstat -A | jc --mpstat-s -r {"cpu":"all","percent_usr":"0.22","percent_nice":"0.00","percent_...} {"cpu":"0","percent_usr":"0.22","percent_nice":"0.00","percent_sy...} {"cpu":"all","intr_s":"37.61","type":"interrupts","time":"03:15:06 PM"} From f9a9062147256d0949457cc7fde838f87202524d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 14 Mar 2022 10:46:23 -0700 Subject: [PATCH 39/87] add documentation argument to parser_info and all_parser_info --- CHANGELOG | 1 + jc/__init__.py | 7 +++++-- jc/lib.py | 29 +++++++++++++++++++++-------- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 2c5f7e66..3b15aac3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,6 +6,7 @@ jc changelog - Add mpstat command parser tested on linux - Add mpstat command streaming parser tested on linux - Add ASCII table parser +- Add documentation option to parser_info() and all_parser_info() 20220305 v1.18.5 - Fix date parser to ensure AM/PM period string is always uppercase diff --git a/jc/__init__.py b/jc/__init__.py index 71053c42..3430c527 100644 --- a/jc/__init__.py +++ b/jc/__init__.py @@ -67,13 +67,16 @@ built-in parsers and local plugin parsers. ### parser_info - parser_info(parser_module_name: str) -> dict + parser_info( + parser_module_name: str, + documentation: bool = False + ) -> dict Get the metadata for a particular parser. ### all_parser_info - all_parser_info() -> list[dict] + all_parser_info(documentation: bool = False) -> list[dict] Get the metadata for all parsers. diff --git a/jc/lib.py b/jc/lib.py index e2944fa4..01a27218 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -1,7 +1,6 @@ """jc - JSON Convert JC lib module """ - import sys import os import re @@ -266,12 +265,18 @@ def streaming_parser_mod_list() -> List[str]: plist.append(_cliname_to_modname(p)) return plist -def parser_info(parser_mod_name: str) -> Dict: +def parser_info(parser_mod_name: str, documentation: bool = False) -> Dict: """ - Returns a dictionary that includes the module metadata. + Returns a dictionary that includes the parser module metadata. - This function will accept **module_name**, **cli-name**, and - **--argument-name** variants of the module name string. + Parameters: + + parser_mod_name: (string) name of the parser module. This + function will accept module_name, + cli-name, and --argument-name + variants of the module name. + + documentation: (boolean) include parser docstring if True """ # ensure parser_mod_name is a true module name and not a cli name parser_mod_name = _cliname_to_modname(parser_mod_name) @@ -291,13 +296,21 @@ def parser_info(parser_mod_name: str) -> Dict: if _modname_to_cliname(parser_mod_name) in local_parsers: info_dict['plugin'] = True + if documentation: + info_dict['documentation'] = parser_mod.__doc__ + return info_dict -def all_parser_info() -> List[Dict]: +def all_parser_info(documentation: bool = False) -> List[Dict]: """ - Returns a list of dictionaries that includes metadata for all modules. + Returns a list of dictionaries that includes metadata for all parser + modules. + + Parameters: + + documentation: (boolean) include parser docstrings if True """ - return [parser_info(p) for p in parsers] + return [parser_info(p, documentation=documentation) for p in parsers] def get_help(parser_mod_name: str) -> None: """ From 32e4d55e86c3e7d94488ec69f13a4f6400c4cef3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 14 Mar 2022 10:46:54 -0700 Subject: [PATCH 40/87] use parser_info for help_doc --- jc/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index 8bb3d846..ce6a4f7b 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -10,7 +10,7 @@ import signal import shlex import subprocess import json -from .lib import (__version__, all_parser_info, parsers, +from .lib import (__version__, parser_info, all_parser_info, parsers, _parser_argument, _get_parser, _parser_is_streaming) from . import utils from . import tracebackplus @@ -235,12 +235,12 @@ def help_doc(options): parser_name = parser_shortname(arg) if parser_name in parsers: - parser = _get_parser(arg) - compatible = ', '.join(parser.info.compatible) + p_info = parser_info(arg, documentation=True) + compatible = ', '.join(p_info['compatible']) doc_text = \ - f'{parser.__doc__}\n'\ + f'{p_info["documentation"]}\n'\ f'Compatibility: {compatible}\n\n'\ - f'Version {parser.info.version} by {parser.info.author} ({parser.info.author_email})\n' + f'Version {p_info["version"]} by {p_info["author"]} ({p_info["author_email"]})\n' return doc_text From 7dac2f8dc3baf619105e3c6ca4d3e45300b86287 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 14 Mar 2022 10:50:10 -0700 Subject: [PATCH 41/87] doc update --- docs/lib.md | 23 +++++++++++++++++------ docs/readme.md | 7 +++++-- man/jc.1 | 2 +- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docs/lib.md b/docs/lib.md index ae714f9e..a74fbc52 100644 --- a/docs/lib.md +++ b/docs/lib.md @@ -133,23 +133,34 @@ subset of `parser_mod_list()`. ### parser\_info ```python -def parser_info(parser_mod_name: str) -> Dict +def parser_info(parser_mod_name: str, documentation: bool = False) -> Dict ``` -Returns a dictionary that includes the module metadata. +Returns a dictionary that includes the parser module metadata. -This function will accept **module_name**, **cli-name**, and -**--argument-name** variants of the module name string. +Parameters: + + parser_mod_name: (string) name of the parser module. This + function will accept module_name, + cli-name, and --argument-name + variants of the module name. + + documentation: (boolean) include parser docstring if True ### all\_parser\_info ```python -def all_parser_info() -> List[Dict] +def all_parser_info(documentation: bool = False) -> List[Dict] ``` -Returns a list of dictionaries that includes metadata for all modules. +Returns a list of dictionaries that includes metadata for all parser +modules. + +Parameters: + + documentation: (boolean) include parser docstrings if True diff --git a/docs/readme.md b/docs/readme.md index d224b4d7..9d2385ba 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -71,13 +71,16 @@ built-in parsers and local plugin parsers. ### parser_info - parser_info(parser_module_name: str) -> dict + parser_info( + parser_module_name: str, + documentation: bool = False + ) -> dict Get the metadata for a particular parser. ### all_parser_info - all_parser_info() -> list[dict] + all_parser_info(documentation: bool = False) -> list[dict] Get the metadata for all parsers. diff --git a/man/jc.1 b/man/jc.1 index 820be574..d88db55f 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-03-11 1.18.6 "JSON Convert" +.TH jc 1 2022-03-14 1.18.6 "JSON Convert" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From ff9527a098a94c996641fb3fe2f905c8ae5e3063 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 14 Mar 2022 11:35:57 -0700 Subject: [PATCH 42/87] import cleanup and use all_parser_info in parsers_text --- jc/cli.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index ce6a4f7b..bc802f38 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -4,14 +4,13 @@ JC cli module import sys import os -import importlib import textwrap import signal import shlex import subprocess import json from .lib import (__version__, parser_info, all_parser_info, parsers, - _parser_argument, _get_parser, _parser_is_streaming) + _get_parser, _parser_is_streaming) from . import utils from . import tracebackplus from .exceptions import LibraryNotInstalled, ParseError @@ -155,17 +154,14 @@ def parser_shortname(parser_arg): def parsers_text(indent=0, pad=0): """Return the argument and description information from each parser""" ptext = '' - for parser in parsers: - parser_arg = _parser_argument(parser) - parser_mod = _get_parser(parser) - - if hasattr(parser_mod, 'info'): - parser_desc = parser_mod.info.description - padding = pad - len(parser_arg) - padding_char = ' ' - indent_text = padding_char * indent - padding_text = padding_char * padding - ptext += indent_text + parser_arg + padding_text + parser_desc + '\n' + padding_char = ' ' + for p in all_parser_info(): + parser_arg = p['argument'] + padding = pad - len(parser_arg) + parser_desc = p['description'] + indent_text = padding_char * indent + padding_text = padding_char * padding + ptext += indent_text + parser_arg + padding_text + parser_desc + '\n' return ptext From 9e80fd2b97b9bfe38fdaa6acdf2e8ed26d34b49d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 14 Mar 2022 12:17:18 -0700 Subject: [PATCH 43/87] fallback if info and doc items don't exist --- jc/cli.py | 21 +++++++++++---------- jc/lib.py | 5 ++++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index bc802f38..1db4adb9 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -156,9 +156,9 @@ def parsers_text(indent=0, pad=0): ptext = '' padding_char = ' ' for p in all_parser_info(): - parser_arg = p['argument'] + parser_arg = p.get('argument', 'UNKNOWN') padding = pad - len(parser_arg) - parser_desc = p['description'] + parser_desc = p.get('description', 'No description available.') indent_text = padding_char * indent padding_text = padding_char * padding ptext += indent_text + parser_arg + padding_text + parser_desc + '\n' @@ -232,11 +232,15 @@ def help_doc(options): if parser_name in parsers: p_info = parser_info(arg, documentation=True) - compatible = ', '.join(p_info['compatible']) + compatible = ', '.join(p_info.get('compatible', ['unknown'])) + documentation = p_info.get('documentation', 'No documentation available.') + version = p_info.get('version', 'unknown') + author = p_info.get('author', 'unknown') + author_email = p_info.get('author_email', 'unknown') doc_text = \ - f'{p_info["documentation"]}\n'\ + f'{documentation}\n'\ f'Compatibility: {compatible}\n\n'\ - f'Version {p_info["version"]} by {p_info["author"]} ({p_info["author_email"]})\n' + f'Version {version} by {author} ({author_email})\n' return doc_text @@ -314,12 +318,9 @@ def magic_parser(args): if len(args_given) == 0: return False, None, None, [] - magic_dict = {} - parser_info = about_jc()['parsers'] - # create a dictionary of magic_commands to their respective parsers. - for entry in parser_info: - # Update the dict with all of the magic commands for this parser, if they exist. + magic_dict = {} + for entry in all_parser_info(): magic_dict.update({mc: entry['argument'] for mc in entry.get('magic_commands', [])}) # find the command and parser diff --git a/jc/lib.py b/jc/lib.py index 01a27218..ce7aa017 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -297,7 +297,10 @@ def parser_info(parser_mod_name: str, documentation: bool = False) -> Dict: info_dict['plugin'] = True if documentation: - info_dict['documentation'] = parser_mod.__doc__ + docs = parser_mod.__doc__ + if not docs: + docs = 'No documentation available.\n' + info_dict['documentation'] = docs return info_dict From 3ab25d02f9dc248706aa1105e1766b2a84bc55a3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 14 Mar 2022 12:30:09 -0700 Subject: [PATCH 44/87] use _parser_is_streaming from lib --- jc/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/cli.py b/jc/cli.py index 1db4adb9..ae8949f9 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -553,7 +553,7 @@ def main(): raise streaming_msg = '' - if getattr(parser.info, 'streaming', None): + if _parser_is_streaming(parser): streaming_msg = 'Use the -qq option to ignore streaming parser errors.' utils.error_message([ From f804c9627fb71341429ca80aec2887e23e0277db Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 15 Mar 2022 16:04:58 -0700 Subject: [PATCH 45/87] copy input list so we don't mutate the caller's data --- jc/parsers/universal.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jc/parsers/universal.py b/jc/parsers/universal.py index e3b9fa62..8bc95000 100644 --- a/jc/parsers/universal.py +++ b/jc/parsers/universal.py @@ -90,6 +90,7 @@ def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]: List of Dictionaries """ + data = data.copy() output: List = [] header_text: str = data.pop(0) header_text = header_text + ' ' From c27bd5ff39f21c6377e661d4d7de1200d2fdd72f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 17 Mar 2022 16:24:18 -0700 Subject: [PATCH 46/87] pad lines in sparse_table_parse and use str.isspace() --- jc/parsers/universal.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/jc/parsers/universal.py b/jc/parsers/universal.py index 8bc95000..81aa14f0 100644 --- a/jc/parsers/universal.py +++ b/jc/parsers/universal.py @@ -1,8 +1,5 @@ """jc - JSON Convert universal parsers""" - - -import string -from typing import List, Dict +from typing import Iterable, List, Dict def simple_table_parse(data: List[str]) -> List[Dict]: @@ -47,7 +44,7 @@ def simple_table_parse(data: List[str]) -> List[Dict]: return raw_output -def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]: +def sparse_table_parse(data: Iterable[str], delim: str = '\u2063') -> List[Dict]: """ Parse tables with missing column data or with spaces in column data. Blank cells are converted to None in the resulting dictionary. Data @@ -69,16 +66,14 @@ def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]: Parameters: - data: (list) Text data to parse that has been split into lines - via .splitlines(). Item 0 must be the header row. - Any spaces in header names should be changed to - underscore '_'. You should also ensure headers are - lowercase by using .lower(). Do not change the - position of header names as the positions are used - to find the data. + data: (iter) An iterable of string lines (e.g. str.splitlines()) + Item 0 must be the header row. Any spaces in header + names should be changed to underscore '_'. You + should also ensure headers are lowercase by using + .lower(). Do not change the position of header + names as the positions are used to find the data. - Also, ensure there are no blank lines (list items) - in the data. + Also, ensure there are no blank line items. delim: (string) Delimiter to use. By default `u\\2063` (invisible separator) is used since it is unlikely @@ -90,7 +85,19 @@ def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]: List of Dictionaries """ - data = data.copy() + # cast iterable to a list. Also keeps from mutating the caller's list + data = list(data) + + # find the longest line and pad all lines with spaces to match + max_len = max([len(x) for x in data]) + + new_data = [] + for line in data: + new_data.append(line + ' ' * (max_len - len(line))) + + data = new_data + + # find header output: List = [] header_text: str = data.pop(0) header_text = header_text + ' ' @@ -123,7 +130,7 @@ def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]: h_end = h_spec['end'] # check if the location contains whitespace. if not # then move to the left until a space is found - while h_end > 0 and entry[h_end] not in string.whitespace: + while h_end > 0 and not entry[h_end].isspace(): h_end -= 1 # insert custom delimiter From 4a3a4e10df8a552a4db3510640695f02b166dc40 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Mar 2022 13:03:43 -0700 Subject: [PATCH 47/87] add asciitable and asciitable-m parsers --- jc/lib.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/jc/lib.py b/jc/lib.py index ce7aa017..bc6754bd 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -1,6 +1,4 @@ -"""jc - JSON Convert -JC lib module -""" +"""jc - JSON Convert lib module""" import sys import os import re @@ -15,6 +13,8 @@ parsers = [ 'airport', 'airport-s', 'arp', + 'asciitable', + 'asciitable-m', 'blkid', 'cksum', 'crontab', @@ -280,7 +280,6 @@ def parser_info(parser_mod_name: str, documentation: bool = False) -> Dict: """ # ensure parser_mod_name is a true module name and not a cli name parser_mod_name = _cliname_to_modname(parser_mod_name) - parser_mod = _get_parser(parser_mod_name) info_dict: Dict = {} From 5a248a8fc58952a5eab3d4aac025d7422ba82d08 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Mar 2022 13:05:57 -0700 Subject: [PATCH 48/87] add multiline asciitable parser --- jc/parsers/asciitable_m.py | 344 +++++++++++++++++++++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 jc/parsers/asciitable_m.py diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py new file mode 100644 index 00000000..6fb67f4d --- /dev/null +++ b/jc/parsers/asciitable_m.py @@ -0,0 +1,344 @@ +"""jc - JSON Convert `asciitable-m` parser + +This parser converts ASCII and Unicode text tables with multi-line rows. +Tables must have some sort of separator line between rows. + +For example: + + ╒══════════╤═════════╤════════╕ + │ foo │ bar baz │ fiz │ + │ │ │ buz │ + ╞══════════╪═════════╪════════╡ + │ good day │ 12345 │ │ + │ mate │ │ │ + ├──────────┼─────────┼────────┤ + │ hi there │ abc def │ 3.14 │ + │ │ │ │ + ╘══════════╧═════════╧════════╛ + +Usage (cli): + + $ cat table.txt | jc --asciitable-m + +Usage (module): + + import jc + result = jc.parse('asciitable_m', asciitable-string) + +Schema: + + [ + { + "asciitable-m": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ asciitable-m | jc --asciitable-m -p + [] + + $ asciitable-m | jc --asciitable-m -p -r + [] +""" +from os import sep +import re +from typing import Iterable, List, Dict, Tuple, Optional, Union, Generator +import jc.utils +from jc.exceptions import ParseError +from jc.parsers.universal import sparse_table_parse + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'multi-line ASCII and Unicode table parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + # remove newlines from values + # for item in proc_data: + # for k, v in item.items(): + # item[k] = v.replace('\n', '') + + return proc_data + + +def _remove_ansi(string: str) -> str: + ansi_escape =re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]') + return ansi_escape.sub('', string) + + +def _lstrip(string: str) -> str: + """find the leftmost non-whitespace character and lstrip to that index""" + lstrip_list = [x for x in string.splitlines() if not len(x.strip()) == 0] + start_points = (len(x) - len(x.lstrip()) for x in lstrip_list) + min_point = min(start_points) + new_lstrip_list = (x[min_point:] for x in lstrip_list) + return '\n'.join(new_lstrip_list) + + +def _table_sniff(string: str) -> str: + """Find the table-type via heuristics""" + # pretty tables + for line in string.splitlines(): + line = line.strip() + if line.startswith('╞═') and line.endswith('═╡')\ + or line.startswith('├─') and line.endswith('─┤')\ + or line.startswith('+=') and line.endswith('=+')\ + or line.startswith('+-') and line.endswith('-+'): + return 'pretty' + + # markdown tables + second_line = string.splitlines()[1] + if second_line.startswith('|-') and second_line.endswith('-|'): + return 'markdown' + + # simple tables + return 'simple' + + +def _pretty_set_separators(table_lines: Iterable, separator: str) -> Generator[str, None, None]: + """Return a generator that yields rows standardized separators""" + for line in table_lines: + strip_line = line.strip() + + # skip any blanks + if not strip_line: + continue + + # yield row separators as a sentinel string + if strip_line.startswith('╒═') and strip_line.endswith('═╕')\ + or strip_line.startswith('╞═') and strip_line.endswith('═╡')\ + or strip_line.startswith('╘═') and strip_line.endswith('═╛')\ + or strip_line.startswith('┌─') and strip_line.endswith('─┐')\ + or strip_line.startswith('├─') and strip_line.endswith('─┤')\ + or strip_line.startswith('└─') and strip_line.endswith('─┘')\ + or strip_line.startswith('+=') and strip_line.endswith('=+')\ + or strip_line.startswith('+-') and strip_line.endswith('-+'): + yield separator + continue + + # remove the table column separator characters and yield the line + line = line.replace('|', ' ').replace('│', ' ') + yield line + + +def _pretty_normalize_rows(table_lines: Iterable, + separator: str, + data_separator: str) -> Generator[str, None, None]: + """ + Return a generator that yields header and data rows with different separators. + Also removes spaces from headers. + """ + header_found = False + data_found = False + + # Removes initial table lines, finds the header row(s) and separates + # the header from the data rows with different separator characters. + for i in table_lines: + if separator in i and not header_found and not data_found: + # top table frame + continue + if not separator in i and not header_found and not data_found: + header_found = True + # first header data found + # remove spaces from header + i = re.sub(r'\b \b', '_', i) + yield i + continue + if not separator in i and header_found and not data_found: + # subsequent header data found + # remove spaces from header + i = re.sub(r'\b \b', '_', i) + yield i + continue + if separator in i and header_found and not data_found: + data_found = True + # table separator found - this is a header separator + yield separator + continue + if not separator in i and header_found and data_found: + # subsequent data row found + yield i + continue + if separator in i and header_found and data_found: + # table separator found - this is a data separator + yield data_separator + continue + + +def _pretty_remove_header_rows(table: List[Dict], sep: str, data_sep: str) -> List[Optional[Dict]]: + """return a table with only data rows.""" + # create a new list of row objects with new key names + data_obj_list: List[Optional[Dict]] = [] + sep_found = False + data_sep_found = False + for obj in table: + #skip to data + for v in obj.values(): + if not sep_found and not str(v).strip() == sep: + continue + if not sep_found and str(v).strip() == sep: + sep_found = True + continue + # append data row objects or None for separators + if sep_found: + for k, v in obj.items(): + if str(v).strip() == data_sep: + data_sep_found = True + break + else: + data_sep_found = False + if data_sep_found: + data_obj_list.append(None) + else: + data_obj_list.append(obj) + + # remove first item, which is a separator + return data_obj_list[1:] + + +def _pretty_map_new_keynames(table: List[Dict], sep: str) -> Dict: + """ + returns a dict of old keyname to new keyname mappings by consolidating + multiline keynames from the input list of dictionaries. + """ + # first get all header objects to find full keynames. Stop when data rows are found. + header_obj_list = [] + sep_found = False + for obj in table: + for v in obj.values(): + if str(v).strip() == sep: + sep_found = True + break + if sep_found: + break + header_obj_list.append(obj) + + if not header_obj_list: + header_obj_list = [{key: None for key in table[0]}] + + # create an old-key to new-key name mapping dict + new_keynames_dict = dict.fromkeys([key for key in header_obj_list[0]], '') + for item in new_keynames_dict: + new_keynames_dict[item] = item + for obj in header_obj_list: + for k, v in obj.items(): + if v: + new_keynames_dict[k] = new_keynames_dict[k] + '_' + v + + # normalize keynames so they are lowercase, no spaces, and no redundat '_'s + for k, v in new_keynames_dict.items(): + new_keynames_dict[k] = v.replace(' ', '_').lower() + new_keynames_dict[k] = re.sub(r'__+', '_', v) + + return new_keynames_dict + + +def _pretty_rename_keys(table: List, new_keynames: Dict) -> List[Optional[Dict]]: + """rename all of the keys in the table based on the new_keynames mapping""" + renamed_key_table: List[Optional[Dict]] = [] + for item in table: + if item: + renamed_key_table.append({new_keynames[k]:v for k, v in item.items()}) + else: + renamed_key_table.append(None) + + return renamed_key_table + + +def _pretty_consolidate_rows(table: List) -> List[Dict]: + """go through all data objects and combine values between data separators""" + consolidated_rows = [] + current_obj = dict.fromkeys([key for key in table[0]], '') + for item in table: + if not item: + consolidated_rows.append(current_obj) + current_obj = dict.fromkeys([key for key in table[0]], '') + continue + else: + for k, v in item.items(): + if v: + if not current_obj[k]: + current_obj[k] = v + else: + current_obj[k] = current_obj[k] + '\n' + v + + return consolidated_rows + + +def _parse_pretty(string: str) -> List: + string_lines = string.splitlines() + sep = '~~~' + data_sep = '===' + separator = ' ' + sep + ' ' + data_separator = ' ' + data_sep + ' ' + + clean_gen = _pretty_set_separators(string_lines, separator) + normalized_gen = _pretty_normalize_rows(clean_gen, separator, data_separator) + raw_table = sparse_table_parse(normalized_gen) + new_keynames_dict = _pretty_map_new_keynames(raw_table, sep) + data_table = _pretty_remove_header_rows(raw_table, sep, data_sep) + table_with_renamed_keys = _pretty_rename_keys(data_table, new_keynames_dict) + final_table = _pretty_consolidate_rows(table_with_renamed_keys) + + return final_table + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + table_type = 'unknown' + + if jc.utils.has_data(data): + data = _remove_ansi(data) + data = _lstrip(data) + table_type = _table_sniff(data) + + if table_type == 'pretty': + raw_output = _parse_pretty(data) + elif table_type == 'markdown': + raise ParseError('Only "pretty" tables supported with multiline. "markdown" table detected. Please try the "asciitable" parser.') + else: + raise ParseError('Only "pretty" tables supported with multiline. "simple" table detected. Please try the "asciitable" parser.') + + return raw_output if raw else _process(raw_output) From 70fafbf3f80f35c5db2c306978b2a488adad40bb Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Mar 2022 14:19:00 -0700 Subject: [PATCH 49/87] remove asciitable so tests pass for now --- jc/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/lib.py b/jc/lib.py index bc6754bd..fd608421 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -13,7 +13,7 @@ parsers = [ 'airport', 'airport-s', 'arp', - 'asciitable', + # 'asciitable', 'asciitable-m', 'blkid', 'cksum', From ff0f794b01395ed9b65fac75a6b07f0772d45a2d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Mar 2022 16:53:23 -0700 Subject: [PATCH 50/87] working --- jc/parsers/asciitable_m.py | 42 +++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 6fb67f4d..3bbfd12d 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -45,10 +45,9 @@ Examples: """ from os import sep import re -from typing import Iterable, List, Dict, Tuple, Optional, Union, Generator +from typing import Iterable, List, Dict, Optional, Generator import jc.utils from jc.exceptions import ParseError -from jc.parsers.universal import sparse_table_parse class info(): @@ -139,7 +138,7 @@ def _pretty_set_separators(table_lines: Iterable, separator: str) -> Generator[s continue # remove the table column separator characters and yield the line - line = line.replace('|', ' ').replace('│', ' ') + # line = line.replace('|', ' ').replace('│', ' ') yield line @@ -187,6 +186,29 @@ def _pretty_normalize_rows(table_lines: Iterable, continue +def _pretty_table_parse(table: Iterable) -> List[Dict]: + temp_table = [] + for line in table: + # normalize separator + line = line.replace('│', '|') + + # remove first separator if it is the first char in the line + if line[0] == '|': + line = line.replace('|', ' ', 1) + + # remove last separator if it is the last char in the line + if line[-1] == '|': + line = line[::-1].replace('|', ' ', 1)[::-1] + + temp_table.append([x.strip() for x in line.split('|')]) + + headers = temp_table[0] + raw_data = temp_table[1:] + result = [dict.fromkeys(headers, None)] + result.extend([dict(zip(headers, r)) for r in raw_data]) + return result + + def _pretty_remove_header_rows(table: List[Dict], sep: str, data_sep: str) -> List[Optional[Dict]]: """return a table with only data rows.""" # create a new list of row objects with new key names @@ -294,13 +316,13 @@ def _parse_pretty(string: str) -> List: separator = ' ' + sep + ' ' data_separator = ' ' + data_sep + ' ' - clean_gen = _pretty_set_separators(string_lines, separator) - normalized_gen = _pretty_normalize_rows(clean_gen, separator, data_separator) - raw_table = sparse_table_parse(normalized_gen) - new_keynames_dict = _pretty_map_new_keynames(raw_table, sep) - data_table = _pretty_remove_header_rows(raw_table, sep, data_sep) - table_with_renamed_keys = _pretty_rename_keys(data_table, new_keynames_dict) - final_table = _pretty_consolidate_rows(table_with_renamed_keys) + clean: Generator = _pretty_set_separators(string_lines, separator) + normalized: Generator = _pretty_normalize_rows(clean, separator, data_separator) + raw_table: List[Dict] = _pretty_table_parse(normalized) # was sparse_table_parse() + new_keynames: Dict = _pretty_map_new_keynames(raw_table, sep) + data_table: List[Optional[Dict]] = _pretty_remove_header_rows(raw_table, sep, data_sep) + table_with_renamed_keys: List[Optional[Dict]] = _pretty_rename_keys(data_table, new_keynames) + final_table: List[Dict] = _pretty_consolidate_rows(table_with_renamed_keys) return final_table From ceacec0f462bd7c2d9e92ddb9029ccd71aef481c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 18 Mar 2022 16:57:47 -0700 Subject: [PATCH 51/87] remove errant os sep import --- jc/parsers/asciitable_m.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 3bbfd12d..c76f54d1 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -43,7 +43,6 @@ Examples: $ asciitable-m | jc --asciitable-m -p -r [] """ -from os import sep import re from typing import Iterable, List, Dict, Optional, Generator import jc.utils From b3a2886fd0af0cd06890f37f1edfc6890bb83e6c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 20 Mar 2022 10:12:29 -0700 Subject: [PATCH 52/87] formatting --- docs/lib.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/lib.md b/docs/lib.md index a74fbc52..85c3122a 100644 --- a/docs/lib.md +++ b/docs/lib.md @@ -14,8 +14,7 @@ # jc.lib -jc - JSON Convert -JC lib module +jc - JSON Convert lib module From 9ecbdb09162da4d563799aa93d422db85b2c324b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 20 Mar 2022 10:16:29 -0700 Subject: [PATCH 53/87] use generator instead of iterable in function return annotation --- docs/parsers/mpstat_s.md | 10 ++++++---- docs/parsers/pidstat_s.md | 10 ++++++---- docs/parsers/rsync_s.md | 10 ++++++---- docs/parsers/universal.md | 19 +++++++++---------- jc/parsers/foo_s.py | 4 ++-- jc/parsers/mpstat_s.py | 4 ++-- jc/parsers/pidstat_s.py | 4 ++-- jc/parsers/rsync_s.py | 4 ++-- 8 files changed, 35 insertions(+), 30 deletions(-) diff --git a/docs/parsers/mpstat_s.md b/docs/parsers/mpstat_s.md index 66f3ef8b..51aceb14 100644 --- a/docs/parsers/mpstat_s.md +++ b/docs/parsers/mpstat_s.md @@ -101,10 +101,12 @@ Examples: ```python @add_jc_meta -def parse(data: Iterable[str], - raw: bool = False, - quiet: bool = False, - ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] +def parse( + data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False +) -> Union[Generator[Dict, None, None], tuple] ``` Main text parsing generator function. Returns an iterator object. diff --git a/docs/parsers/pidstat_s.md b/docs/parsers/pidstat_s.md index d94feebb..e8fc7872 100644 --- a/docs/parsers/pidstat_s.md +++ b/docs/parsers/pidstat_s.md @@ -83,10 +83,12 @@ Examples: ```python @add_jc_meta -def parse(data: Iterable[str], - raw: bool = False, - quiet: bool = False, - ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] +def parse( + data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False +) -> Union[Generator[Dict, None, None], tuple] ``` Main text parsing generator function. Returns an iterator object. diff --git a/docs/parsers/rsync_s.md b/docs/parsers/rsync_s.md index a77a2a2e..d78ad24c 100644 --- a/docs/parsers/rsync_s.md +++ b/docs/parsers/rsync_s.md @@ -90,10 +90,12 @@ Examples: ```python @add_jc_meta -def parse(data: Iterable[str], - raw: bool = False, - quiet: bool = False, - ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] +def parse( + data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False +) -> Union[Generator[Dict, None, None], tuple] ``` Main text parsing generator function. Returns an iterator object. diff --git a/docs/parsers/universal.md b/docs/parsers/universal.md index 2a01569f..40889af5 100644 --- a/docs/parsers/universal.md +++ b/docs/parsers/universal.md @@ -54,7 +54,8 @@ Returns: ### sparse\_table\_parse ```python -def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict] +def sparse_table_parse(data: Iterable[str], + delim: str = '\u2063') -> List[Dict] ``` Parse tables with missing column data or with spaces in column data. @@ -77,16 +78,14 @@ Example Table: Parameters: - data: (list) Text data to parse that has been split into lines - via .splitlines(). Item 0 must be the header row. - Any spaces in header names should be changed to - underscore '_'. You should also ensure headers are - lowercase by using .lower(). Do not change the - position of header names as the positions are used - to find the data. + data: (iter) An iterable of string lines (e.g. str.splitlines()) + Item 0 must be the header row. Any spaces in header + names should be changed to underscore '_'. You + should also ensure headers are lowercase by using + .lower(). Do not change the position of header + names as the positions are used to find the data. - Also, ensure there are no blank lines (list items) - in the data. + Also, ensure there are no blank line items. delim: (string) Delimiter to use. By default `u\\2063` (invisible separator) is used since it is unlikely diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index 229ac345..496cd005 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -40,7 +40,7 @@ Examples: {example output} ... """ -from typing import Dict, Iterable, Union +from typing import Dict, Iterable, Generator, Union import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -90,7 +90,7 @@ def parse( raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Union[Iterable[Dict], tuple]: +) -> Union[Generator[Dict, None, None], tuple]: """ Main text parsing generator function. Returns an iterator object. diff --git a/jc/parsers/mpstat_s.py b/jc/parsers/mpstat_s.py index 8eea4730..e06f91b6 100644 --- a/jc/parsers/mpstat_s.py +++ b/jc/parsers/mpstat_s.py @@ -90,7 +90,7 @@ Examples: {"cpu":"all","intr_s":"37.61","type":"interrupts","time":"03:15:06 PM"} ... """ -from typing import Dict, Iterable, Union +from typing import Dict, Iterable, Generator, Union import jc.utils from jc.parsers.universal import simple_table_parse from jc.streaming import ( @@ -145,7 +145,7 @@ def parse( raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Union[Iterable[Dict], tuple]: +) -> Union[Generator[Dict, None, None], tuple]: """ Main text parsing generator function. Returns an iterator object. diff --git a/jc/parsers/pidstat_s.py b/jc/parsers/pidstat_s.py index 3a7b7b62..d9b688bd 100644 --- a/jc/parsers/pidstat_s.py +++ b/jc/parsers/pidstat_s.py @@ -72,7 +72,7 @@ Examples: {"time":"1646859134","uid":"0","pid":"9","percent_usr":"0.00","perc...} ... """ -from typing import Dict, Iterable, Union +from typing import Dict, Iterable, Generator, Union import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -126,7 +126,7 @@ def parse( raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Union[Iterable[Dict], tuple]: +) -> Union[Generator[Dict, None, None], tuple]: """ Main text parsing generator function. Returns an iterator object. diff --git a/jc/parsers/rsync_s.py b/jc/parsers/rsync_s.py index 4ecfe9d2..f6ecddd6 100644 --- a/jc/parsers/rsync_s.py +++ b/jc/parsers/rsync_s.py @@ -80,7 +80,7 @@ Examples: ... """ import re -from typing import Dict, Iterable, Union +from typing import Dict, Iterable, Generator, Union import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -139,7 +139,7 @@ def parse( raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Union[Iterable[Dict], tuple]: +) -> Union[Generator[Dict, None, None], tuple]: """ Main text parsing generator function. Returns an iterator object. From 51ae5ebcac7263ef2571c11975b275eff352bbc5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 21 Mar 2022 13:06:34 -0700 Subject: [PATCH 54/87] new streamlined parser --- jc/parsers/asciitable_m.py | 335 +++++++++++++++++-------------------- 1 file changed, 154 insertions(+), 181 deletions(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index c76f54d1..5af62101 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -44,7 +44,7 @@ Examples: [] """ import re -from typing import Iterable, List, Dict, Optional, Generator +from typing import Iterable, Tuple, List, Dict import jc.utils from jc.exceptions import ParseError @@ -73,11 +73,6 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - # remove newlines from values - # for item in proc_data: - # for k, v in item.items(): - # item[k] = v.replace('\n', '') - return proc_data @@ -115,213 +110,191 @@ def _table_sniff(string: str) -> str: return 'simple' -def _pretty_set_separators(table_lines: Iterable, separator: str) -> Generator[str, None, None]: - """Return a generator that yields rows standardized separators""" - for line in table_lines: - strip_line = line.strip() - - # skip any blanks - if not strip_line: - continue - - # yield row separators as a sentinel string - if strip_line.startswith('╒═') and strip_line.endswith('═╕')\ - or strip_line.startswith('╞═') and strip_line.endswith('═╡')\ - or strip_line.startswith('╘═') and strip_line.endswith('═╛')\ - or strip_line.startswith('┌─') and strip_line.endswith('─┐')\ - or strip_line.startswith('├─') and strip_line.endswith('─┤')\ - or strip_line.startswith('└─') and strip_line.endswith('─┘')\ - or strip_line.startswith('+=') and strip_line.endswith('=+')\ - or strip_line.startswith('+-') and strip_line.endswith('-+'): - yield separator - continue - - # remove the table column separator characters and yield the line - # line = line.replace('|', ' ').replace('│', ' ') - yield line +def _is_separator(line: str) -> bool: + """Returns true if a table separator line is found""" + strip_line = line.strip() + if strip_line.startswith('╒═') and strip_line.endswith('═╕')\ + or strip_line.startswith('╞═') and strip_line.endswith('═╡')\ + or strip_line.startswith('╘═') and strip_line.endswith('═╛')\ + or strip_line.startswith('┌─') and strip_line.endswith('─┐')\ + or strip_line.startswith('├─') and strip_line.endswith('─┤')\ + or strip_line.startswith('└─') and strip_line.endswith('─┘')\ + or strip_line.startswith('+=') and strip_line.endswith('=+')\ + or strip_line.startswith('+-') and strip_line.endswith('-+'): + return True + return False -def _pretty_normalize_rows(table_lines: Iterable, - separator: str, - data_separator: str) -> Generator[str, None, None]: +def _snake_case(line: str) -> str: + """replace spaces between words with an underscores and set to lowercase""" + return re.sub(r'\b \b', '_', line).lower() + + +def _fixup_separators(line: str) -> str: + """Normalize separators, and remove first and last separators""" + # normalize separator + line = line.replace('│', '|') + + # remove first separator if it is the first char in the line + if line[0] == '|': + line = line.replace('|', ' ', 1) + + # remove last separator if it is the last char in the line + if line[-1] == '|': + line = line[::-1].replace('|', ' ', 1)[::-1] + + return line + + +def _normalize_rows(table_lines: Iterable[str]) -> List[Tuple[int, List[str]]]: """ - Return a generator that yields header and data rows with different separators. - Also removes spaces from headers. + Return a List of tuples of a row counters and data lines. """ + result = [] header_found = False data_found = False + row_counter = 0 - # Removes initial table lines, finds the header row(s) and separates - # the header from the data rows with different separator characters. - for i in table_lines: - if separator in i and not header_found and not data_found: - # top table frame + for line in table_lines: + # skip blank lines + if not line.strip(): continue - if not separator in i and not header_found and not data_found: + + # skip top table frame + if _is_separator(line) and not header_found and not data_found: + continue + + # first header row found + if not _is_separator(line) and not header_found and not data_found: header_found = True - # first header data found - # remove spaces from header - i = re.sub(r'\b \b', '_', i) - yield i + line = _snake_case(line) + line = _fixup_separators(line) + line_list = line.split('|') + line_list = [x.strip() for x in line_list] + result.append((row_counter, line_list)) continue - if not separator in i and header_found and not data_found: - # subsequent header data found - # remove spaces from header - i = re.sub(r'\b \b', '_', i) - yield i + + # subsequent header row found + if not _is_separator(line) and header_found and not data_found: + line = _snake_case(line) + line = _fixup_separators(line) + line_list = line.split('|') + line_list = [x.strip() for x in line_list] + result.append((row_counter, line_list)) continue - if separator in i and header_found and not data_found: + + # table separator found - this is a header separator + if _is_separator(line) and header_found and not data_found: data_found = True - # table separator found - this is a header separator - yield separator - continue - if not separator in i and header_found and data_found: - # subsequent data row found - yield i - continue - if separator in i and header_found and data_found: - # table separator found - this is a data separator - yield data_separator + row_counter += 1 continue + # subsequent data row found + if not _is_separator(line) and header_found and data_found: + line = _fixup_separators(line) + line_list = line.split('|') + line_list = [x.strip() for x in line_list] + result.append((row_counter, line_list)) + continue -def _pretty_table_parse(table: Iterable) -> List[Dict]: - temp_table = [] - for line in table: - # normalize separator - line = line.replace('│', '|') + # table separator found - this is a data separator + if _is_separator(line) and header_found and data_found: + row_counter += 1 + continue - # remove first separator if it is the first char in the line - if line[0] == '|': - line = line.replace('|', ' ', 1) - - # remove last separator if it is the last char in the line - if line[-1] == '|': - line = line[::-1].replace('|', ' ', 1)[::-1] - - temp_table.append([x.strip() for x in line.split('|')]) - - headers = temp_table[0] - raw_data = temp_table[1:] - result = [dict.fromkeys(headers, None)] - result.extend([dict(zip(headers, r)) for r in raw_data]) return result -def _pretty_remove_header_rows(table: List[Dict], sep: str, data_sep: str) -> List[Optional[Dict]]: - """return a table with only data rows.""" - # create a new list of row objects with new key names - data_obj_list: List[Optional[Dict]] = [] - sep_found = False - data_sep_found = False - for obj in table: - #skip to data - for v in obj.values(): - if not sep_found and not str(v).strip() == sep: - continue - if not sep_found and str(v).strip() == sep: - sep_found = True - continue - # append data row objects or None for separators - if sep_found: - for k, v in obj.items(): - if str(v).strip() == data_sep: - data_sep_found = True - break - else: - data_sep_found = False - if data_sep_found: - data_obj_list.append(None) +def _get_headers(table: Iterable[Tuple[int, List]]) -> List[List[str]]: + """ + return a list of all of the header rows (which are lists of strings. + [ # headers + ['str', 'str', 'str'], # header rows + ['str', 'str', 'str'] + ] + """ + result = [] + for row_num, line in table: + if row_num == 0: + result.append(line) + return result + + +def _get_data(table: Iterable[Tuple[int, List]]) -> List[List[List[str]]]: + """ + return a list of rows, which are lists made up of lists of strings: + [ # data + [ # rows + ['str', 'str', 'str'] # lines + ] + ] + """ + result: List[List[List[str]]] = [] + current_row = 1 + this_line: List[List[str]] = [] + for row_num, line in table: + if row_num != 0: + if row_num != current_row: + result.append(this_line) + current_row = row_num + this_line = [] + + this_line.append(line) + + if this_line: + result.append(this_line) + + return result + + +def _collapse_headers(table: List[List[str]]) -> List[str]: + """append each column string to return the full header list""" + result = table[0] + for line in table[1:]: + new_line: List[str] = [] + for i, header in enumerate(line): + if header: + new_header = result[i] + '_' + header + new_header = re.sub(r'__+', '_', new_header) + new_line.append(new_header) else: - data_obj_list.append(obj) + new_line.append(result[i]) + result = new_line - # remove first item, which is a separator - return data_obj_list[1:] + return result -def _pretty_map_new_keynames(table: List[Dict], sep: str) -> Dict: - """ - returns a dict of old keyname to new keyname mappings by consolidating - multiline keynames from the input list of dictionaries. - """ - # first get all header objects to find full keynames. Stop when data rows are found. - header_obj_list = [] - sep_found = False - for obj in table: - for v in obj.values(): - if str(v).strip() == sep: - sep_found = True - break - if sep_found: - break - header_obj_list.append(obj) +def _collapse_data(table: List[List[List[str]]]) -> List[List[str]]: + """combine data rows to return a simple list of lists""" + result: List[List[str]] = [] - if not header_obj_list: - header_obj_list = [{key: None for key in table[0]}] + for row in table: + new_row: List[str] = [] + for line in row: + if new_row: + for i, item in enumerate(line): + new_row[i] = (new_row[i] + '\n' + item).strip() + else: + new_row = line - # create an old-key to new-key name mapping dict - new_keynames_dict = dict.fromkeys([key for key in header_obj_list[0]], '') - for item in new_keynames_dict: - new_keynames_dict[item] = item - for obj in header_obj_list: - for k, v in obj.items(): - if v: - new_keynames_dict[k] = new_keynames_dict[k] + '_' + v + result.append(new_row) - # normalize keynames so they are lowercase, no spaces, and no redundat '_'s - for k, v in new_keynames_dict.items(): - new_keynames_dict[k] = v.replace(' ', '_').lower() - new_keynames_dict[k] = re.sub(r'__+', '_', v) - - return new_keynames_dict + return result -def _pretty_rename_keys(table: List, new_keynames: Dict) -> List[Optional[Dict]]: - """rename all of the keys in the table based on the new_keynames mapping""" - renamed_key_table: List[Optional[Dict]] = [] - for item in table: - if item: - renamed_key_table.append({new_keynames[k]:v for k, v in item.items()}) - else: - renamed_key_table.append(None) - - return renamed_key_table - - -def _pretty_consolidate_rows(table: List) -> List[Dict]: - """go through all data objects and combine values between data separators""" - consolidated_rows = [] - current_obj = dict.fromkeys([key for key in table[0]], '') - for item in table: - if not item: - consolidated_rows.append(current_obj) - current_obj = dict.fromkeys([key for key in table[0]], '') - continue - else: - for k, v in item.items(): - if v: - if not current_obj[k]: - current_obj[k] = v - else: - current_obj[k] = current_obj[k] + '\n' + v - - return consolidated_rows +def _create_table_dict(header: List[str], data: List[List[str]]) -> List[Dict[str, str]]: + return [dict(zip(header, r)) for r in data] def _parse_pretty(string: str) -> List: - string_lines = string.splitlines() - sep = '~~~' - data_sep = '===' - separator = ' ' + sep + ' ' - data_separator = ' ' + data_sep + ' ' + string_lines: List[str] = string.splitlines() + clean: List[Tuple[int, List[str]]] = _normalize_rows(string_lines) + raw_headers: List[List[str]] = _get_headers(clean) + raw_data: List[List[List[str]]] = _get_data(clean) - clean: Generator = _pretty_set_separators(string_lines, separator) - normalized: Generator = _pretty_normalize_rows(clean, separator, data_separator) - raw_table: List[Dict] = _pretty_table_parse(normalized) # was sparse_table_parse() - new_keynames: Dict = _pretty_map_new_keynames(raw_table, sep) - data_table: List[Optional[Dict]] = _pretty_remove_header_rows(raw_table, sep, data_sep) - table_with_renamed_keys: List[Optional[Dict]] = _pretty_rename_keys(data_table, new_keynames) - final_table: List[Dict] = _pretty_consolidate_rows(table_with_renamed_keys) + new_headers: List[str] = _collapse_headers(raw_headers) + new_data: List[List[str]] = _collapse_data(raw_data) + final_table: List[Dict[str, str]] = _create_table_dict(new_headers, new_data) return final_table From 4eb2d725d5fe6f2cc7935c7dc891912cf7b8e940 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 21 Mar 2022 13:09:50 -0700 Subject: [PATCH 55/87] formatting --- jc/parsers/asciitable_m.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 5af62101..053c91a5 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -77,7 +77,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]: def _remove_ansi(string: str) -> str: - ansi_escape =re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]') + ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]') return ansi_escape.sub('', string) From 90e79b7df3a4d5b37835fdf1c2debc1bad0a40b9 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 21 Mar 2022 13:27:44 -0700 Subject: [PATCH 56/87] formatting --- jc/parsers/asciitable_m.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 053c91a5..5761c242 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -126,7 +126,7 @@ def _is_separator(line: str) -> bool: def _snake_case(line: str) -> str: - """replace spaces between words with an underscores and set to lowercase""" + """replace spaces between words with an underscore and set to lowercase""" return re.sub(r'\b \b', '_', line).lower() From f611d08b50219f4e573c1d1f0beae27583e76432 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 21 Mar 2022 13:33:35 -0700 Subject: [PATCH 57/87] formatting --- jc/parsers/asciitable_m.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 5761c242..f94bf8e3 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -224,8 +224,9 @@ def _get_data(table: Iterable[Tuple[int, List]]) -> List[List[List[str]]]: """ return a list of rows, which are lists made up of lists of strings: [ # data - [ # rows - ['str', 'str', 'str'] # lines + [ # data rows + ['str', 'str', 'str'], # data lines + ['str', 'str', 'str'] ] ] """ From 00c39450f9878527b60545ad61c04cfb98f418dc Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 21 Mar 2022 13:36:54 -0700 Subject: [PATCH 58/87] enhance type annotation --- jc/parsers/asciitable_m.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index f94bf8e3..abdd208e 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -287,7 +287,7 @@ def _create_table_dict(header: List[str], data: List[List[str]]) -> List[Dict[st return [dict(zip(header, r)) for r in data] -def _parse_pretty(string: str) -> List: +def _parse_pretty(string: str) -> List[Dict[str, str]]: string_lines: List[str] = string.splitlines() clean: List[Tuple[int, List[str]]] = _normalize_rows(string_lines) raw_headers: List[List[str]] = _get_headers(clean) From ab564f5be8282f1c7ee2d602bcef398fdfd4570f Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 21 Mar 2022 17:57:14 -0700 Subject: [PATCH 59/87] add tests --- jc/parsers/asciitable_m.py | 16 ++- tests/test_asciitable_m.py | 222 +++++++++++++++++++++++++++++++++++++ 2 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 tests/test_asciitable_m.py diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index abdd208e..e6907d4c 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -90,6 +90,20 @@ def _lstrip(string: str) -> str: return '\n'.join(new_lstrip_list) +def _rstrip(string: str) -> str: + """find the rightmost non-whitespace character and rstrip to that index""" + rstrip_list = [x for x in string.splitlines() if not len(x.strip()) == 0] + end_points = (len(x.rstrip()) for x in rstrip_list) + max_point = max(end_points) + new_rstrip_list = (x[:max_point] for x in rstrip_list) + return '\n'.join(new_rstrip_list) + + +def _strip(string: str) -> str: + string = _lstrip(string) + string = _rstrip(string) + return string + def _table_sniff(string: str) -> str: """Find the table-type via heuristics""" # pretty tables @@ -326,7 +340,7 @@ def parse( if jc.utils.has_data(data): data = _remove_ansi(data) - data = _lstrip(data) + data = _strip(data) table_type = _table_sniff(data) if table_type == 'pretty': diff --git a/tests/test_asciitable_m.py b/tests/test_asciitable_m.py new file mode 100644 index 00000000..d0b29f7d --- /dev/null +++ b/tests/test_asciitable_m.py @@ -0,0 +1,222 @@ +import os +import unittest +import jc.parsers.asciitable_m + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def test_asciitable_m_nodata(self): + """ + Test 'asciitable_m' with no data + """ + self.assertEqual(jc.parsers.asciitable_m.parse('', quiet=True), []) + + def test_asciitable_m_pure_ascii(self): + """ + Test 'asciitable_m' with a pure ASCII table + """ + input = ''' ++========+========+========+========+========+========+========+ +| type | tota | used | fr ee | shar | buff | avai | +| | l | | | ed | _cac | labl | +| | | | | | he | e | ++========+========+========+========+========+========+========+ +| Mem | 3861 | 2228 | 3364 | 1183 | 2743 | 3389 | +| | 332 | 20 | 176 | 2 | 36 | 588 | ++--------+--------+--------+--------+--------+--------+--------+ +| | | | | | | | +| | | | | test 2 | | | ++--------+--------+--------+--------+--------+--------+--------+ +| last | last | last | ab cde | | | final | ++========+========+========+========+========+========+========+ + ''' + expected = [ + { + "type": "Mem", + "tota_l": "3861\n332", + "used": "2228\n20", + "fr_ee": "3364\n176", + "shar_ed": "1183\n2", + "buff_cac_he": "2743\n36", + "avai_labl_e": "3389\n588" + }, + { + "type": "", + "tota_l": "", + "used": "", + "fr_ee": "", + "shar_ed": "test 2", + "buff_cac_he": "", + "avai_labl_e": "" + }, + { + "type": "last", + "tota_l": "last", + "used": "last", + "fr_ee": "ab cde", + "shar_ed": "", + "buff_cac_he": "", + "avai_labl_e": "final" + } + ] + + self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected) + + def test_asciitable_m_unicode(self): + """ + Test 'asciitable_m' with a unicode table + """ + input = ''' +╒════════╤════════╤════════╤════════╤════════╤════════╤════════╕ +│ type │ tota │ used │ fr ee │ shar │ buff │ avai │ +│ │ l │ │ │ ed │ _cac │ labl │ +│ │ │ │ │ │ he │ e │ +╞════════╪════════╪════════╪════════╪════════╪════════╪════════╡ +│ Mem │ 3861 │ 2228 │ 3364 │ 1183 │ 2743 │ 3389 │ +│ │ 332 │ 20 │ 176 │ 2 │ 36 │ 588 │ +├────────┼────────┼────────┼────────┼────────┼────────┼────────┤ +│ Swap │ 2097 │ 0 │ 2097 │ │ │ │ +│ │ 148 │ │ 148 │ │ │ │ +│ │ │ │ kb │ │ │ │ +├────────┼────────┼────────┼────────┼────────┼────────┼────────┤ +│ last │ last │ last │ ab cde │ │ │ final │ +╘════════╧════════╧════════╧════════╧════════╧════════╧════════╛ + ''' + expected = [ + { + "type": "Mem", + "tota_l": "3861\n332", + "used": "2228\n20", + "fr_ee": "3364\n176", + "shar_ed": "1183\n2", + "buff_cac_he": "2743\n36", + "avai_labl_e": "3389\n588" + }, + { + "type": "Swap", + "tota_l": "2097\n148", + "used": "0", + "fr_ee": "2097\n148\nkb", + "shar_ed": "", + "buff_cac_he": "", + "avai_labl_e": "" + }, + { + "type": "last", + "tota_l": "last", + "used": "last", + "fr_ee": "ab cde", + "shar_ed": "", + "buff_cac_he": "", + "avai_labl_e": "final" + } + ] + + self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected) + + def test_asciitable_m_pure_ascii_extra_spaces(self): + """ + Test 'asciitable_m' with a pure ASCII table that has heading and + trailing spaces and newlines. + """ + input = ''' + + + +========+========+========+========+========+========+========+ + | type | tota | used | fr ee | shar | buff | avai + | | l | | | ed | _cac | labl + | | | | | | he | e | + +========+========+========+========+========+========+========+ + | Mem | 3861 | 2228 | 3364 | 1183 | 2743 | 3389 | + | | 332 | 20 | 176 | 2 | 36 | 588 | + +--------+--------+--------+--------+--------+--------+--------+ + | | | | | | | | + | | | | | test 2 | | | + +--------+--------+--------+--------+--------+--------+--------+ + | last | last | last | ab cde | | | final + +========+========+========+========+========+========+========+ + + + ''' + expected = [ + { + "type": "Mem", + "tota_l": "3861\n332", + "used": "2228\n20", + "fr_ee": "3364\n176", + "shar_ed": "1183\n2", + "buff_cac_he": "2743\n36", + "avai_labl_e": "3389\n588" + }, + { + "type": "", + "tota_l": "", + "used": "", + "fr_ee": "", + "shar_ed": "test 2", + "buff_cac_he": "", + "avai_labl_e": "" + }, + { + "type": "last", + "tota_l": "last", + "used": "last", + "fr_ee": "ab cde", + "shar_ed": "", + "buff_cac_he": "", + "avai_labl_e": "final" + } + ] + + self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected) + + def test_asciitable_m_unicode_extra_spaces(self): + """ + Test 'asciitable_m' with a pure ASCII table that has heading and + trailing spaces and newlines. + """ + input = ''' + + + ╒════════╤════════╤════════╤════════╤════════╤════════╤════════╕ + type │ tota │ used │ free │ shar │ buff │ avai + │ l │ │ │ ed │ _cac │ labl + │ │ │ │ │ he │ e + ╞════════╪════════╪════════╪════════╪════════╪════════╪════════╡ + Mem │ 3861 │ 2228 │ 3364 │ 1183 │ 2743 │ 3389 + │ 332 │ 20 │ 176 │ 2 │ 36 │ 588 + ├────────┼────────┼────────┼────────┼────────┼────────┼────────┤ + Swap │ 2097 │ 0 │ 2097 │ │ │ + │ 148 │ │ 148 │ │ │ + ╘════════╧════════╧════════╧════════╧════════╧════════╧════════╛ + + + ''' + expected = [ + { + "type": "Mem", + "tota_l": "3861\n332", + "used": "2228\n20", + "free": "3364\n176", + "shar_ed": "1183\n2", + "buff_cac_he": "2743\n36", + "avai_labl_e": "3389\n588" + }, + { + "type": "Swap", + "tota_l": "2097\n148", + "used": "0", + "free": "2097\n148", + "shar_ed": "", + "buff_cac_he": "", + "avai_labl_e": "" + } + ] + + self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected) + + +if __name__ == '__main__': + unittest.main() From 53dd05e52c329a15c30a756a36a165ac59ef8a14 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 21 Mar 2022 19:10:02 -0700 Subject: [PATCH 60/87] fix rstrip and add tests --- jc/parsers/asciitable_m.py | 3 ++- tests/test_asciitable_m.py | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index e6907d4c..2ae31c89 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -95,7 +95,7 @@ def _rstrip(string: str) -> str: rstrip_list = [x for x in string.splitlines() if not len(x.strip()) == 0] end_points = (len(x.rstrip()) for x in rstrip_list) max_point = max(end_points) - new_rstrip_list = (x[:max_point] for x in rstrip_list) + new_rstrip_list = ((x + ' ' * max_point)[:max_point] for x in rstrip_list) return '\n'.join(new_rstrip_list) @@ -341,6 +341,7 @@ def parse( if jc.utils.has_data(data): data = _remove_ansi(data) data = _strip(data) + print(data.replace(' ', '~')) table_type = _table_sniff(data) if table_type == 'pretty': diff --git a/tests/test_asciitable_m.py b/tests/test_asciitable_m.py index d0b29f7d..ea398b4e 100644 --- a/tests/test_asciitable_m.py +++ b/tests/test_asciitable_m.py @@ -1,5 +1,6 @@ import os import unittest +from jc.exceptions import ParseError import jc.parsers.asciitable_m THIS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -217,6 +218,32 @@ class MyTests(unittest.TestCase): self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected) + def test_asciitable_m_markdown(self): + """ + Test 'asciitable_m' with a markdown table. Should raise a ParseError + """ + input = ''' + | type | total | used | free | shared | buff cache | available | + |--------|---------|--------|---------|----------|--------------|-------------| + | Mem | 3861332 | 222820 | 3364176 | 11832 | 274336 | 3389588 | + | Swap | 2097148 | 0 | 2097148 | | | | + ''' + + self.assertRaises(ParseError, jc.parsers.asciitable_m.parse, input, quiet=True) + + def test_asciitable_m_simple(self): + """ + Test 'asciitable_m' with a simple table. Should raise a ParseError + """ + input = ''' + type total used free shared buff cache available + ------ ------- ------ ------- -------- ------------ ----------- + Mem 3861332 222820 3364176 11832 274336 3389588 + Swap 2097148 0 2097148 + ''' + + self.assertRaises(ParseError, jc.parsers.asciitable_m.parse, input, quiet=True) + if __name__ == '__main__': unittest.main() From 01d53da68ed04b736934851137c59216ad02d8c8 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 21 Mar 2022 19:11:27 -0700 Subject: [PATCH 61/87] remove debug print --- jc/parsers/asciitable_m.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 2ae31c89..7daf602e 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -341,7 +341,6 @@ def parse( if jc.utils.has_data(data): data = _remove_ansi(data) data = _strip(data) - print(data.replace(' ', '~')) table_type = _table_sniff(data) if table_type == 'pretty': From d451c309bb0d8889dc09a698212bf01c2154a582 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 06:30:07 -0700 Subject: [PATCH 62/87] change multiple or statements to any() --- jc/parsers/asciitable_m.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 7daf602e..b80fbadd 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -104,6 +104,7 @@ def _strip(string: str) -> str: string = _rstrip(string) return string + def _table_sniff(string: str) -> str: """Find the table-type via heuristics""" # pretty tables @@ -127,14 +128,16 @@ def _table_sniff(string: str) -> str: def _is_separator(line: str) -> bool: """Returns true if a table separator line is found""" strip_line = line.strip() - if strip_line.startswith('╒═') and strip_line.endswith('═╕')\ - or strip_line.startswith('╞═') and strip_line.endswith('═╡')\ - or strip_line.startswith('╘═') and strip_line.endswith('═╛')\ - or strip_line.startswith('┌─') and strip_line.endswith('─┐')\ - or strip_line.startswith('├─') and strip_line.endswith('─┤')\ - or strip_line.startswith('└─') and strip_line.endswith('─┘')\ - or strip_line.startswith('+=') and strip_line.endswith('=+')\ - or strip_line.startswith('+-') and strip_line.endswith('-+'): + if any(( + strip_line.startswith('╒═') and strip_line.endswith('═╕'), + strip_line.startswith('╞═') and strip_line.endswith('═╡'), + strip_line.startswith('╘═') and strip_line.endswith('═╛'), + strip_line.startswith('┌─') and strip_line.endswith('─┐'), + strip_line.startswith('├─') and strip_line.endswith('─┤'), + strip_line.startswith('└─') and strip_line.endswith('─┘'), + strip_line.startswith('+=') and strip_line.endswith('=+'), + strip_line.startswith('+-') and strip_line.endswith('-+') + )): return True return False From b204c423c1b947c17cb251a5dd3c444cd71dd2c0 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 07:05:14 -0700 Subject: [PATCH 63/87] doc update --- README.md | 1 + docs/parsers/asciitable_m.md | 122 +++++++++++++++++++++++++++++++++++ jc/parsers/asciitable_m.py | 78 ++++++++++++++++++---- man/jc.1 | 7 +- 4 files changed, 193 insertions(+), 15 deletions(-) create mode 100644 docs/parsers/asciitable_m.md diff --git a/README.md b/README.md index 46316b3d..83613911 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,7 @@ option. - `--airport` enables the `airport -I` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/airport)) - `--airport-s` enables the `airport -s` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/airport_s)) - `--arp` enables the `arp` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/arp)) +- `--asciitable-m` enables the multi-line ASCII and Unicode table parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable_m)) - `--blkid` enables the `blkid` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/blkid)) - `--cksum` enables the `cksum` and `sum` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/cksum)) - `--crontab` enables the `crontab` command and file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/crontab)) diff --git a/docs/parsers/asciitable_m.md b/docs/parsers/asciitable_m.md new file mode 100644 index 00000000..23056ca9 --- /dev/null +++ b/docs/parsers/asciitable_m.md @@ -0,0 +1,122 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.asciitable\_m + +jc - JSON Convert `asciitable-m` parser + +This parser converts various styles of ASCII and Unicode text tables with +multi-line rows. Tables must have a header row and separator line between +rows. + +For example: + + ╒══════════╤═════════╤════════╕ + │ foo │ bar baz │ fiz │ + │ │ │ buz │ + ╞══════════╪═════════╪════════╡ + │ good day │ 12345 │ │ + │ mate │ │ │ + ├──────────┼─────────┼────────┤ + │ hi there │ abc def │ 3.14 │ + │ │ │ │ + ╘══════════╧═════════╧════════╛ + +Cells with multiple lines within rows will be joined with a new-line +character ('\n'). + +Headers (keys) are converted to snake case and newlines between multi-line +headers are joined with an underscore. All values are returned as strings. + +Usage (cli): + + $ cat table.txt | jc --asciitable-m + +Usage (module): + + import jc + result = jc.parse('asciitable_m', asciitable-string) + +Schema: + + [ + { + "column_name1": string, + "column_name2": string + } + ] + +Examples: + + $ echo ' + > +----------+---------+--------+ + > | foo | bar | baz | + > | | | buz | + > +==========+=========+========+ + > | good day | 12345 | | + > | mate | | | + > +----------+---------+--------+ + > | hi there | abc def | 3.14 | + > | | | | + > +==========+=========+========+' | jc --asciitable-m -p + [ + { + "foo": "good day\nmate", + "bar": "12345", + "baz_buz": "" + }, + { + "foo": "hi there", + "bar": "abc def", + "baz_buz": "3.14" + } + ] + + $ echo ' + > ╒══════════╤═════════╤════════╕ + > │ foo │ bar │ baz │ + > │ │ │ buz │ + > ╞══════════╪═════════╪════════╡ + > │ good day │ 12345 │ │ + > │ mate │ │ │ + > ├──────────┼─────────┼────────┤ + > │ hi there │ abc def │ 3.14 │ + > │ │ │ │ + > ╘══════════╧═════════╧════════╛' | jc --asciitable-m -p + [ + { + "foo": "good day\nmate", + "bar": "12345", + "baz_buz": "" + }, + { + "foo": "hi there", + "bar": "abc def", + "baz_buz": "3.14" + } + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index b80fbadd..bf19edb9 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -1,7 +1,8 @@ """jc - JSON Convert `asciitable-m` parser -This parser converts ASCII and Unicode text tables with multi-line rows. -Tables must have some sort of separator line between rows. +This parser converts various styles of ASCII and Unicode text tables with +multi-line rows. Tables must have a header row and separator line between +rows. For example: @@ -16,6 +17,12 @@ For example: │ │ │ │ ╘══════════╧═════════╧════════╛ +Cells with multiple lines within rows will be joined with a new-line +character ('\n'). + +Headers (keys) are converted to snake case and newlines between multi-line +headers are joined with an underscore. All values are returned as strings. + Usage (cli): $ cat table.txt | jc --asciitable-m @@ -29,19 +36,60 @@ Schema: [ { - "asciitable-m": string, - "bar": boolean, - "baz": integer + "column_name1": string, + "column_name2": string } ] Examples: - $ asciitable-m | jc --asciitable-m -p - [] + $ echo ' + > +----------+---------+--------+ + > | foo | bar | baz | + > | | | buz | + > +==========+=========+========+ + > | good day | 12345 | | + > | mate | | | + > +----------+---------+--------+ + > | hi there | abc def | 3.14 | + > | | | | + > +==========+=========+========+' | jc --asciitable-m -p + [ + { + "foo": "good day\nmate", + "bar": "12345", + "baz_buz": "" + }, + { + "foo": "hi there", + "bar": "abc def", + "baz_buz": "3.14" + } + ] - $ asciitable-m | jc --asciitable-m -p -r - [] + $ echo ' + > ╒══════════╤═════════╤════════╕ + > │ foo │ bar │ baz │ + > │ │ │ buz │ + > ╞══════════╪═════════╪════════╡ + > │ good day │ 12345 │ │ + > │ mate │ │ │ + > ├──────────┼─────────┼────────┤ + > │ hi there │ abc def │ 3.14 │ + > │ │ │ │ + > ╘══════════╧═════════╧════════╛' | jc --asciitable-m -p + [ + { + "foo": "good day\nmate", + "bar": "12345", + "baz_buz": "" + }, + { + "foo": "hi there", + "bar": "abc def", + "baz_buz": "3.14" + } + ] """ import re from typing import Iterable, Tuple, List, Dict @@ -91,7 +139,7 @@ def _lstrip(string: str) -> str: def _rstrip(string: str) -> str: - """find the rightmost non-whitespace character and rstrip to that index""" + """find the rightmost non-whitespace character and rstrip and pad to that index""" rstrip_list = [x for x in string.splitlines() if not len(x.strip()) == 0] end_points = (len(x.rstrip()) for x in rstrip_list) max_point = max(end_points) @@ -110,10 +158,12 @@ def _table_sniff(string: str) -> str: # pretty tables for line in string.splitlines(): line = line.strip() - if line.startswith('╞═') and line.endswith('═╡')\ - or line.startswith('├─') and line.endswith('─┤')\ - or line.startswith('+=') and line.endswith('=+')\ - or line.startswith('+-') and line.endswith('-+'): + if any(( + line.startswith('╞═') and line.endswith('═╡'), + line.startswith('├─') and line.endswith('─┤'), + line.startswith('+=') and line.endswith('=+'), + line.startswith('+-') and line.endswith('-+') + )): return 'pretty' # markdown tables diff --git a/man/jc.1 b/man/jc.1 index d88db55f..e29c47d6 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-03-14 1.18.6 "JSON Convert" +.TH jc 1 2022-03-22 1.18.6 "JSON Convert" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -37,6 +37,11 @@ Parsers: \fB--arp\fP `arp` command parser +.TP +.B +\fB--asciitable-m\fP +multi-line ASCII and Unicode table parser + .TP .B \fB--blkid\fP From b3b140066b74ee31dd1c460502caf2f26df321b6 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 07:21:19 -0700 Subject: [PATCH 64/87] doc update --- docs/parsers/asciitable_m.md | 4 ++-- jc/parsers/asciitable_m.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/parsers/asciitable_m.md b/docs/parsers/asciitable_m.md index 23056ca9..8afd2e3d 100644 --- a/docs/parsers/asciitable_m.md +++ b/docs/parsers/asciitable_m.md @@ -22,10 +22,10 @@ For example: │ │ │ │ ╘══════════╧═════════╧════════╛ -Cells with multiple lines within rows will be joined with a new-line +Cells with multiple lines within rows will be joined with a newline character ('\n'). -Headers (keys) are converted to snake case and newlines between multi-line +Headers (keys) are converted to snake-case and newlines between multi-line headers are joined with an underscore. All values are returned as strings. Usage (cli): diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index bf19edb9..c3f1c575 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -17,10 +17,10 @@ For example: │ │ │ │ ╘══════════╧═════════╧════════╛ -Cells with multiple lines within rows will be joined with a new-line +Cells with multiple lines within rows will be joined with a newline character ('\n'). -Headers (keys) are converted to snake case and newlines between multi-line +Headers (keys) are converted to snake-case and newlines between multi-line headers are joined with an underscore. All values are returned as strings. Usage (cli): From 7d95d679bf4093ea77f30a1be55384b6dbf08341 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 12:25:24 -0700 Subject: [PATCH 65/87] add asciitable parser --- jc/lib.py | 2 +- jc/parsers/asciitable.py | 211 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 jc/parsers/asciitable.py diff --git a/jc/lib.py b/jc/lib.py index fd608421..bc6754bd 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -13,7 +13,7 @@ parsers = [ 'airport', 'airport-s', 'arp', - # 'asciitable', + 'asciitable', 'asciitable-m', 'blkid', 'cksum', diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py new file mode 100644 index 00000000..f61de96c --- /dev/null +++ b/jc/parsers/asciitable.py @@ -0,0 +1,211 @@ +"""jc - JSON Convert `asciitable` parser + +This parser converts ASCII and Unicode text tables with single-line rows. + +Column headers must be at least two spaces apart from each other and must +be unique. + +For example: + + ╒══════════╤═════════╤════════╕ + │ foo │ bar │ baz │ + ╞══════════╪═════════╪════════╡ + │ good day │ │ 12345 │ + ├──────────┼─────────┼────────┤ + │ hi there │ abc def │ 3.14 │ + ╘══════════╧═════════╧════════╛ + + or + + +-----------------------------+ + | foo bar baz | + +-----------------------------+ + | good day 12345 | + | hi there abc def 3.14 | + +-----------------------------+ + + or + + | foo | bar | baz | + |----------|---------|--------| + | good day | | 12345 | + | hi there | abc def | 3.14 | + + or + + foo bar baz + --------- -------- ------ + good day 12345 + hi there abc def + + etc. + +Usage (cli): + + $ cat table.txt | jc --asciitable + +Usage (module): + + import jc + result = jc.parse('asciitable', asciitable_string) + +Schema: + + [ + { + "column_name1": string, # empty string is null + "column_name2": string # empty string is null + } + ] + +Examples: + + $ asciitable | jc --asciitable -p + [] + + $ asciitable | jc --asciitable -p -r + [] +""" +import re +from typing import List, Dict +import jc.utils +from jc.parsers.universal import sparse_table_parse + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'ASCII and Unicode table parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def _remove_ansi(string: str) -> str: + ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]') + return ansi_escape.sub('', string) + + +def _lstrip(string: str) -> str: + """find the leftmost non-whitespace character and lstrip to that index""" + lstrip_list = [x for x in string.splitlines() if not len(x.strip()) == 0] + start_points = (len(x) - len(x.lstrip()) for x in lstrip_list) + min_point = min(start_points) + new_lstrip_list = (x[min_point:] for x in lstrip_list) + return '\n'.join(new_lstrip_list) + + +def _rstrip(string: str) -> str: + """find the rightmost non-whitespace character and rstrip and pad to that index""" + rstrip_list = [x for x in string.splitlines() if not len(x.strip()) == 0] + end_points = (len(x.rstrip()) for x in rstrip_list) + max_point = max(end_points) + new_rstrip_list = ((x + ' ' * max_point)[:max_point] for x in rstrip_list) + return '\n'.join(new_rstrip_list) + + +def _strip(string: str) -> str: + string = _lstrip(string) + string = _rstrip(string) + return string + + +def _is_separator(line: str) -> bool: + """Returns true if a table separator line is found""" + strip_line = line.strip() + if any(( + strip_line.startswith('╒═') and strip_line.endswith('═╕'), + strip_line.startswith('╞═') and strip_line.endswith('═╡'), + strip_line.startswith('╘═') and strip_line.endswith('═╛'), + strip_line.startswith('┌─') and strip_line.endswith('─┐'), + strip_line.startswith('├─') and strip_line.endswith('─┤'), + strip_line.startswith('└─') and strip_line.endswith('─┘'), + strip_line.startswith('+=') and strip_line.endswith('=+'), + strip_line.startswith('+-') and strip_line.endswith('-+'), + strip_line.startswith('--') and strip_line.endswith('--'), + strip_line.startswith('==') and strip_line.endswith('=='), + strip_line.startswith('|-') and strip_line.endswith('-|') + )): + return True + return False + + +def _snake_case(line: str) -> str: + """replace spaces between words with an underscore and set to lowercase""" + return re.sub(r'\b \b', '_', line).lower() + + +def _normalize_rows(table: str) -> List[str]: + """ + Return a List row strings. Header is snake-cased + """ + result = [] + for line in table.splitlines(): + # skip blank lines + if not line.strip(): + continue + + # skip separators + if _is_separator(line): + continue + + # data row - remove column separators + line = line.replace('│', ' ').replace('|', ' ') + result.append(line) + + result[0] = _snake_case(result[0]) + return result + + +def _parse_pretty(table: List[str]) -> List[Dict[str, str]]: + return sparse_table_parse(table) + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + data = _remove_ansi(data) + data = _strip(data) + data_list = _normalize_rows(data) + raw_output = _parse_pretty(data_list) + + return raw_output if raw else _process(raw_output) From aea2e1b0a9c6bb2a02a6e73889bb04e8f4f22588 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 12:25:51 -0700 Subject: [PATCH 66/87] fix tests so blank strings are now None --- tests/test_asciitable_m.py | 48 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/test_asciitable_m.py b/tests/test_asciitable_m.py index ea398b4e..22c57f0a 100644 --- a/tests/test_asciitable_m.py +++ b/tests/test_asciitable_m.py @@ -44,21 +44,21 @@ class MyTests(unittest.TestCase): "avai_labl_e": "3389\n588" }, { - "type": "", - "tota_l": "", - "used": "", - "fr_ee": "", + "type": None, + "tota_l": None, + "used": None, + "fr_ee": None, "shar_ed": "test 2", - "buff_cac_he": "", - "avai_labl_e": "" + "buff_cac_he": None, + "avai_labl_e": None }, { "type": "last", "tota_l": "last", "used": "last", "fr_ee": "ab cde", - "shar_ed": "", - "buff_cac_he": "", + "shar_ed": None, + "buff_cac_he": None, "avai_labl_e": "final" } ] @@ -100,17 +100,17 @@ class MyTests(unittest.TestCase): "tota_l": "2097\n148", "used": "0", "fr_ee": "2097\n148\nkb", - "shar_ed": "", - "buff_cac_he": "", - "avai_labl_e": "" + "shar_ed": None, + "buff_cac_he": None, + "avai_labl_e": None }, { "type": "last", "tota_l": "last", "used": "last", "fr_ee": "ab cde", - "shar_ed": "", - "buff_cac_he": "", + "shar_ed": None, + "buff_cac_he": None, "avai_labl_e": "final" } ] @@ -152,21 +152,21 @@ class MyTests(unittest.TestCase): "avai_labl_e": "3389\n588" }, { - "type": "", - "tota_l": "", - "used": "", - "fr_ee": "", + "type": None, + "tota_l": None, + "used": None, + "fr_ee": None, "shar_ed": "test 2", - "buff_cac_he": "", - "avai_labl_e": "" + "buff_cac_he": None, + "avai_labl_e": None }, { "type": "last", "tota_l": "last", "used": "last", "fr_ee": "ab cde", - "shar_ed": "", - "buff_cac_he": "", + "shar_ed": None, + "buff_cac_he": None, "avai_labl_e": "final" } ] @@ -210,9 +210,9 @@ class MyTests(unittest.TestCase): "tota_l": "2097\n148", "used": "0", "free": "2097\n148", - "shar_ed": "", - "buff_cac_he": "", - "avai_labl_e": "" + "shar_ed": None, + "buff_cac_he": None, + "avai_labl_e": None } ] From f23f19da453a0f7bcf70e417a10f87e27571aa43 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 12:25:59 -0700 Subject: [PATCH 67/87] doc update --- README.md | 1 + docs/parsers/asciitable.md | 97 ++++++++++++++++++++++++++++++++++++ docs/parsers/asciitable_m.md | 4 +- jc/parsers/asciitable_m.py | 24 ++++++--- man/jc.1 | 5 ++ 5 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 docs/parsers/asciitable.md diff --git a/README.md b/README.md index 83613911..45a7d0d4 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,7 @@ option. - `--airport` enables the `airport -I` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/airport)) - `--airport-s` enables the `airport -s` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/airport_s)) - `--arp` enables the `arp` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/arp)) +- `--asciitable` enables the ASCII and Unicode table parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable)) - `--asciitable-m` enables the multi-line ASCII and Unicode table parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable_m)) - `--blkid` enables the `blkid` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/blkid)) - `--cksum` enables the `cksum` and `sum` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/cksum)) diff --git a/docs/parsers/asciitable.md b/docs/parsers/asciitable.md new file mode 100644 index 00000000..8a4ced2a --- /dev/null +++ b/docs/parsers/asciitable.md @@ -0,0 +1,97 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.asciitable + +jc - JSON Convert `asciitable` parser + +This parser converts ASCII and Unicode text tables with single-line rows. + +Column headers must be at least two spaces apart from each other and must +be unique. + +For example: + + ╒══════════╤═════════╤════════╕ + │ foo │ bar │ baz │ + ╞══════════╪═════════╪════════╡ + │ good day │ │ 12345 │ + ├──────────┼─────────┼────────┤ + │ hi there │ abc def │ 3.14 │ + ╘══════════╧═════════╧════════╛ + + or + + +-----------------------------+ + | foo bar baz | + +-----------------------------+ + | good day 12345 | + | hi there abc def 3.14 | + +-----------------------------+ + + or + + | foo | bar | baz | + |----------|---------|--------| + | good day | | 12345 | + | hi there | abc def | 3.14 | + + or + + foo bar baz + --------- -------- ------ + good day 12345 + hi there abc def + + etc. + +Usage (cli): + + $ cat table.txt | jc --asciitable + +Usage (module): + + import jc + result = jc.parse('asciitable', asciitable_string) + +Schema: + + [ + { + "column_name1": string, # empty string is null + "column_name2": string # empty string is null + } + ] + +Examples: + + $ asciitable | jc --asciitable -p + [] + + $ asciitable | jc --asciitable -p -r + [] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, cygwin, win32, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/asciitable_m.md b/docs/parsers/asciitable_m.md index 8afd2e3d..7c416618 100644 --- a/docs/parsers/asciitable_m.md +++ b/docs/parsers/asciitable_m.md @@ -41,8 +41,8 @@ Schema: [ { - "column_name1": string, - "column_name2": string + "column_name1": string, # empty string is null + "column_name2": string # empty string is null } ] diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index c3f1c575..415fef0b 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -36,8 +36,8 @@ Schema: [ { - "column_name1": string, - "column_name2": string + "column_name1": string, # empty string is null + "column_name2": string # empty string is null } ] @@ -92,7 +92,7 @@ Examples: ] """ import re -from typing import Iterable, Tuple, List, Dict +from typing import Iterable, Tuple, List, Dict, Optional import jc.utils from jc.exceptions import ParseError @@ -350,11 +350,21 @@ def _collapse_data(table: List[List[List[str]]]) -> List[List[str]]: return result -def _create_table_dict(header: List[str], data: List[List[str]]) -> List[Dict[str, str]]: - return [dict(zip(header, r)) for r in data] +def _create_table_dict(header: List[str], data: List[List[str]]) -> List[Dict[str, Optional[str]]]: + """ + zip the headers and data to create a list of dictionaries. Also convert + empty strings to None. + """ + table_list_dict: List[Dict[str, Optional[str]]] = [dict(zip(header, r)) for r in data] + for row in table_list_dict: + for k, v in row.items(): + if v == '': + row[k] = None + + return table_list_dict -def _parse_pretty(string: str) -> List[Dict[str, str]]: +def _parse_pretty(string: str) -> List[Dict[str, Optional[str]]]: string_lines: List[str] = string.splitlines() clean: List[Tuple[int, List[str]]] = _normalize_rows(string_lines) raw_headers: List[List[str]] = _get_headers(clean) @@ -362,7 +372,7 @@ def _parse_pretty(string: str) -> List[Dict[str, str]]: new_headers: List[str] = _collapse_headers(raw_headers) new_data: List[List[str]] = _collapse_data(raw_data) - final_table: List[Dict[str, str]] = _create_table_dict(new_headers, new_data) + final_table: List[Dict[str, Optional[str]]] = _create_table_dict(new_headers, new_data) return final_table diff --git a/man/jc.1 b/man/jc.1 index e29c47d6..7b642f1b 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -37,6 +37,11 @@ Parsers: \fB--arp\fP `arp` command parser +.TP +.B +\fB--asciitable\fP +ASCII and Unicode table parser + .TP .B \fB--asciitable-m\fP From 9c1ec9940e68ce9848da20e81b8c0b7e35a215df Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 12:35:56 -0700 Subject: [PATCH 68/87] doc update --- docs/parsers/asciitable.md | 44 ++++++++++++++++++++++++++++++++---- docs/parsers/asciitable_m.md | 4 ++-- jc/parsers/asciitable.py | 44 ++++++++++++++++++++++++++++++++---- jc/parsers/asciitable_m.py | 4 ++-- 4 files changed, 82 insertions(+), 14 deletions(-) diff --git a/docs/parsers/asciitable.md b/docs/parsers/asciitable.md index 8a4ced2a..15b869bd 100644 --- a/docs/parsers/asciitable.md +++ b/docs/parsers/asciitable.md @@ -43,7 +43,7 @@ For example: good day 12345 hi there abc def - etc. + etc... Usage (cli): @@ -65,11 +65,45 @@ Schema: Examples: - $ asciitable | jc --asciitable -p - [] + $ echo ' + > ╒══════════╤═════════╤════════╕ + > │ foo │ bar │ baz │ + > ╞══════════╪═════════╪════════╡ + > │ good day │ │ 12345 │ + > ├──────────┼─────────┼────────┤ + > │ hi there │ abc def │ 3.14 │ + > ╘══════════╧═════════╧════════╛' | jc --asciitable -p + [ + { + "foo": "good day", + "bar": null, + "baz": "12345" + }, + { + "foo": "hi there", + "bar": "abc def", + "baz": "3.14" + } + ] - $ asciitable | jc --asciitable -p -r - [] + + $ echo ' + > foo bar baz + > --------- -------- ------ + > good day 12345 + > hi there abc def' | jc --asciitable -p + [ + { + "foo": "good day", + "bar": null, + "baz": "12345" + }, + { + "foo": "hi there", + "bar": "abc def", + "baz": null + } + ] diff --git a/docs/parsers/asciitable_m.md b/docs/parsers/asciitable_m.md index 7c416618..2e72cfca 100644 --- a/docs/parsers/asciitable_m.md +++ b/docs/parsers/asciitable_m.md @@ -63,7 +63,7 @@ Examples: { "foo": "good day\nmate", "bar": "12345", - "baz_buz": "" + "baz_buz": null }, { "foo": "hi there", @@ -87,7 +87,7 @@ Examples: { "foo": "good day\nmate", "bar": "12345", - "baz_buz": "" + "baz_buz": null }, { "foo": "hi there", diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index f61de96c..8e0af9c3 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -38,7 +38,7 @@ For example: good day 12345 hi there abc def - etc. + etc... Usage (cli): @@ -60,11 +60,45 @@ Schema: Examples: - $ asciitable | jc --asciitable -p - [] + $ echo ' + > ╒══════════╤═════════╤════════╕ + > │ foo │ bar │ baz │ + > ╞══════════╪═════════╪════════╡ + > │ good day │ │ 12345 │ + > ├──────────┼─────────┼────────┤ + > │ hi there │ abc def │ 3.14 │ + > ╘══════════╧═════════╧════════╛' | jc --asciitable -p + [ + { + "foo": "good day", + "bar": null, + "baz": "12345" + }, + { + "foo": "hi there", + "bar": "abc def", + "baz": "3.14" + } + ] - $ asciitable | jc --asciitable -p -r - [] + + $ echo ' + > foo bar baz + > --------- -------- ------ + > good day 12345 + > hi there abc def' | jc --asciitable -p + [ + { + "foo": "good day", + "bar": null, + "baz": "12345" + }, + { + "foo": "hi there", + "bar": "abc def", + "baz": null + } + ] """ import re from typing import List, Dict diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 415fef0b..63f59a27 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -58,7 +58,7 @@ Examples: { "foo": "good day\nmate", "bar": "12345", - "baz_buz": "" + "baz_buz": null }, { "foo": "hi there", @@ -82,7 +82,7 @@ Examples: { "foo": "good day\nmate", "bar": "12345", - "baz_buz": "" + "baz_buz": null }, { "foo": "hi there", From 82a63fe15922456853aa4597aa10a5a02b20bdef Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 12:42:07 -0700 Subject: [PATCH 69/87] doc update --- docs/parsers/asciitable.md | 7 +++---- jc/parsers/asciitable.py | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/parsers/asciitable.md b/docs/parsers/asciitable.md index 15b869bd..10238221 100644 --- a/docs/parsers/asciitable.md +++ b/docs/parsers/asciitable.md @@ -41,7 +41,7 @@ For example: foo bar baz --------- -------- ------ good day 12345 - hi there abc def + hi there abc def 3.14 etc... @@ -86,12 +86,11 @@ Examples: } ] - $ echo ' > foo bar baz > --------- -------- ------ > good day 12345 - > hi there abc def' | jc --asciitable -p + > hi there abc def 3.14' | jc --asciitable -p [ { "foo": "good day", @@ -101,7 +100,7 @@ Examples: { "foo": "hi there", "bar": "abc def", - "baz": null + "baz": "3.14" } ] diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index 8e0af9c3..281f7578 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -36,7 +36,7 @@ For example: foo bar baz --------- -------- ------ good day 12345 - hi there abc def + hi there abc def 3.14 etc... @@ -81,12 +81,11 @@ Examples: } ] - $ echo ' > foo bar baz > --------- -------- ------ > good day 12345 - > hi there abc def' | jc --asciitable -p + > hi there abc def 3.14' | jc --asciitable -p [ { "foo": "good day", @@ -96,7 +95,7 @@ Examples: { "foo": "hi there", "bar": "abc def", - "baz": null + "baz": "3.14" } ] """ From 720480e39ce3cc2079d7f4f5bcff2725081fec34 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 13:21:10 -0700 Subject: [PATCH 70/87] doc update --- docs/parsers/asciitable.md | 3 +++ docs/parsers/asciitable_m.md | 3 ++- jc/parsers/asciitable.py | 3 +++ jc/parsers/asciitable_m.py | 3 ++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/parsers/asciitable.md b/docs/parsers/asciitable.md index 10238221..de06fcc0 100644 --- a/docs/parsers/asciitable.md +++ b/docs/parsers/asciitable.md @@ -45,6 +45,9 @@ For example: etc... + Headers (keys) are converted to snake-case. All values are returned as + strings, except empty strings, which are converted to None/null. + Usage (cli): $ cat table.txt | jc --asciitable diff --git a/docs/parsers/asciitable_m.md b/docs/parsers/asciitable_m.md index 2e72cfca..90ae4774 100644 --- a/docs/parsers/asciitable_m.md +++ b/docs/parsers/asciitable_m.md @@ -26,7 +26,8 @@ Cells with multiple lines within rows will be joined with a newline character ('\n'). Headers (keys) are converted to snake-case and newlines between multi-line -headers are joined with an underscore. All values are returned as strings. +headers are joined with an underscore. All values are returned as strings, +except empty strings, which are converted to None/null. Usage (cli): diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index 281f7578..c45805e0 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -40,6 +40,9 @@ For example: etc... + Headers (keys) are converted to snake-case. All values are returned as + strings, except empty strings, which are converted to None/null. + Usage (cli): $ cat table.txt | jc --asciitable diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 63f59a27..c8907868 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -21,7 +21,8 @@ Cells with multiple lines within rows will be joined with a newline character ('\n'). Headers (keys) are converted to snake-case and newlines between multi-line -headers are joined with an underscore. All values are returned as strings. +headers are joined with an underscore. All values are returned as strings, +except empty strings, which are converted to None/null. Usage (cli): From 94e061b881e2ed28d2b462ad038e5ba77182b6c4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 14:53:34 -0700 Subject: [PATCH 71/87] add asciitable parser tests --- tests/test_asciitable.py | 275 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 tests/test_asciitable.py diff --git a/tests/test_asciitable.py b/tests/test_asciitable.py new file mode 100644 index 00000000..e3f4de95 --- /dev/null +++ b/tests/test_asciitable.py @@ -0,0 +1,275 @@ +import os +import unittest +from jc.exceptions import ParseError +import jc.parsers.asciitable + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class MyTests(unittest.TestCase): + + def test_asciitable_nodata(self): + """ + Test 'asciitable' with no data + """ + self.assertEqual(jc.parsers.asciitable.parse('', quiet=True), []) + + def test_asciitable_m_pure_ascii(self): + """ + Test 'asciitable' with a pure ASCII table + """ + input = ''' ++========+========+========+========+========+========+========+ +| type | tota | used | fr ee | shar | buff | avai | + ++========+========+========+========+========+========+========+ +| Mem | 3861 | 2228 | 3364 | 1183 | 2743 | 3389 | ++--------+--------+--------+--------+--------+--------+--------+ +| | | | | test 2 | | | ++--------+--------+--------+--------+--------+--------+--------+ +| last | last | last | ab cde | | | final | ++========+========+========+========+========+========+========+ + ''' + expected = [ + { + "type": "Mem", + "tota": "3861", + "used": "2228", + "fr_ee": "3364", + "shar": "1183", + "buff": "2743", + "avai": "3389" + }, + { + "type": None, + "tota": None, + "used": None, + "fr_ee": None, + "shar": "test 2", + "buff": None, + "avai": None + }, + { + "type": "last", + "tota": "last", + "used": "last", + "fr_ee": "ab cde", + "shar": None, + "buff": None, + "avai": "final" + } + ] + + self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) + + def test_asciitable_m_unicode(self): + """ + Test 'asciitable' with a unicode table + """ + input = ''' +╒════════╤════════╤════════╤════════╤════════╤════════╤════════╕ +│ type │ total │ used │ fr ee │ shar │ buff │ avai │ +╞════════╪════════╪════════╪════════╪════════╪════════╪════════╡ +│ Mem │ 3861 │ 2228 │ 3364 │ 1183 │ 2743 │ 3389 │ +├────────┼────────┼────────┼────────┼────────┼────────┼────────┤ +│ Swap │ 2097 │ 0 │ 2097 │ │ │ │ +├────────┼────────┼────────┼────────┼────────┼────────┼────────┤ +│ last │ last │ last │ ab cde │ │ │ final │ +╘════════╧════════╧════════╧════════╧════════╧════════╧════════╛ + ''' + expected = [ + { + "type": "Mem", + "total": "3861", + "used": "2228", + "fr_ee": "3364", + "shar": "1183", + "buff": "2743", + "avai": "3389" + }, + { + "type": "Swap", + "total": "2097", + "used": "0", + "fr_ee": "2097", + "shar": None, + "buff": None, + "avai": None + }, + { + "type": "last", + "total": "last", + "used": "last", + "fr_ee": "ab cde", + "shar": None, + "buff": None, + "avai": "final" + } + ] + + self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) + + def test_asciitable_pure_ascii_extra_spaces(self): + """ + Test 'asciitable' with a pure ASCII table that has heading and + trailing spaces and newlines. + """ + input = ''' + + + +========+========+========+========+========+========+========+ + | type | total | used | fr ee | shar | buff | avai + +========+========+========+========+========+========+========+ + | Mem | 3861 | 2228 | 3364 | 1183 | 2743 | 3389 + +--------+--------+--------+--------+--------+--------+--------+ + | | | | test 2 | | + +--------+--------+--------+--------+--------+--------+--------+ + | last | last | last | ab cde | | | final | + +========+========+========+========+========+========+========+ + + + ''' + expected = [ + { + "type": "Mem", + "total": "3861", + "used": "2228", + "fr_ee": "3364", + "shar": "1183", + "buff": "2743", + "avai": "3389" + }, + { + "type": None, + "total": None, + "used": None, + "fr_ee": None, + "shar": "test 2", + "buff": None, + "avai": None + }, + { + "type": "last", + "total": "last", + "used": "last", + "fr_ee": "ab cde", + "shar": None, + "buff": None, + "avai": "final" + } + ] + + self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) + + def test_asciitable_unicode_extra_spaces(self): + """ + Test 'asciitable' with a pure ASCII table that has heading and + trailing spaces and newlines. + """ + input = ''' + + + ╒════════╤════════╤════════╤════════╤════════╤════════╤════════╕ + type │ total │ used │ free │ shar │ buff │ avai + ╞════════╪════════╪════════╪════════╪════════╪════════╪════════╡ + Mem │ 3861 │ 2228 │ 3364 │ 1183 │ 2743 │ 3389 + ├────────┼────────┼────────┼────────┼────────┼────────┼────────┤ + Swap │ 2097 │ 0 │ 2097 │ │ │ + ╘════════╧════════╧════════╧════════╧════════╧════════╧════════╛ + + + ''' + expected = [ + { + "type": "Mem", + "total": "3861", + "used": "2228", + "free": "3364", + "shar": "1183", + "buff": "2743", + "avai": "3389" + }, + { + "type": "Swap", + "total": "2097", + "used": "0", + "free": "2097", + "shar": None, + "buff": None, + "avai": None + } + ] + + self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) + + def test_asciitable_markdown(self): + """ + Test 'asciitable' with a markdown table + """ + input = ''' + | type | total | used | free | shared | buff cache | available | + |--------|---------|--------|---------|----------|--------------|-------------| + | Mem | 3861332 | 222820 | 3364176 | 11832 | 274336 | 3389588 | + | Swap | 2097148 | 0 | 2097148 | | | | + ''' + + expected = [ + { + "type": "Mem", + "total": "3861332", + "used": "222820", + "free": "3364176", + "shared": "11832", + "buff_cache": "274336", + "available": "3389588" + }, + { + "type": "Swap", + "total": "2097148", + "used": "0", + "free": "2097148", + "shared": None, + "buff_cache": None, + "available": None + } + ] + + self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) + + def test_asciitable_simple(self): + """ + Test 'asciitable' with a simple table + """ + input = ''' + type total used free shared buff cache available + ------ ------- ------ ------- -------- ------------ ----------- + Mem 3861332 222820 3364176 11832 274336 3389588 + Swap 2097148 0 2097148 + ''' + + expected = [ + { + "type": "Mem", + "total": "3861332", + "used": "222820", + "free": "3364176", + "shared": "11832", + "buff_cache": "274336", + "available": "3389588" + }, + { + "type": "Swap", + "total": "2097148", + "used": "0", + "free": "2097148", + "shared": None, + "buff_cache": None, + "available": None + } + ] + + self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) + + +if __name__ == '__main__': + unittest.main() From cf83e6398b3314b233581eb313510872177de20a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 15:13:07 -0700 Subject: [PATCH 72/87] add fancy separators --- jc/parsers/asciitable.py | 4 ++++ jc/parsers/asciitable_m.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index c45805e0..17624fb0 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -174,6 +174,10 @@ def _is_separator(line: str) -> bool: strip_line.startswith('┌─') and strip_line.endswith('─┐'), strip_line.startswith('├─') and strip_line.endswith('─┤'), strip_line.startswith('└─') and strip_line.endswith('─┘'), + strip_line.startswith('┏━') and strip_line.endswith('━┓'), + strip_line.startswith('┡━') and strip_line.endswith('━┩'), + strip_line.startswith('──') and strip_line.endswith('──'), + strip_line.startswith('══') and strip_line.endswith('══'), strip_line.startswith('+=') and strip_line.endswith('=+'), strip_line.startswith('+-') and strip_line.endswith('-+'), strip_line.startswith('--') and strip_line.endswith('--'), diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index c8907868..a8eb2ddc 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -162,6 +162,7 @@ def _table_sniff(string: str) -> str: if any(( line.startswith('╞═') and line.endswith('═╡'), line.startswith('├─') and line.endswith('─┤'), + line.startswith('┡━') and line.endswith('━┩'), line.startswith('+=') and line.endswith('=+'), line.startswith('+-') and line.endswith('-+') )): @@ -183,6 +184,8 @@ def _is_separator(line: str) -> bool: strip_line.startswith('╒═') and strip_line.endswith('═╕'), strip_line.startswith('╞═') and strip_line.endswith('═╡'), strip_line.startswith('╘═') and strip_line.endswith('═╛'), + strip_line.startswith('┏━') and strip_line.endswith('━┓'), + strip_line.startswith('┡━') and strip_line.endswith('━┩'), strip_line.startswith('┌─') and strip_line.endswith('─┐'), strip_line.startswith('├─') and strip_line.endswith('─┤'), strip_line.startswith('└─') and strip_line.endswith('─┘'), From 17c3c2f029a0a47eb02d681567619d471b99949c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 15:55:59 -0700 Subject: [PATCH 73/87] add bold bar seperator and ANSI code tests --- jc/parsers/asciitable.py | 3 ++- jc/parsers/asciitable_m.py | 2 +- tests/test_asciitable.py | 31 +++++++++++++++++++++++++++++++ tests/test_asciitable_m.py | 24 ++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index 17624fb0..10a1986f 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -208,7 +208,7 @@ def _normalize_rows(table: str) -> List[str]: continue # data row - remove column separators - line = line.replace('│', ' ').replace('|', ' ') + line = line.replace('│', ' ').replace('|', ' ').replace('┃', ' ') result.append(line) result[0] = _snake_case(result[0]) @@ -244,6 +244,7 @@ def parse( if jc.utils.has_data(data): data = _remove_ansi(data) + print(data) data = _strip(data) data_list = _normalize_rows(data) raw_output = _parse_pretty(data_list) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index a8eb2ddc..edb95845 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -204,7 +204,7 @@ def _snake_case(line: str) -> str: def _fixup_separators(line: str) -> str: """Normalize separators, and remove first and last separators""" # normalize separator - line = line.replace('│', '|') + line = line.replace('│', '|').replace('┃', '|') # remove first separator if it is the first char in the line if line[0] == '|': diff --git a/tests/test_asciitable.py b/tests/test_asciitable.py index e3f4de95..a8cdc14f 100644 --- a/tests/test_asciitable.py +++ b/tests/test_asciitable.py @@ -270,6 +270,37 @@ class MyTests(unittest.TestCase): self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) + def test_asciitable_pretty_ansi(self): + """ + Test 'asciitable' with a pretty table with ANSI codes + """ + input = '''┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┓ \n ┃\x1b[1m \x1b[0m\x1b[1mReleased \x1b[0m\x1b[1m \x1b[0m┃\x1b[1m \x1b[0m\x1b[1mTitle \x1b[0m\x1b[1m \x1b[0m┃\x1b[1m \x1b[0m\x1b[1m Box Office\x1b[0m\x1b[1m \x1b[0m┃ \n ┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━┩ \n │\x1b[36m \x1b[0m\x1b[36mDec 20, 2019\x1b[0m\x1b[36m \x1b[0m│\x1b[35m \x1b[0m\x1b[35mStar Wars: The Rise of Skywalker \x1b[0m\x1b[35m \x1b[0m│\x1b[32m \x1b[0m\x1b[32m $952,110,690\x1b[0m\x1b[32m \x1b[0m│ \n │\x1b[36m \x1b[0m\x1b[36mMay 25, 2018\x1b[0m\x1b[36m \x1b[0m│\x1b[35m \x1b[0m\x1b[35mSolo: A Star Wars Story \x1b[0m\x1b[35m \x1b[0m│\x1b[32m \x1b[0m\x1b[32m $393,151,347\x1b[0m\x1b[32m \x1b[0m│ \n │\x1b[36m \x1b[0m\x1b[36mDec 15, 2017\x1b[0m\x1b[36m \x1b[0m│\x1b[35m \x1b[0m\x1b[35mStar Wars Ep. V111: The Last Jedi\x1b[0m\x1b[35m \x1b[0m│\x1b[32m \x1b[0m\x1b[32m$1,332,539,889\x1b[0m\x1b[32m \x1b[0m│ \n │\x1b[36m \x1b[0m\x1b[36mDec 16, 2016\x1b[0m\x1b[36m \x1b[0m│\x1b[35m \x1b[0m\x1b[35mRogue One: A Star Wars Story \x1b[0m\x1b[35m \x1b[0m│\x1b[32m \x1b[0m\x1b[32m$1,332,439,889\x1b[0m\x1b[32m \x1b[0m│ \n └──────────────┴───────────────────────────────────┴────────────────┘ \n''' + + expected = [ + { + "released": "Dec 20, 2019", + "title": "Star Wars: The Rise of Skywalker", + "box_office": "$952,110,690" + }, + { + "released": "May 25, 2018", + "title": "Solo: A Star Wars Story", + "box_office": "$393,151,347" + }, + { + "released": "Dec 15, 2017", + "title": "Star Wars Ep. V111: The Last Jedi", + "box_office": "$1,332,539,889" + }, + { + "released": "Dec 16, 2016", + "title": "Rogue One: A Star Wars Story", + "box_office": "$1,332,439,889" + } + ] + + self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_asciitable_m.py b/tests/test_asciitable_m.py index 22c57f0a..d2c6f483 100644 --- a/tests/test_asciitable_m.py +++ b/tests/test_asciitable_m.py @@ -218,6 +218,30 @@ class MyTests(unittest.TestCase): self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected) + def test_asciitable_m_pretty_ansi(self): + """ + Test 'asciitable-m' with a pretty table with ANSI codes + """ + input = ''' +┏━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┓ +┃\x1b[1m \x1b[0m\x1b[1mReleased \x1b[0m\x1b[1m \x1b[0m┃\x1b[1m \x1b[0m\x1b[1mTitle \x1b[0m\x1b[1m \x1b[0m┃\x1b[1m \x1b[0m\x1b[1m Box Office\x1b[0m\x1b[1m \x1b[0m┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━┩ +│\x1b[36m \x1b[0m\x1b[36mDec 20, 2019\x1b[0m\x1b[36m \x1b[0m│\x1b[35m \x1b[0m\x1b[35mStar Wars: The Rise of Skywalker \x1b[0m\x1b[35m \x1b[0m│\x1b[32m \x1b[0m\x1b[32m $952,110,690\x1b[0m\x1b[32m \x1b[0m│ +│\x1b[36m \x1b[0m\x1b[36mMay 25, 2018\x1b[0m\x1b[36m \x1b[0m│\x1b[35m \x1b[0m\x1b[35mSolo: A Star Wars Story \x1b[0m\x1b[35m \x1b[0m│\x1b[32m \x1b[0m\x1b[32m $393,151,347\x1b[0m\x1b[32m \x1b[0m│ +│\x1b[36m \x1b[0m\x1b[36mDec 15, 2017\x1b[0m\x1b[36m \x1b[0m│\x1b[35m \x1b[0m\x1b[35mStar Wars Ep. V111: The Last Jedi\x1b[0m\x1b[35m \x1b[0m│\x1b[32m \x1b[0m\x1b[32m$1,332,539,889\x1b[0m\x1b[32m \x1b[0m│ +│\x1b[36m \x1b[0m\x1b[36mDec 16, 2016\x1b[0m\x1b[36m \x1b[0m│\x1b[35m \x1b[0m\x1b[35mRogue One: A Star Wars Story \x1b[0m\x1b[35m \x1b[0m│\x1b[32m \x1b[0m\x1b[32m$1,332,439,889\x1b[0m\x1b[32m \x1b[0m│ +└──────────────┴───────────────────────────────────┴────────────────┘ +''' + expected = [ + { + "released": "Dec 20, 2019\nMay 25, 2018\nDec 15, 2017\nDec 16, 2016", + "title": "Star Wars: The Rise of Skywalker\nSolo: A Star Wars Story\nStar Wars Ep. V111: The Last Jedi\nRogue One: A Star Wars Story", + "box_office": "$952,110,690\n$393,151,347\n$1,332,539,889\n$1,332,439,889" + } + ] + + self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected) + def test_asciitable_m_markdown(self): """ Test 'asciitable_m' with a markdown table. Should raise a ParseError From 35e0e9c32afbc5450c661bcb3a23904d135358ad Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 15:57:43 -0700 Subject: [PATCH 74/87] remove print statement --- jc/parsers/asciitable.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index 10a1986f..ce648057 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -244,7 +244,6 @@ def parse( if jc.utils.has_data(data): data = _remove_ansi(data) - print(data) data = _strip(data) data_list = _normalize_rows(data) raw_output = _parse_pretty(data_list) From e5b478218c4ba3781afd085056dba888c3f6ae02 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 16:05:07 -0700 Subject: [PATCH 75/87] add quiet=True to parse tests --- tests/test_xrandr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_xrandr.py b/tests/test_xrandr.py index 9ab8a5fa..8323adb4 100644 --- a/tests/test_xrandr.py +++ b/tests/test_xrandr.py @@ -146,7 +146,7 @@ class XrandrTests(unittest.TestCase): self.maxDiff = None with open("tests/fixtures/generic/xrandr.out", "r") as f: txt = f.read() - actual = parse(txt) + actual = parse(txt, quiet=True) self.assertEqual(1, len(actual["screens"])) self.assertEqual(4, len(actual["unassociated_devices"])) @@ -156,7 +156,7 @@ class XrandrTests(unittest.TestCase): with open("tests/fixtures/generic/xrandr_2.out", "r") as f: txt = f.read() - actual = parse(txt) + actual = parse(txt, quiet=True) self.assertEqual(1, len(actual["screens"])) self.assertEqual(3, len(actual["unassociated_devices"])) @@ -166,7 +166,7 @@ class XrandrTests(unittest.TestCase): with open("tests/fixtures/generic/xrandr_simple.out", "r") as f: txt = f.read() - actual = parse(txt) + actual = parse(txt, quiet=True) with open("tests/fixtures/generic/xrandr_simple.json", "w") as f: json.dump(actual, f, indent=True) From f3aa797d96c8e2cef2124dbb2f49d1fadaee3fa2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 22 Mar 2022 17:47:19 -0700 Subject: [PATCH 76/87] add more pretty table separators --- jc/parsers/asciitable.py | 43 +++++++++++++++++++++++++++++++------- jc/parsers/asciitable_m.py | 35 ++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index ce648057..3c21fa60 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -171,12 +171,34 @@ def _is_separator(line: str) -> bool: strip_line.startswith('╒═') and strip_line.endswith('═╕'), strip_line.startswith('╞═') and strip_line.endswith('═╡'), strip_line.startswith('╘═') and strip_line.endswith('═╛'), + strip_line.startswith('┏━') and strip_line.endswith('━┓'), + strip_line.startswith('┣━') and strip_line.endswith('━┫'), + strip_line.startswith('┗━') and strip_line.endswith('━┛'), + strip_line.startswith('┡━') and strip_line.endswith('━┩'), + strip_line.startswith('┢━') and strip_line.endswith('━┪'), + strip_line.startswith('┟─') and strip_line.endswith('─┧'), + strip_line.startswith('┞─') and strip_line.endswith('─┦'), + strip_line.startswith('┠─') and strip_line.endswith('─┨'), + strip_line.startswith('┝━') and strip_line.endswith('━┥'), + strip_line.startswith('┍━') and strip_line.endswith('━┑'), + strip_line.startswith('┕━') and strip_line.endswith('━┙'), + strip_line.startswith('┎─') and strip_line.endswith('─┒'), + strip_line.startswith('┖─') and strip_line.endswith('─┚'), + strip_line.startswith('╓─') and strip_line.endswith('─╖'), + strip_line.startswith('╟─') and strip_line.endswith('─╢'), + strip_line.startswith('╙─') and strip_line.endswith('─╜'), + strip_line.startswith('╔═') and strip_line.endswith('═╗'), + strip_line.startswith('╠═') and strip_line.endswith('═╣'), + strip_line.startswith('╚═') and strip_line.endswith('═╝'), strip_line.startswith('┌─') and strip_line.endswith('─┐'), strip_line.startswith('├─') and strip_line.endswith('─┤'), strip_line.startswith('└─') and strip_line.endswith('─┘'), - strip_line.startswith('┏━') and strip_line.endswith('━┓'), - strip_line.startswith('┡━') and strip_line.endswith('━┩'), + strip_line.startswith('━━') and strip_line.endswith('━━'), strip_line.startswith('──') and strip_line.endswith('──'), + strip_line.startswith('┄┄') and strip_line.endswith('┄┄'), + strip_line.startswith('┅┅') and strip_line.endswith('┅┅'), + strip_line.startswith('┈┈') and strip_line.endswith('┈┈'), + strip_line.startswith('┉┉') and strip_line.endswith('┉┉'), strip_line.startswith('══') and strip_line.endswith('══'), strip_line.startswith('+=') and strip_line.endswith('=+'), strip_line.startswith('+-') and strip_line.endswith('-+'), @@ -208,17 +230,22 @@ def _normalize_rows(table: str) -> List[str]: continue # data row - remove column separators - line = line.replace('│', ' ').replace('|', ' ').replace('┃', ' ') + line = line.replace('|', ' ')\ + .replace('│', ' ')\ + .replace('┃', ' ')\ + .replace('┆', ' ')\ + .replace('┇', ' ')\ + .replace('┊', ' ')\ + .replace('┋', ' ')\ + .replace('╎', ' ')\ + .replace('╏', ' ')\ + .replace('║', ' ') result.append(line) result[0] = _snake_case(result[0]) return result -def _parse_pretty(table: List[str]) -> List[Dict[str, str]]: - return sparse_table_parse(table) - - def parse( data: str, raw: bool = False, @@ -246,6 +273,6 @@ def parse( data = _remove_ansi(data) data = _strip(data) data_list = _normalize_rows(data) - raw_output = _parse_pretty(data_list) + raw_output = sparse_table_parse(data_list) return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index edb95845..d11f2e5c 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -163,6 +163,14 @@ def _table_sniff(string: str) -> str: line.startswith('╞═') and line.endswith('═╡'), line.startswith('├─') and line.endswith('─┤'), line.startswith('┡━') and line.endswith('━┩'), + line.startswith('┣━') and line.endswith('━┫'), + line.startswith('┢━') and line.endswith('━┪'), + line.startswith('┟─') and line.endswith('─┧'), + line.startswith('┞─') and line.endswith('─┦'), + line.startswith('┠─') and line.endswith('─┨'), + line.startswith('┝━') and line.endswith('━┥'), + line.startswith('╟─') and line.endswith('─╢'), + line.startswith('╠═') and line.endswith('═╣'), line.startswith('+=') and line.endswith('=+'), line.startswith('+-') and line.endswith('-+') )): @@ -185,7 +193,24 @@ def _is_separator(line: str) -> bool: strip_line.startswith('╞═') and strip_line.endswith('═╡'), strip_line.startswith('╘═') and strip_line.endswith('═╛'), strip_line.startswith('┏━') and strip_line.endswith('━┓'), + strip_line.startswith('┣━') and strip_line.endswith('━┫'), + strip_line.startswith('┗━') and strip_line.endswith('━┛'), strip_line.startswith('┡━') and strip_line.endswith('━┩'), + strip_line.startswith('┢━') and strip_line.endswith('━┪'), + strip_line.startswith('┟─') and strip_line.endswith('─┧'), + strip_line.startswith('┞─') and strip_line.endswith('─┦'), + strip_line.startswith('┠─') and strip_line.endswith('─┨'), + strip_line.startswith('┝━') and strip_line.endswith('━┥'), + strip_line.startswith('┍━') and strip_line.endswith('━┑'), + strip_line.startswith('┕━') and strip_line.endswith('━┙'), + strip_line.startswith('┎─') and strip_line.endswith('─┒'), + strip_line.startswith('┖─') and strip_line.endswith('─┚'), + strip_line.startswith('╓─') and strip_line.endswith('─╖'), + strip_line.startswith('╟─') and strip_line.endswith('─╢'), + strip_line.startswith('╙─') and strip_line.endswith('─╜'), + strip_line.startswith('╔═') and strip_line.endswith('═╗'), + strip_line.startswith('╠═') and strip_line.endswith('═╣'), + strip_line.startswith('╚═') and strip_line.endswith('═╝'), strip_line.startswith('┌─') and strip_line.endswith('─┐'), strip_line.startswith('├─') and strip_line.endswith('─┤'), strip_line.startswith('└─') and strip_line.endswith('─┘'), @@ -204,7 +229,15 @@ def _snake_case(line: str) -> str: def _fixup_separators(line: str) -> str: """Normalize separators, and remove first and last separators""" # normalize separator - line = line.replace('│', '|').replace('┃', '|') + line = line.replace('│', '|')\ + .replace('┃', '|')\ + .replace('┆', '|')\ + .replace('┇', '|')\ + .replace('┊', '|')\ + .replace('┋', '|')\ + .replace('╎', '|')\ + .replace('╏', '|')\ + .replace('║', '|') # remove first separator if it is the first char in the line if line[0] == '|': From e66a82ff49b40a674f9ef373a12d59fc71bfa730 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 23 Mar 2022 10:50:34 -0700 Subject: [PATCH 77/87] doc update --- CHANGELOG | 5 +++-- jc/parsers/asciitable_m.py | 4 ++-- man/jc.1 | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 3b15aac3..407d8686 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,11 +1,12 @@ jc changelog -20220309 v1.18.6 (in progress) +20220323 v1.18.6 - Add pidstat command parser tested on linux - Add pidstat command streaming parser tested on linux - Add mpstat command parser tested on linux - Add mpstat command streaming parser tested on linux -- Add ASCII table parser +- Add single-line ASCII and Unicode table parser +- Add multi-line ASCII and Unicode table parser - Add documentation option to parser_info() and all_parser_info() 20220305 v1.18.5 diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index d11f2e5c..5f51b819 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -252,7 +252,7 @@ def _fixup_separators(line: str) -> str: def _normalize_rows(table_lines: Iterable[str]) -> List[Tuple[int, List[str]]]: """ - Return a List of tuples of a row counters and data lines. + Return a List of tuples of row-counters and data lines. """ result = [] header_found = False @@ -293,7 +293,7 @@ def _normalize_rows(table_lines: Iterable[str]) -> List[Tuple[int, List[str]]]: row_counter += 1 continue - # subsequent data row found + # data row found if not _is_separator(line) and header_found and data_found: line = _fixup_separators(line) line_list = line.split('|') diff --git a/man/jc.1 b/man/jc.1 index 7b642f1b..3d16284f 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-03-22 1.18.6 "JSON Convert" +.TH jc 1 2022-03-23 1.18.6 "JSON Convert" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 0a462978b731770d1290187a502706c2322f5f91 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 23 Mar 2022 15:08:33 -0700 Subject: [PATCH 78/87] fix for special characters in headers --- jc/parsers/asciitable.py | 25 ++++++++++++++++++++-- jc/parsers/asciitable_m.py | 8 ++++++- tests/test_asciitable.py | 43 ++++++++++++++++++++++++++++++++++++++ tests/test_asciitable_m.py | 28 +++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 3 deletions(-) diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index 3c21fa60..9346278c 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -211,7 +211,11 @@ def _is_separator(line: str) -> bool: def _snake_case(line: str) -> str: - """replace spaces between words with an underscore and set to lowercase""" + """ + replace spaces between words and special characters with an underscore + and set to lowercase + """ + line = re.sub(r'[^a-zA-Z0-9 ]', '_', line) return re.sub(r'\b \b', '_', line).lower() @@ -246,6 +250,22 @@ def _normalize_rows(table: str) -> List[str]: return result +def _fixup_headers(table: List[Dict]) -> List[Dict]: + """remove consecutive underscores and any trailing underscores""" + new_table = [] + for row in table: + new_row = row.copy() + for k, v in row.items(): + k_new = k + # remove consecutive underscores + k_new = re.sub(r'__+', '_', k_new) + # remove trailing underscores + k_new = re.sub(r'_+$', '', k_new) + new_row[k_new] = new_row.pop(k) + new_table.append(new_row) + + return new_table + def parse( data: str, raw: bool = False, @@ -273,6 +293,7 @@ def parse( data = _remove_ansi(data) data = _strip(data) data_list = _normalize_rows(data) - raw_output = sparse_table_parse(data_list) + raw_table = sparse_table_parse(data_list) + raw_output = _fixup_headers(raw_table) return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 5f51b819..6ad45640 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -222,7 +222,12 @@ def _is_separator(line: str) -> bool: def _snake_case(line: str) -> str: - """replace spaces between words with an underscore and set to lowercase""" + """ + replace spaces between words and special characters with an underscore + and set to lowercase + """ + # must include all column separator characters in regex + line = re.sub(r'[^a-zA-Z0-9 |│┃┆┇┊┋╎╏║]', '_', line) return re.sub(r'\b \b', '_', line).lower() @@ -360,6 +365,7 @@ def _collapse_headers(table: List[List[str]]) -> List[str]: for i, header in enumerate(line): if header: new_header = result[i] + '_' + header + # remove consecutive underscores new_header = re.sub(r'__+', '_', new_header) new_line.append(new_header) else: diff --git a/tests/test_asciitable.py b/tests/test_asciitable.py index a8cdc14f..e8e9967a 100644 --- a/tests/test_asciitable.py +++ b/tests/test_asciitable.py @@ -301,6 +301,49 @@ class MyTests(unittest.TestCase): self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) + def test_asciitable_special_chars_in_header(self): + """ + Test 'asciitable' with a pure ASCII table that has special + characters in the header. These should be converted to underscores + and no trailing or consecutive underscores should end up in the + resulting key names. + """ + input = ''' +Protocol Address Age (min) Hardware Addr Type Interface +Internet 10.12.13.1 98 0950.5785.5cd1 ARPA FastEthernet2.13 +Internet 10.12.13.3 131 0150.7685.14d5 ARPA GigabitEthernet2.13 +Internet 10.12.13.4 198 0950.5C8A.5c41 ARPA GigabitEthernet2.17 + ''' + + expected = [ + { + "protocol": "Internet", + "address": "10.12.13.1", + "age_min": "98", + "hardware_addr": "0950.5785.5cd1", + "type": "ARPA", + "interface": "FastEthernet2.13" + }, + { + "protocol": "Internet", + "address": "10.12.13.3", + "age_min": "131", + "hardware_addr": "0150.7685.14d5", + "type": "ARPA", + "interface": "GigabitEthernet2.13" + }, + { + "protocol": "Internet", + "address": "10.12.13.4", + "age_min": "198", + "hardware_addr": "0950.5C8A.5c41", + "type": "ARPA", + "interface": "GigabitEthernet2.17" + } + ] + + self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_asciitable_m.py b/tests/test_asciitable_m.py index d2c6f483..0b8b0aa9 100644 --- a/tests/test_asciitable_m.py +++ b/tests/test_asciitable_m.py @@ -242,6 +242,34 @@ class MyTests(unittest.TestCase): self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected) + def test_asciitable_m_special_chars_in_header(self): + """ + Test 'asciitable_m' with a pure ASCII table that has special + characters in the header. These should be converted to underscores + and no trailing or consecutive underscores should end up in the + resulting key names. + """ + input = ''' ++----------+------------+-----------+----------------+-------+--------------------+ +| Protocol | Address | Age (min) | Hardware Addr | Type | Interface | +| | | of int | | | | ++----------+------------+-----------+----------------+-------+--------------------+ +| Internet | 10.12.13.1 | 98 | 0950.5785.5cd1 | ARPA | FastEthernet2.13 | ++----------+------------+-----------+----------------+-------+--------------------+ + ''' + expected = [ + { + "protocol": "Internet", + "address": "10.12.13.1", + "age_min_of_int": "98", + "hardware_addr": "0950.5785.5cd1", + "type": "ARPA", + "interface": "FastEthernet2.13" + } + ] + + self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected) + def test_asciitable_m_markdown(self): """ Test 'asciitable_m' with a markdown table. Should raise a ParseError From 6748c3cc91f90bddea3d07b8b30a049ee55eaf35 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Mar 2022 09:31:12 -0700 Subject: [PATCH 79/87] remove lines from corner detection and add rounded corners --- jc/parsers/asciitable.py | 52 ++++++++++++++------------- jc/parsers/asciitable_m.py | 74 +++++++++++++++++++------------------- 2 files changed, 65 insertions(+), 61 deletions(-) diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index 9346278c..de4cb56c 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -168,31 +168,33 @@ def _is_separator(line: str) -> bool: """Returns true if a table separator line is found""" strip_line = line.strip() if any(( - strip_line.startswith('╒═') and strip_line.endswith('═╕'), - strip_line.startswith('╞═') and strip_line.endswith('═╡'), - strip_line.startswith('╘═') and strip_line.endswith('═╛'), - strip_line.startswith('┏━') and strip_line.endswith('━┓'), - strip_line.startswith('┣━') and strip_line.endswith('━┫'), - strip_line.startswith('┗━') and strip_line.endswith('━┛'), - strip_line.startswith('┡━') and strip_line.endswith('━┩'), - strip_line.startswith('┢━') and strip_line.endswith('━┪'), - strip_line.startswith('┟─') and strip_line.endswith('─┧'), - strip_line.startswith('┞─') and strip_line.endswith('─┦'), - strip_line.startswith('┠─') and strip_line.endswith('─┨'), - strip_line.startswith('┝━') and strip_line.endswith('━┥'), - strip_line.startswith('┍━') and strip_line.endswith('━┑'), - strip_line.startswith('┕━') and strip_line.endswith('━┙'), - strip_line.startswith('┎─') and strip_line.endswith('─┒'), - strip_line.startswith('┖─') and strip_line.endswith('─┚'), - strip_line.startswith('╓─') and strip_line.endswith('─╖'), - strip_line.startswith('╟─') and strip_line.endswith('─╢'), - strip_line.startswith('╙─') and strip_line.endswith('─╜'), - strip_line.startswith('╔═') and strip_line.endswith('═╗'), - strip_line.startswith('╠═') and strip_line.endswith('═╣'), - strip_line.startswith('╚═') and strip_line.endswith('═╝'), - strip_line.startswith('┌─') and strip_line.endswith('─┐'), - strip_line.startswith('├─') and strip_line.endswith('─┤'), - strip_line.startswith('└─') and strip_line.endswith('─┘'), + strip_line.startswith('╒') and strip_line.endswith('╕'), + strip_line.startswith('╞') and strip_line.endswith('╡'), + strip_line.startswith('╘') and strip_line.endswith('╛'), + strip_line.startswith('┏') and strip_line.endswith('┓'), + strip_line.startswith('┣') and strip_line.endswith('┫'), + strip_line.startswith('┗') and strip_line.endswith('┛'), + strip_line.startswith('┡') and strip_line.endswith('┩'), + strip_line.startswith('┢') and strip_line.endswith('┪'), + strip_line.startswith('┟') and strip_line.endswith('┧'), + strip_line.startswith('┞') and strip_line.endswith('┦'), + strip_line.startswith('┠') and strip_line.endswith('┨'), + strip_line.startswith('┝') and strip_line.endswith('┥'), + strip_line.startswith('┍') and strip_line.endswith('┑'), + strip_line.startswith('┕') and strip_line.endswith('┙'), + strip_line.startswith('┎') and strip_line.endswith('┒'), + strip_line.startswith('┖') and strip_line.endswith('┚'), + strip_line.startswith('╓') and strip_line.endswith('╖'), + strip_line.startswith('╟') and strip_line.endswith('╢'), + strip_line.startswith('╙') and strip_line.endswith('╜'), + strip_line.startswith('╔') and strip_line.endswith('╗'), + strip_line.startswith('╠') and strip_line.endswith('╣'), + strip_line.startswith('╚') and strip_line.endswith('╝'), + strip_line.startswith('┌') and strip_line.endswith('┐'), + strip_line.startswith('├') and strip_line.endswith('┤'), + strip_line.startswith('└') and strip_line.endswith('┘'), + strip_line.startswith('╭') and strip_line.endswith('╮'), + strip_line.startswith('╰') and strip_line.endswith('╯'), strip_line.startswith('━━') and strip_line.endswith('━━'), strip_line.startswith('──') and strip_line.endswith('──'), strip_line.startswith('┄┄') and strip_line.endswith('┄┄'), diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index 6ad45640..e5f1a914 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -160,17 +160,17 @@ def _table_sniff(string: str) -> str: for line in string.splitlines(): line = line.strip() if any(( - line.startswith('╞═') and line.endswith('═╡'), - line.startswith('├─') and line.endswith('─┤'), - line.startswith('┡━') and line.endswith('━┩'), - line.startswith('┣━') and line.endswith('━┫'), - line.startswith('┢━') and line.endswith('━┪'), - line.startswith('┟─') and line.endswith('─┧'), - line.startswith('┞─') and line.endswith('─┦'), - line.startswith('┠─') and line.endswith('─┨'), - line.startswith('┝━') and line.endswith('━┥'), - line.startswith('╟─') and line.endswith('─╢'), - line.startswith('╠═') and line.endswith('═╣'), + line.startswith('╞') and line.endswith('╡'), + line.startswith('├') and line.endswith('┤'), + line.startswith('┡') and line.endswith('┩'), + line.startswith('┣') and line.endswith('┫'), + line.startswith('┢') and line.endswith('┪'), + line.startswith('┟') and line.endswith('┧'), + line.startswith('┞') and line.endswith('┦'), + line.startswith('┠') and line.endswith('┨'), + line.startswith('┝') and line.endswith('┥'), + line.startswith('╟') and line.endswith('╢'), + line.startswith('╠') and line.endswith('╣'), line.startswith('+=') and line.endswith('=+'), line.startswith('+-') and line.endswith('-+') )): @@ -189,31 +189,33 @@ def _is_separator(line: str) -> bool: """Returns true if a table separator line is found""" strip_line = line.strip() if any(( - strip_line.startswith('╒═') and strip_line.endswith('═╕'), - strip_line.startswith('╞═') and strip_line.endswith('═╡'), - strip_line.startswith('╘═') and strip_line.endswith('═╛'), - strip_line.startswith('┏━') and strip_line.endswith('━┓'), - strip_line.startswith('┣━') and strip_line.endswith('━┫'), - strip_line.startswith('┗━') and strip_line.endswith('━┛'), - strip_line.startswith('┡━') and strip_line.endswith('━┩'), - strip_line.startswith('┢━') and strip_line.endswith('━┪'), - strip_line.startswith('┟─') and strip_line.endswith('─┧'), - strip_line.startswith('┞─') and strip_line.endswith('─┦'), - strip_line.startswith('┠─') and strip_line.endswith('─┨'), - strip_line.startswith('┝━') and strip_line.endswith('━┥'), - strip_line.startswith('┍━') and strip_line.endswith('━┑'), - strip_line.startswith('┕━') and strip_line.endswith('━┙'), - strip_line.startswith('┎─') and strip_line.endswith('─┒'), - strip_line.startswith('┖─') and strip_line.endswith('─┚'), - strip_line.startswith('╓─') and strip_line.endswith('─╖'), - strip_line.startswith('╟─') and strip_line.endswith('─╢'), - strip_line.startswith('╙─') and strip_line.endswith('─╜'), - strip_line.startswith('╔═') and strip_line.endswith('═╗'), - strip_line.startswith('╠═') and strip_line.endswith('═╣'), - strip_line.startswith('╚═') and strip_line.endswith('═╝'), - strip_line.startswith('┌─') and strip_line.endswith('─┐'), - strip_line.startswith('├─') and strip_line.endswith('─┤'), - strip_line.startswith('└─') and strip_line.endswith('─┘'), + strip_line.startswith('╒') and strip_line.endswith('╕'), + strip_line.startswith('╞') and strip_line.endswith('╡'), + strip_line.startswith('╘') and strip_line.endswith('╛'), + strip_line.startswith('┏') and strip_line.endswith('┓'), + strip_line.startswith('┣') and strip_line.endswith('┫'), + strip_line.startswith('┗') and strip_line.endswith('┛'), + strip_line.startswith('┡') and strip_line.endswith('┩'), + strip_line.startswith('┢') and strip_line.endswith('┪'), + strip_line.startswith('┟') and strip_line.endswith('┧'), + strip_line.startswith('┞') and strip_line.endswith('┦'), + strip_line.startswith('┠') and strip_line.endswith('┨'), + strip_line.startswith('┝') and strip_line.endswith('┥'), + strip_line.startswith('┍') and strip_line.endswith('┑'), + strip_line.startswith('┕') and strip_line.endswith('┙'), + strip_line.startswith('┎') and strip_line.endswith('┒'), + strip_line.startswith('┖') and strip_line.endswith('┚'), + strip_line.startswith('╓') and strip_line.endswith('╖'), + strip_line.startswith('╟') and strip_line.endswith('╢'), + strip_line.startswith('╙') and strip_line.endswith('╜'), + strip_line.startswith('╔') and strip_line.endswith('╗'), + strip_line.startswith('╠') and strip_line.endswith('╣'), + strip_line.startswith('╚') and strip_line.endswith('╝'), + strip_line.startswith('┌') and strip_line.endswith('┐'), + strip_line.startswith('├') and strip_line.endswith('┤'), + strip_line.startswith('└') and strip_line.endswith('┘'), + strip_line.startswith('╭') and strip_line.endswith('╮'), + strip_line.startswith('╰') and strip_line.endswith('╯'), strip_line.startswith('+=') and strip_line.endswith('=+'), strip_line.startswith('+-') and strip_line.endswith('-+') )): From 619de68a61be0117e635032347dde417d77c748c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Mar 2022 09:39:53 -0700 Subject: [PATCH 80/87] formatting --- jc/parsers/asciitable.py | 7 ++++--- jc/parsers/asciitable_m.py | 10 ++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index de4cb56c..7b6c848a 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -165,7 +165,7 @@ def _strip(string: str) -> str: def _is_separator(line: str) -> bool: - """Returns true if a table separator line is found""" + """returns true if a table separator line is found""" strip_line = line.strip() if any(( strip_line.startswith('╒') and strip_line.endswith('╕'), @@ -223,7 +223,7 @@ def _snake_case(line: str) -> str: def _normalize_rows(table: str) -> List[str]: """ - Return a List row strings. Header is snake-cased + returns a List of row strings. Header is snake-cased """ result = [] for line in table.splitlines(): @@ -257,7 +257,7 @@ def _fixup_headers(table: List[Dict]) -> List[Dict]: new_table = [] for row in table: new_row = row.copy() - for k, v in row.items(): + for k in row: k_new = k # remove consecutive underscores k_new = re.sub(r'__+', '_', k_new) @@ -268,6 +268,7 @@ def _fixup_headers(table: List[Dict]) -> List[Dict]: return new_table + def parse( data: str, raw: bool = False, diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index e5f1a914..b18e0688 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -155,7 +155,7 @@ def _strip(string: str) -> str: def _table_sniff(string: str) -> str: - """Find the table-type via heuristics""" + """find the table-type via heuristics""" # pretty tables for line in string.splitlines(): line = line.strip() @@ -186,7 +186,7 @@ def _table_sniff(string: str) -> str: def _is_separator(line: str) -> bool: - """Returns true if a table separator line is found""" + """returns true if a table separator line is found""" strip_line = line.strip() if any(( strip_line.startswith('╒') and strip_line.endswith('╕'), @@ -234,7 +234,7 @@ def _snake_case(line: str) -> str: def _fixup_separators(line: str) -> str: - """Normalize separators, and remove first and last separators""" + """normalize separators, and remove first and last separators""" # normalize separator line = line.replace('│', '|')\ .replace('┃', '|')\ @@ -258,9 +258,7 @@ def _fixup_separators(line: str) -> str: def _normalize_rows(table_lines: Iterable[str]) -> List[Tuple[int, List[str]]]: - """ - Return a List of tuples of row-counters and data lines. - """ + """return a List of tuples of row-counters and data lines.""" result = [] header_found = False data_found = False From 5e6a5068cff71b3b30525d0a12868bdf6551456c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Mar 2022 11:57:01 -0700 Subject: [PATCH 81/87] allow iterables for simple table parser --- jc/parsers/universal.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jc/parsers/universal.py b/jc/parsers/universal.py index 81aa14f0..b9f392e2 100644 --- a/jc/parsers/universal.py +++ b/jc/parsers/universal.py @@ -2,7 +2,7 @@ from typing import Iterable, List, Dict -def simple_table_parse(data: List[str]) -> List[Dict]: +def simple_table_parse(data: Iterable[str]) -> List[Dict]: """ Parse simple tables. There should be no blank cells. The last column may contain data with spaces. @@ -22,7 +22,7 @@ def simple_table_parse(data: List[str]) -> List[Dict]: Parameters: - data: (list) Text data to parse that has been split into lines + data: (iter) Text data to parse that has been split into lines via .splitlines(). Item 0 must be the header row. Any spaces in header names should be changed to underscore '_'. You should also ensure headers are @@ -37,6 +37,10 @@ def simple_table_parse(data: List[str]) -> List[Dict]: """ # code adapted from Conor Heine at: # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 + + # cast iterable to a list. Also keeps from mutating the caller's list + data = list(data) + headers = [h for h in ' '.join(data[0].strip().split()).split() if h] raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), data[1:]) raw_output = [dict(zip(headers, r)) for r in raw_data] From 17df5bfcfc25f226a2976aa7ab8a80e4aaa915c3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Mar 2022 11:58:13 -0700 Subject: [PATCH 82/87] cache _is_separator function --- jc/parsers/asciitable.py | 10 +++++++++- jc/parsers/asciitable_m.py | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index 7b6c848a..1f174c38 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -38,6 +38,12 @@ For example: good day 12345 hi there abc def 3.14 + or + + foo bar baz + good day 12345 + hi there abc def 3.14 + etc... Headers (keys) are converted to snake-case. All values are returned as @@ -103,6 +109,7 @@ Examples: ] """ import re +from functools import lru_cache from typing import List, Dict import jc.utils from jc.parsers.universal import sparse_table_parse @@ -163,9 +170,10 @@ def _strip(string: str) -> str: string = _rstrip(string) return string - +@lru_cache(maxsize=32) def _is_separator(line: str) -> bool: """returns true if a table separator line is found""" + # This function is cacheable since tables have identical separators strip_line = line.strip() if any(( strip_line.startswith('╒') and strip_line.endswith('╕'), diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index b18e0688..fc13f7f7 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -93,6 +93,7 @@ Examples: ] """ import re +from functools import lru_cache from typing import Iterable, Tuple, List, Dict, Optional import jc.utils from jc.exceptions import ParseError @@ -184,9 +185,10 @@ def _table_sniff(string: str) -> str: # simple tables return 'simple' - +@lru_cache(maxsize=32) def _is_separator(line: str) -> bool: """returns true if a table separator line is found""" + # This function is cacheable since tables have identical separators strip_line = line.strip() if any(( strip_line.startswith('╒') and strip_line.endswith('╕'), From 978caf45221188a29c0e67147804f20f317c066d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Mar 2022 12:37:46 -0700 Subject: [PATCH 83/87] minor optimization by reordering expressions --- jc/parsers/asciitable.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/jc/parsers/asciitable.py b/jc/parsers/asciitable.py index 1f174c38..52881c01 100644 --- a/jc/parsers/asciitable.py +++ b/jc/parsers/asciitable.py @@ -176,6 +176,18 @@ def _is_separator(line: str) -> bool: # This function is cacheable since tables have identical separators strip_line = line.strip() if any(( + strip_line.startswith('|-') and strip_line.endswith('-|'), + strip_line.startswith('━━') and strip_line.endswith('━━'), + strip_line.startswith('──') and strip_line.endswith('──'), + strip_line.startswith('┄┄') and strip_line.endswith('┄┄'), + strip_line.startswith('┅┅') and strip_line.endswith('┅┅'), + strip_line.startswith('┈┈') and strip_line.endswith('┈┈'), + strip_line.startswith('┉┉') and strip_line.endswith('┉┉'), + strip_line.startswith('══') and strip_line.endswith('══'), + strip_line.startswith('--') and strip_line.endswith('--'), + strip_line.startswith('==') and strip_line.endswith('=='), + strip_line.startswith('+=') and strip_line.endswith('=+'), + strip_line.startswith('+-') and strip_line.endswith('-+'), strip_line.startswith('╒') and strip_line.endswith('╕'), strip_line.startswith('╞') and strip_line.endswith('╡'), strip_line.startswith('╘') and strip_line.endswith('╛'), @@ -202,19 +214,7 @@ def _is_separator(line: str) -> bool: strip_line.startswith('├') and strip_line.endswith('┤'), strip_line.startswith('└') and strip_line.endswith('┘'), strip_line.startswith('╭') and strip_line.endswith('╮'), - strip_line.startswith('╰') and strip_line.endswith('╯'), - strip_line.startswith('━━') and strip_line.endswith('━━'), - strip_line.startswith('──') and strip_line.endswith('──'), - strip_line.startswith('┄┄') and strip_line.endswith('┄┄'), - strip_line.startswith('┅┅') and strip_line.endswith('┅┅'), - strip_line.startswith('┈┈') and strip_line.endswith('┈┈'), - strip_line.startswith('┉┉') and strip_line.endswith('┉┉'), - strip_line.startswith('══') and strip_line.endswith('══'), - strip_line.startswith('+=') and strip_line.endswith('=+'), - strip_line.startswith('+-') and strip_line.endswith('-+'), - strip_line.startswith('--') and strip_line.endswith('--'), - strip_line.startswith('==') and strip_line.endswith('=='), - strip_line.startswith('|-') and strip_line.endswith('-|') + strip_line.startswith('╰') and strip_line.endswith('╯') )): return True return False From 304ae6268f4dff6e6f901628e7ff89db1d6e0767 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Mar 2022 16:58:39 -0700 Subject: [PATCH 84/87] minor optimization by changing the expression order --- jc/parsers/asciitable_m.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jc/parsers/asciitable_m.py b/jc/parsers/asciitable_m.py index fc13f7f7..8ff3eed1 100644 --- a/jc/parsers/asciitable_m.py +++ b/jc/parsers/asciitable_m.py @@ -272,11 +272,11 @@ def _normalize_rows(table_lines: Iterable[str]) -> List[Tuple[int, List[str]]]: continue # skip top table frame - if _is_separator(line) and not header_found and not data_found: + if not header_found and not data_found and _is_separator(line): continue # first header row found - if not _is_separator(line) and not header_found and not data_found: + if not header_found and not data_found and not _is_separator(line): header_found = True line = _snake_case(line) line = _fixup_separators(line) @@ -286,7 +286,7 @@ def _normalize_rows(table_lines: Iterable[str]) -> List[Tuple[int, List[str]]]: continue # subsequent header row found - if not _is_separator(line) and header_found and not data_found: + if header_found and not data_found and not _is_separator(line): line = _snake_case(line) line = _fixup_separators(line) line_list = line.split('|') @@ -295,13 +295,13 @@ def _normalize_rows(table_lines: Iterable[str]) -> List[Tuple[int, List[str]]]: continue # table separator found - this is a header separator - if _is_separator(line) and header_found and not data_found: + if header_found and not data_found and _is_separator(line): data_found = True row_counter += 1 continue # data row found - if not _is_separator(line) and header_found and data_found: + if header_found and data_found and not _is_separator(line): line = _fixup_separators(line) line_list = line.split('|') line_list = [x.strip() for x in line_list] @@ -309,7 +309,7 @@ def _normalize_rows(table_lines: Iterable[str]) -> List[Tuple[int, List[str]]]: continue # table separator found - this is a data separator - if _is_separator(line) and header_found and data_found: + if header_found and data_found and _is_separator(line): row_counter += 1 continue From 38de059a1bc285c8681086d9fb43d0d23ac01d49 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 24 Mar 2022 16:58:45 -0700 Subject: [PATCH 85/87] doc update --- docs/parsers/asciitable.md | 6 ++++++ docs/parsers/universal.md | 4 ++-- man/jc.1 | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/parsers/asciitable.md b/docs/parsers/asciitable.md index de06fcc0..7fd59e0c 100644 --- a/docs/parsers/asciitable.md +++ b/docs/parsers/asciitable.md @@ -43,6 +43,12 @@ For example: good day 12345 hi there abc def 3.14 + or + + foo bar baz + good day 12345 + hi there abc def 3.14 + etc... Headers (keys) are converted to snake-case. All values are returned as diff --git a/docs/parsers/universal.md b/docs/parsers/universal.md index 40889af5..da8eddd0 100644 --- a/docs/parsers/universal.md +++ b/docs/parsers/universal.md @@ -15,7 +15,7 @@ jc - JSON Convert universal parsers ### simple\_table\_parse ```python -def simple_table_parse(data: List[str]) -> List[Dict] +def simple_table_parse(data: Iterable[str]) -> List[Dict] ``` Parse simple tables. There should be no blank cells. The last column @@ -36,7 +36,7 @@ Example Table: Parameters: - data: (list) Text data to parse that has been split into lines + data: (iter) Text data to parse that has been split into lines via .splitlines(). Item 0 must be the header row. Any spaces in header names should be changed to underscore '_'. You should also ensure headers are diff --git a/man/jc.1 b/man/jc.1 index 3d16284f..bb90e889 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-03-23 1.18.6 "JSON Convert" +.TH jc 1 2022-03-24 1.18.6 "JSON Convert" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 34bc7753174be28d6c1a00dadc662bc374e554c1 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Mar 2022 15:32:11 -0700 Subject: [PATCH 86/87] doc update --- man/jc.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/jc.1 b/man/jc.1 index bb90e889..1be3e8e5 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-03-24 1.18.6 "JSON Convert" +.TH jc 1 2022-03-25 1.18.6 "JSON Convert" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 789f0735dfd63e728b8716d97cf0c6045ede5011 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 25 Mar 2022 15:32:37 -0700 Subject: [PATCH 87/87] doc update --- CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 407d8686..e81410ec 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,6 @@ jc changelog -20220323 v1.18.6 +20220325 v1.18.6 - Add pidstat command parser tested on linux - Add pidstat command streaming parser tested on linux - Add mpstat command parser tested on linux