From c693c868cacbcd81f44fd383719723dd7a137c87 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 11 Mar 2022 15:59:38 -0800 Subject: [PATCH] add mpstat streaming parser --- README.md | 1 + docs/parsers/mpstat.md | 2 + docs/parsers/mpstat_s.md | 130 ++++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/mpstat.py | 2 + jc/parsers/mpstat_s.py | 211 +++++++++++++++++++++++++++++++++++++++ man/jc.1 | 5 + 7 files changed, 352 insertions(+) create mode 100644 docs/parsers/mpstat_s.md create mode 100644 jc/parsers/mpstat_s.py diff --git a/README.md b/README.md index 21bf10c6..46316b3d 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,7 @@ option. - `--lsusb` enables the `lsusb` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/lsusb)) - `--mount` enables the `mount` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/mount)) - `--mpstat` enables the `mpstat` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat)) +- `--mpstat-s` enables the `mpstat` command streaming parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/mpstat_s)) - `--netstat` enables the `netstat` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/netstat)) - `--nmcli` enables the `nmcli` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/nmcli)) - `--ntpq` enables the `ntpq -p` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ntpq)) diff --git a/docs/parsers/mpstat.md b/docs/parsers/mpstat.md index 588b7612..a6653e40 100644 --- a/docs/parsers/mpstat.md +++ b/docs/parsers/mpstat.md @@ -5,6 +5,8 @@ jc - JSON Convert `mpstat` command output parser +Note: Latest versions of `mpstat` support JSON output (v11.5.1+) + Usage (cli): $ mpstat | jc --mpstat diff --git a/docs/parsers/mpstat_s.md b/docs/parsers/mpstat_s.md new file mode 100644 index 00000000..25fe3b03 --- /dev/null +++ b/docs/parsers/mpstat_s.md @@ -0,0 +1,130 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.mpstat\_s + +jc - JSON Convert `mpstat` command output streaming parser + +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) + +Note: Latest versions of `mpstat` support JSON output (v11.5.1+) + +Usage (cli): + + $ mpstat | jc --mpstat-s + +Usage (module): + + import jc + + result = jc.parse('mpstat_s', mpstat_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "type": string, + "time": string, + "cpu": string, + "average": boolean, + "percent_usr": float, + "percent_nice": float, + "percent_sys": float, + "percent_iowait": float, + "percent_irq": float, + "percent_soft": float, + "percent_steal": float, + "percent_guest": float, + "percent_gnice": float, + "percent_idle": float, + "intr_s": float, + "_s": float, # is an integer + "nmi_s": float, + "loc_s": float, + "spu_s": float, + "pmi_s": float, + "iwi_s": float, + "rtr_s": float, + "res_s": float, + "cal_s": float, + "tlb_s": float, + "trm_s": float, + "thr_s": float, + "dfr_s": float, + "mce_s": float, + "mcp_s": float, + "err_s": float, + "mis_s": float, + "pin_s": float, + "npi_s": float, + "piw_s": float, + "hi_s": float, + "timer_s": float, + "net_tx_s": float, + "net_rx_s": float, + "block_s": float, + "block_iopoll_s": float, + "tasklet_s": float, + "sched_s": float, + "hrtimer_s": float, + "rcu_s": float, + + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + +Examples: + + $ mpstat -A | jc --mpstat-s + {"cpu":"all","percent_usr":0.22,"percent_nice":0.0,"percent_sys":...} + {"cpu":"0","percent_usr":0.22,"percent_nice":0.0,"percent_sys":0....} + {"cpu":"all","intr_s":37.61,"type":"interrupts","time":"03:15:06 PM"} + ... + + $ mpstat | jc --mpstat-s -r + {"cpu":"all","percent_usr":"0.22","percent_nice":"0.00","percent_...} + {"cpu":"0","percent_usr":"0.22","percent_nice":"0.00","percent_sy...} + {"cpu":"all","intr_s":"37.61","type":"interrupts","time":"03:15:06 PM"} + ... + + + +### parse + +```python +@add_jc_meta +def parse(data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] +``` + +Main text parsing generator function. Returns an iterator object. + +Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + +Yields: + + Dictionary. Raw or processed structured data. + +Returns: + + Iterator object (generator) + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index f9f03e3f..e2944fa4 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -60,6 +60,7 @@ parsers = [ 'lsusb', 'mount', 'mpstat', + 'mpstat-s', 'netstat', 'nmcli', 'ntpq', diff --git a/jc/parsers/mpstat.py b/jc/parsers/mpstat.py index e3f92a3e..93764713 100644 --- a/jc/parsers/mpstat.py +++ b/jc/parsers/mpstat.py @@ -1,5 +1,7 @@ """jc - JSON Convert `mpstat` command output parser +Note: Latest versions of `mpstat` support JSON output (v11.5.1+) + Usage (cli): $ mpstat | jc --mpstat diff --git a/jc/parsers/mpstat_s.py b/jc/parsers/mpstat_s.py new file mode 100644 index 00000000..9c5d7c0c --- /dev/null +++ b/jc/parsers/mpstat_s.py @@ -0,0 +1,211 @@ +"""jc - JSON Convert `mpstat` command output streaming parser + +> This streaming parser outputs JSON Lines (cli) or returns a Generator + iterator of Dictionaries (module) + +Note: Latest versions of `mpstat` support JSON output (v11.5.1+) + +Usage (cli): + + $ mpstat | jc --mpstat-s + +Usage (module): + + import jc + + result = jc.parse('mpstat_s', mpstat_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "type": string, + "time": string, + "cpu": string, + "average": boolean, + "percent_usr": float, + "percent_nice": float, + "percent_sys": float, + "percent_iowait": float, + "percent_irq": float, + "percent_soft": float, + "percent_steal": float, + "percent_guest": float, + "percent_gnice": float, + "percent_idle": float, + "intr_s": float, + "_s": float, # is an integer + "nmi_s": float, + "loc_s": float, + "spu_s": float, + "pmi_s": float, + "iwi_s": float, + "rtr_s": float, + "res_s": float, + "cal_s": float, + "tlb_s": float, + "trm_s": float, + "thr_s": float, + "dfr_s": float, + "mce_s": float, + "mcp_s": float, + "err_s": float, + "mis_s": float, + "pin_s": float, + "npi_s": float, + "piw_s": float, + "hi_s": float, + "timer_s": float, + "net_tx_s": float, + "net_rx_s": float, + "block_s": float, + "block_iopoll_s": float, + "tasklet_s": float, + "sched_s": float, + "hrtimer_s": float, + "rcu_s": float, + + # below object only exists if using -qq or ignore_exceptions=True + "_jc_meta": { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + +Examples: + + $ mpstat -A | jc --mpstat-s + {"cpu":"all","percent_usr":0.22,"percent_nice":0.0,"percent_sys":...} + {"cpu":"0","percent_usr":0.22,"percent_nice":0.0,"percent_sys":0....} + {"cpu":"all","intr_s":37.61,"type":"interrupts","time":"03:15:06 PM"} + ... + + $ mpstat | jc --mpstat-s -r + {"cpu":"all","percent_usr":"0.22","percent_nice":"0.00","percent_...} + {"cpu":"0","percent_usr":"0.22","percent_nice":"0.00","percent_sy...} + {"cpu":"all","intr_s":"37.61","type":"interrupts","time":"03:15:06 PM"} + ... +""" +from typing import Dict, Iterable, Union +import jc.utils +from jc.parsers.universal import simple_table_parse +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) +from jc.exceptions import ParseError + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`mpstat` command streaming parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + streaming = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured data to conform to the schema. + """ + float_list = [ + "percent_usr", "percent_nice", "percent_sys", "percent_iowait", "percent_irq", + "percent_soft", "percent_steal", "percent_guest", "percent_gnice", "percent_idle", "intr_s", + "nmi_s", "loc_s", "spu_s", "pmi_s", "iwi_s", "rtr_s", "res_s", "cal_s", "tlb_s", "trm_s", + "thr_s", "dfr_s", "mce_s", "mcp_s", "err_s", "mis_s", "pin_s", "npi_s", "piw_s", "hi_s", + "timer_s", "net_tx_s", "net_rx_s", "block_s", "block_iopoll_s", "tasklet_s", "sched_s", + "hrtimer_s", "rcu_s" + ] + for key in proc_data: + if (key in float_list or (key[0].isdigit() and key.endswith('_s'))): + proc_data[key] = jc.utils.convert_to_float(proc_data[key]) + + return proc_data + + +@add_jc_meta +def parse( + data: Iterable[str], + raw: bool = False, + quiet: bool = False, + ignore_exceptions: bool = False +) -> Union[Iterable[Dict], tuple]: + """ + Main text parsing generator function. Returns an iterator object. + + Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + + Yields: + + Dictionary. Raw or processed structured data. + + Returns: + + Iterator object (generator) + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + streaming_input_type_check(data) + + header_found: bool = False + + for line in data: + try: + streaming_line_input_type_check(line) + + # skip blank lines + if not line.strip(): + continue + + output_line: Dict = {} + + # check for header, normalize it, and fix the time column + if ' CPU ' in line: + header_found = True + if '%usr' in line: + stat_type = 'cpu' + else: + stat_type = 'interrupts' + + header_text: str = line.replace('/', '_')\ + .replace('%', 'percent_')\ + .lower() + header_start = line.find('CPU ') + header_text = header_text[header_start:] + continue + + # data line - pull time from beginning and then parse as a table + if header_found: + output_line = simple_table_parse([header_text, line[header_start:]])[0] + output_line['type'] = stat_type + item_time = line[:header_start].strip() + if 'Average:' not in item_time: + output_line['time'] = line[:header_start].strip() + else: + output_line['average'] = True + + if output_line: + yield output_line if raw else _process(output_line) + + except Exception as e: + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/man/jc.1 b/man/jc.1 index 40ce7249..820be574 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -257,6 +257,11 @@ Key/Value file parser \fB--mpstat\fP `mpstat` command parser +.TP +.B +\fB--mpstat-s\fP +`mpstat` command streaming parser + .TP .B \fB--netstat\fP