From e42af3353e60dace8ef2d6bce4295b72b0daee17 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 1 Oct 2023 11:25:56 -0700 Subject: [PATCH] fix pidstat parsers for -T ALL option --- CHANGELOG | 2 +- docs/parsers/pidstat.md | 5 +- docs/parsers/pidstat_s.md | 6 +- jc/parsers/pidstat.py | 61 +++++++++------ jc/parsers/pidstat_s.py | 74 ++++++++++++------- man/jc.1 | 2 +- .../generic/pidstat-ht-streaming.json | 1 + tests/fixtures/generic/pidstat-ht.json | 1 + tests/fixtures/generic/pidstat-ht.out | 8 ++ tests/test_pidstat.py | 12 +++ tests/test_pidstat_s.py | 12 +++ 11 files changed, 129 insertions(+), 55 deletions(-) create mode 100644 tests/fixtures/generic/pidstat-ht-streaming.json create mode 100644 tests/fixtures/generic/pidstat-ht.json create mode 100644 tests/fixtures/generic/pidstat-ht.out diff --git a/CHANGELOG b/CHANGELOG index c666dcba..93be592c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,7 +3,7 @@ jc changelog 20230930 v1.23.5 - Add `host` command parser - Add `nsd-control` command parser -- TODO: Fix `pidstat` command parser when using `-T ALL` +- Fix `pidstat` command parser when using `-T ALL` - Fix `x509-cert` parser to allow negative serial numbers - Fix `x509-cert` parser for cases when bitstrings are larger than standard - Fix `xrandr` command parser for associated device issues diff --git a/docs/parsers/pidstat.md b/docs/parsers/pidstat.md index 46ca1ea6..526a42b7 100644 --- a/docs/parsers/pidstat.md +++ b/docs/parsers/pidstat.md @@ -45,6 +45,9 @@ Schema: "kb_ccwr_s": float, "cswch_s": float, "nvcswch_s": float, + "usr_ms": integer, + "system_ms": integer, + "guest_ms": integer, "command": string } ] @@ -148,4 +151,4 @@ Returns: ### Parser Information Compatibility: linux -Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/pidstat_s.md b/docs/parsers/pidstat_s.md index f3639c16..61aa043d 100644 --- a/docs/parsers/pidstat_s.md +++ b/docs/parsers/pidstat_s.md @@ -39,6 +39,7 @@ Schema: "percent_usr": float, "percent_system": float, "percent_guest": float, + "percent_wait": float, "percent_cpu": float, "cpu": integer, "minflt_s": float, @@ -53,6 +54,9 @@ Schema: "kb_ccwr_s": float, "cswch_s": float, "nvcswch_s": float, + "usr_ms": integer, + "system_ms": integer, + "guest_ms": integer, "command": string, # below object only exists if using -qq or ignore_exceptions=True @@ -107,4 +111,4 @@ Returns: ### Parser Information Compatibility: linux -Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/pidstat.py b/jc/parsers/pidstat.py index 4227607e..29ac3aea 100644 --- a/jc/parsers/pidstat.py +++ b/jc/parsers/pidstat.py @@ -40,6 +40,9 @@ Schema: "kb_ccwr_s": float, "cswch_s": float, "nvcswch_s": float, + "usr_ms": integer, + "system_ms": integer, + "guest_ms": integer, "command": string } ] @@ -128,7 +131,7 @@ from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.2' + version = '1.3' description = '`pidstat -H` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -152,11 +155,16 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = {'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'} + int_list = { + 'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref', + 'usr_ms', 'system_ms', 'guest_ms' + } - float_list = {'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu', - 'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s', - 'kb_ccwr_s', 'cswch_s', 'nvcswch_s', 'percent_wait'} + float_list = { + 'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu', + 'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s', + 'kb_ccwr_s', 'cswch_s', 'nvcswch_s', 'percent_wait' + } for entry in proc_data: for key in entry: @@ -169,6 +177,14 @@ def _process(proc_data: List[Dict]) -> List[Dict]: return proc_data +def normalize_header(header: str) -> str: + return header.replace('#', ' ')\ + .replace('-', '_')\ + .replace('/', '_')\ + .replace('%', 'percent_')\ + .lower() + + def parse( data: str, raw: bool = False, @@ -191,29 +207,28 @@ def parse( jc.utils.input_type_check(data) raw_output: List = [] + table_list: List = [] + header_found = False if jc.utils.has_data(data): - # check for line starting with # as the start of the table data_list = list(filter(None, data.splitlines())) - for line in data_list.copy(): - if line.startswith('#'): - break - else: - data_list.pop(0) - if not data_list: + for line in data_list: + if line.startswith('#'): + header_found = True + if len(table_list) > 1: + raw_output.extend(simple_table_parse(table_list)) + table_list = [normalize_header(line)] + continue + + if header_found: + table_list.append(line) + + if len(table_list) > 1: + raw_output.extend(simple_table_parse(table_list)) + + if not header_found: raise ParseError('Could not parse pidstat output. Make sure to use "pidstat -h".') - # normalize header - data_list[0] = data_list[0].replace('#', ' ')\ - .replace('/', '_')\ - .replace('%', 'percent_')\ - .lower() - - # remove remaining header lines (e.g. pidstat -H 2 5) - data_list = [i for i in data_list if not i.startswith('#')] - - raw_output = simple_table_parse(data_list) - return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/pidstat_s.py b/jc/parsers/pidstat_s.py index dbaee0d6..c347a13a 100644 --- a/jc/parsers/pidstat_s.py +++ b/jc/parsers/pidstat_s.py @@ -34,6 +34,7 @@ Schema: "percent_usr": float, "percent_system": float, "percent_guest": float, + "percent_wait": float, "percent_cpu": float, "cpu": integer, "minflt_s": float, @@ -48,6 +49,9 @@ Schema: "kb_ccwr_s": float, "cswch_s": float, "nvcswch_s": float, + "usr_ms": integer, + "system_ms": integer, + "guest_ms": integer, "command": string, # below object only exists if using -qq or ignore_exceptions=True @@ -72,7 +76,7 @@ Examples: {"time":"1646859134","uid":"0","pid":"9","percent_usr":"0.00","perc...} ... """ -from typing import Dict, Iterable, Union +from typing import List, Dict, Iterable, Union import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -83,7 +87,7 @@ from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.1' + version = '1.2' description = '`pidstat -H` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -107,11 +111,16 @@ def _process(proc_data: Dict) -> Dict: Dictionary. Structured data to conform to the schema. """ - int_list = {'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'} + int_list = { + 'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref', + 'usr_ms', 'system_ms', 'guest_ms' + } - float_list = {'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu', - 'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s', - 'kb_ccwr_s', 'cswch_s', 'nvcswch_s'} + float_list = { + 'percent_usr', 'percent_system', 'percent_guest', 'percent_wait', + 'percent_cpu', 'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', + 'kb_wr_s', 'kb_ccwr_s', 'cswch_s', 'nvcswch_s' + } for key in proc_data: if key in int_list: @@ -123,6 +132,14 @@ def _process(proc_data: Dict) -> Dict: return proc_data +def normalize_header(header: str) -> str: + return header.replace('#', ' ')\ + .replace('-', '_')\ + .replace('/', '_')\ + .replace('%', 'percent_')\ + .lower() + + @add_jc_meta def parse( data: Iterable[str], @@ -149,8 +166,8 @@ def parse( jc.utils.compatibility(__name__, info.compatible, quiet) streaming_input_type_check(data) - found_first_hash = False - header = '' + table_list: List = [] + header: str = '' for line in data: try: @@ -161,29 +178,30 @@ def parse( # skip blank lines continue - if not line.startswith('#') and not found_first_hash: - # skip preamble lines before header row + if line.startswith('#'): + if len(table_list) > 1: + output_line = simple_table_parse(table_list)[0] + yield output_line if raw else _process(output_line) + header = '' + + header = normalize_header(line) + table_list = [header] continue - if line.startswith('#') and not found_first_hash: - # normalize header - header = line.replace('#', ' ')\ - .replace('/', '_')\ - .replace('%', 'percent_')\ - .lower() - found_first_hash = True - continue - - if line.startswith('#') and found_first_hash: - # skip header lines after first one is found - continue - - output_line = simple_table_parse([header, line])[0] - - if output_line: + if header: + table_list.append(line) + output_line = simple_table_parse(table_list)[0] yield output_line if raw else _process(output_line) - else: - raise ParseError('Not pidstat data') + table_list = [header] + continue except Exception as e: yield raise_or_yield(ignore_exceptions, e, line) + + try: + if len(table_list) > 1: + output_line = simple_table_parse(table_list)[0] + yield output_line if raw else _process(output_line) + + except Exception as e: + yield raise_or_yield(ignore_exceptions, e, str(table_list)) diff --git a/man/jc.1 b/man/jc.1 index d6981936..382b51cc 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2023-09-30 1.23.5 "JSON Convert" +.TH jc 1 2023-10-01 1.23.5 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings diff --git a/tests/fixtures/generic/pidstat-ht-streaming.json b/tests/fixtures/generic/pidstat-ht-streaming.json new file mode 100644 index 00000000..57be15b3 --- /dev/null +++ b/tests/fixtures/generic/pidstat-ht-streaming.json @@ -0,0 +1 @@ +[{"time":1692579199,"uid":0,"pid":1,"percent_usr":0.0,"percent_system":0.07,"percent_guest":0.0,"percent_wait":0.08,"percent_cpu":0.08,"cpu":0,"command":"systemd"},{"time":1692579199,"uid":0,"pid":1,"usr_ms":2890,"system_ms":4260,"guest_ms":0,"command":"systemd"}] diff --git a/tests/fixtures/generic/pidstat-ht.json b/tests/fixtures/generic/pidstat-ht.json new file mode 100644 index 00000000..57be15b3 --- /dev/null +++ b/tests/fixtures/generic/pidstat-ht.json @@ -0,0 +1 @@ +[{"time":1692579199,"uid":0,"pid":1,"percent_usr":0.0,"percent_system":0.07,"percent_guest":0.0,"percent_wait":0.08,"percent_cpu":0.08,"cpu":0,"command":"systemd"},{"time":1692579199,"uid":0,"pid":1,"usr_ms":2890,"system_ms":4260,"guest_ms":0,"command":"systemd"}] diff --git a/tests/fixtures/generic/pidstat-ht.out b/tests/fixtures/generic/pidstat-ht.out new file mode 100644 index 00000000..b655c7de --- /dev/null +++ b/tests/fixtures/generic/pidstat-ht.out @@ -0,0 +1,8 @@ +Linux 4.18.0-477.10.1.el8_8.x86_64 (localhost.localdomain) 08/20/2023 _x86_64_ (1 CPU) + +# Time UID PID %usr %system %guest %wait %CPU CPU Command +1692579199 0 1 0.00 0.07 0.00 0.08 0.08 0 systemd + +# Time UID PID usr-ms system-ms guest-ms Command +1692579199 0 1 2890 4260 0 systemd + diff --git a/tests/test_pidstat.py b/tests/test_pidstat.py index 7544d0f3..ea97cc61 100644 --- a/tests/test_pidstat.py +++ b/tests/test_pidstat.py @@ -22,6 +22,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out'), 'r', encoding='utf-8') as f: centos_7_7_pidstat_hdlrsuw_2_5 = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht.out'), 'r', encoding='utf-8') as f: + generic_pidstat_ht = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hl.json'), 'r', encoding='utf-8') as f: centos_7_7_pidstat_hl_json = json.loads(f.read()) @@ -32,6 +35,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.json'), 'r', encoding='utf-8') as f: centos_7_7_pidstat_hdlrsuw_2_5_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht.json'), 'r', encoding='utf-8') as f: + generic_pidstat_ht_json = json.loads(f.read()) + def test_pidstat_nodata(self): """ @@ -63,6 +69,12 @@ class MyTests(unittest.TestCase): """ self.assertEqual(jc.parsers.pidstat.parse(self.centos_7_7_pidstat_hdlrsuw_2_5, quiet=True), self.centos_7_7_pidstat_hdlrsuw_2_5_json) + def test_pidstat_ht(self): + """ + Test 'pidstat -hT' + """ + self.assertEqual(jc.parsers.pidstat.parse(self.generic_pidstat_ht, quiet=True), self.generic_pidstat_ht_json) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_pidstat_s.py b/tests/test_pidstat_s.py index fce3156d..a474755f 100644 --- a/tests/test_pidstat_s.py +++ b/tests/test_pidstat_s.py @@ -24,6 +24,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out'), 'r', encoding='utf-8') as f: centos_7_7_pidstat_hdlrsuw_2_5 = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht.out'), 'r', encoding='utf-8') as f: + generic_pidstat_ht = f.read() + # output with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hl-streaming.json'), 'r', encoding='utf-8') as f: centos_7_7_pidstat_hl_streaming_json = json.loads(f.read()) @@ -34,6 +37,9 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5-streaming.json'), 'r', encoding='utf-8') as f: centos_7_7_pidstat_hdlrsuw_2_5_streaming_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht-streaming.json'), 'r', encoding='utf-8') as f: + generic_pidstat_ht_streaming_json = json.loads(f.read()) + def test_pidstat_s_nodata(self): """ @@ -65,6 +71,12 @@ class MyTests(unittest.TestCase): """ self.assertEqual(list(jc.parsers.pidstat_s.parse(self.centos_7_7_pidstat_hdlrsuw_2_5.splitlines(), quiet=True)), self.centos_7_7_pidstat_hdlrsuw_2_5_streaming_json) + def test_pidstat_s_ht(self): + """ + Test 'pidstat -hT' + """ + self.assertEqual(list(jc.parsers.pidstat_s.parse(self.generic_pidstat_ht.splitlines(), quiet=True)), self.generic_pidstat_ht_streaming_json) + if __name__ == '__main__': unittest.main()