python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-25 00:37:31 +02:00
Code Issues Releases Activity

fix pidstat parsers for -T ALL option

Browse Source
This commit is contained in:
Kelly Brazil
2023-10-01 11:25:56 -07:00
parent 4ec2b16f42
commit e42af3353e
11 changed files with 129 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG
View File

@ -3,7 +3,7 @@ jc changelog
20230930 v1.23.5
- Add `host` command parser
- Add `nsd-control` command parser
- TODO: Fix `pidstat` command parser when using `-T ALL`
- Fix `pidstat` command parser when using `-T ALL`
- Fix `x509-cert` parser to allow negative serial numbers
- Fix `x509-cert` parser for cases when bitstrings are larger than standard
- Fix `xrandr` command parser for associated device issues

5
docs/parsers/pidstat.md
View File

@ -45,6 +45,9 @@ Schema:
"kb_ccwr_s": float,
"cswch_s": float,
"nvcswch_s": float,
"usr_ms": integer,
"system_ms": integer,
"guest_ms": integer,
"command": string
}
]
@ -148,4 +151,4 @@ Returns:
### Parser Information
Compatibility: linux
Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)

6
docs/parsers/pidstat_s.md
View File

@ -39,6 +39,7 @@ Schema:
"percent_usr": float,
"percent_system": float,
"percent_guest": float,
"percent_wait": float,
"percent_cpu": float,
"cpu": integer,
"minflt_s": float,
@ -53,6 +54,9 @@ Schema:
"kb_ccwr_s": float,
"cswch_s": float,
"nvcswch_s": float,
"usr_ms": integer,
"system_ms": integer,
"guest_ms": integer,
"command": string,
# below object only exists if using -qq or ignore_exceptions=True
@ -107,4 +111,4 @@ Returns:
### Parser Information
Compatibility: linux
Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

61
jc/parsers/pidstat.py
View File

@ -40,6 +40,9 @@ Schema:
"kb_ccwr_s": float,
"cswch_s": float,
"nvcswch_s": float,
"usr_ms": integer,
"system_ms": integer,
"guest_ms": integer,
"command": string
}
]
@ -128,7 +131,7 @@ from jc.exceptions import ParseError
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.2'
version = '1.3'
description = '`pidstat -H` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -152,11 +155,16 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
List of Dictionaries. Structured to conform to the schema.
"""
int_list = {'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'}
int_list = {
'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref',
'usr_ms', 'system_ms', 'guest_ms'
}
float_list = {'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu',
'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s',
'kb_ccwr_s', 'cswch_s', 'nvcswch_s', 'percent_wait'}
float_list = {
'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu',
'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s',
'kb_ccwr_s', 'cswch_s', 'nvcswch_s', 'percent_wait'
}
for entry in proc_data:
for key in entry:
@ -169,6 +177,14 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
return proc_data
def normalize_header(header: str) -> str:
return header.replace('#', ' ')\
.replace('-', '_')\
.replace('/', '_')\
.replace('%', 'percent_')\
.lower()
def parse(
data: str,
raw: bool = False,
@ -191,29 +207,28 @@ def parse(
jc.utils.input_type_check(data)
raw_output: List = []
table_list: List = []
header_found = False
if jc.utils.has_data(data):
# check for line starting with # as the start of the table
data_list = list(filter(None, data.splitlines()))
for line in data_list.copy():
if line.startswith('#'):
break
else:
data_list.pop(0)
if not data_list:
for line in data_list:
if line.startswith('#'):
header_found = True
if len(table_list) > 1:
raw_output.extend(simple_table_parse(table_list))
table_list = [normalize_header(line)]
continue
if header_found:
table_list.append(line)
if len(table_list) > 1:
raw_output.extend(simple_table_parse(table_list))
if not header_found:
raise ParseError('Could not parse pidstat output. Make sure to use "pidstat -h".')
# normalize header
data_list[0] = data_list[0].replace('#', ' ')\
.replace('/', '_')\
.replace('%', 'percent_')\
.lower()
# remove remaining header lines (e.g. pidstat -H 2 5)
data_list = [i for i in data_list if not i.startswith('#')]
raw_output = simple_table_parse(data_list)
return raw_output if raw else _process(raw_output)

74
jc/parsers/pidstat_s.py
View File

@ -34,6 +34,7 @@ Schema:
"percent_usr": float,
"percent_system": float,
"percent_guest": float,
"percent_wait": float,
"percent_cpu": float,
"cpu": integer,
"minflt_s": float,
@ -48,6 +49,9 @@ Schema:
"kb_ccwr_s": float,
"cswch_s": float,
"nvcswch_s": float,
"usr_ms": integer,
"system_ms": integer,
"guest_ms": integer,
"command": string,
# below object only exists if using -qq or ignore_exceptions=True
@ -72,7 +76,7 @@ Examples:
{"time":"1646859134","uid":"0","pid":"9","percent_usr":"0.00","perc...}
...
"""
from typing import Dict, Iterable, Union
from typing import List, Dict, Iterable, Union
import jc.utils
from jc.streaming import (
add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield
@ -83,7 +87,7 @@ from jc.exceptions import ParseError
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.1'
version = '1.2'
description = '`pidstat -H` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -107,11 +111,16 @@ def _process(proc_data: Dict) -> Dict:
Dictionary. Structured data to conform to the schema.
"""
int_list = {'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'}
int_list = {
'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref',
'usr_ms', 'system_ms', 'guest_ms'
}
float_list = {'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu',
'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s',
'kb_ccwr_s', 'cswch_s', 'nvcswch_s'}
float_list = {
'percent_usr', 'percent_system', 'percent_guest', 'percent_wait',
'percent_cpu', 'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s',
'kb_wr_s', 'kb_ccwr_s', 'cswch_s', 'nvcswch_s'
}
for key in proc_data:
if key in int_list:
@ -123,6 +132,14 @@ def _process(proc_data: Dict) -> Dict:
return proc_data
def normalize_header(header: str) -> str:
return header.replace('#', ' ')\
.replace('-', '_')\
.replace('/', '_')\
.replace('%', 'percent_')\
.lower()
@add_jc_meta
def parse(
data: Iterable[str],
@ -149,8 +166,8 @@ def parse(
jc.utils.compatibility(__name__, info.compatible, quiet)
streaming_input_type_check(data)
found_first_hash = False
header = ''
table_list: List = []
header: str = ''
for line in data:
try:
@ -161,29 +178,30 @@ def parse(
# skip blank lines
continue
if not line.startswith('#') and not found_first_hash:
# skip preamble lines before header row
if line.startswith('#'):
if len(table_list) > 1:
output_line = simple_table_parse(table_list)[0]
yield output_line if raw else _process(output_line)
header = ''
header = normalize_header(line)
table_list = [header]
continue
if line.startswith('#') and not found_first_hash:
# normalize header
header = line.replace('#', ' ')\
.replace('/', '_')\
.replace('%', 'percent_')\
.lower()
found_first_hash = True
continue
if line.startswith('#') and found_first_hash:
# skip header lines after first one is found
continue
output_line = simple_table_parse([header, line])[0]
if output_line:
if header:
table_list.append(line)
output_line = simple_table_parse(table_list)[0]
yield output_line if raw else _process(output_line)
else:
raise ParseError('Not pidstat data')
table_list = [header]
continue
except Exception as e:
yield raise_or_yield(ignore_exceptions, e, line)
try:
if len(table_list) > 1:
output_line = simple_table_parse(table_list)[0]
yield output_line if raw else _process(output_line)
except Exception as e:
yield raise_or_yield(ignore_exceptions, e, str(table_list))

2
man/jc.1
View File

@ -1,4 +1,4 @@
.TH jc 1 2023-09-30 1.23.5 "JSON Convert"
.TH jc 1 2023-10-01 1.23.5 "JSON Convert"
.SH NAME
\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types,
and strings

1
tests/fixtures/generic/pidstat-ht-streaming.json vendored Normal file
View File

@ -0,0 +1 @@
[{"time":1692579199,"uid":0,"pid":1,"percent_usr":0.0,"percent_system":0.07,"percent_guest":0.0,"percent_wait":0.08,"percent_cpu":0.08,"cpu":0,"command":"systemd"},{"time":1692579199,"uid":0,"pid":1,"usr_ms":2890,"system_ms":4260,"guest_ms":0,"command":"systemd"}]

1
tests/fixtures/generic/pidstat-ht.json vendored Normal file
View File

@ -0,0 +1 @@
[{"time":1692579199,"uid":0,"pid":1,"percent_usr":0.0,"percent_system":0.07,"percent_guest":0.0,"percent_wait":0.08,"percent_cpu":0.08,"cpu":0,"command":"systemd"},{"time":1692579199,"uid":0,"pid":1,"usr_ms":2890,"system_ms":4260,"guest_ms":0,"command":"systemd"}]

8
tests/fixtures/generic/pidstat-ht.out vendored Normal file
View File

@ -0,0 +1,8 @@
Linux 4.18.0-477.10.1.el8_8.x86_64 (localhost.localdomain) 08/20/2023 _x86_64_ (1 CPU)
# Time UID PID %usr %system %guest %wait %CPU CPU Command
1692579199 0 1 0.00 0.07 0.00 0.08 0.08 0 systemd
# Time UID PID usr-ms system-ms guest-ms Command
1692579199 0 1 2890 4260 0 systemd

12
tests/test_pidstat.py
View File

@ -22,6 +22,9 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out'), 'r', encoding='utf-8') as f:
centos_7_7_pidstat_hdlrsuw_2_5 = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht.out'), 'r', encoding='utf-8') as f:
generic_pidstat_ht = f.read()
# output
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hl.json'), 'r', encoding='utf-8') as f:
centos_7_7_pidstat_hl_json = json.loads(f.read())
@ -32,6 +35,9 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.json'), 'r', encoding='utf-8') as f:
centos_7_7_pidstat_hdlrsuw_2_5_json = json.loads(f.read())
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht.json'), 'r', encoding='utf-8') as f:
generic_pidstat_ht_json = json.loads(f.read())
def test_pidstat_nodata(self):
"""
@ -63,6 +69,12 @@ class MyTests(unittest.TestCase):
"""
self.assertEqual(jc.parsers.pidstat.parse(self.centos_7_7_pidstat_hdlrsuw_2_5, quiet=True), self.centos_7_7_pidstat_hdlrsuw_2_5_json)
def test_pidstat_ht(self):
"""
Test 'pidstat -hT'
"""
self.assertEqual(jc.parsers.pidstat.parse(self.generic_pidstat_ht, quiet=True), self.generic_pidstat_ht_json)
if __name__ == '__main__':
unittest.main()

12
tests/test_pidstat_s.py
View File

@ -24,6 +24,9 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out'), 'r', encoding='utf-8') as f:
centos_7_7_pidstat_hdlrsuw_2_5 = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht.out'), 'r', encoding='utf-8') as f:
generic_pidstat_ht = f.read()
# output
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hl-streaming.json'), 'r', encoding='utf-8') as f:
centos_7_7_pidstat_hl_streaming_json = json.loads(f.read())
@ -34,6 +37,9 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5-streaming.json'), 'r', encoding='utf-8') as f:
centos_7_7_pidstat_hdlrsuw_2_5_streaming_json = json.loads(f.read())
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht-streaming.json'), 'r', encoding='utf-8') as f:
generic_pidstat_ht_streaming_json = json.loads(f.read())
def test_pidstat_s_nodata(self):
"""
@ -65,6 +71,12 @@ class MyTests(unittest.TestCase):
"""
self.assertEqual(list(jc.parsers.pidstat_s.parse(self.centos_7_7_pidstat_hdlrsuw_2_5.splitlines(), quiet=True)), self.centos_7_7_pidstat_hdlrsuw_2_5_streaming_json)
def test_pidstat_s_ht(self):
"""
Test 'pidstat -hT'
"""
self.assertEqual(list(jc.parsers.pidstat_s.parse(self.generic_pidstat_ht.splitlines(), quiet=True)), self.generic_pidstat_ht_streaming_json)
if __name__ == '__main__':
unittest.main()