fix pidstat parsers for -T ALL option

CHANGELOG

@@ -3,7 +3,7 @@ jc changelog
 20230930 v1.23.5
 - Add `host` command parser
 - Add `nsd-control` command parser
-- TODO: Fix `pidstat` command parser when using `-T ALL`
+- Fix `pidstat` command parser when using `-T ALL`
 - Fix `x509-cert` parser to allow negative serial numbers
 - Fix `x509-cert` parser for cases when bitstrings are larger than standard
 - Fix `xrandr` command parser for associated device issues

docs/parsers/pidstat.md

@@ -45,6 +45,9 @@ Schema:
         "kb_ccwr_s":        float,
         "cswch_s":          float,
         "nvcswch_s":        float,
+        "usr_ms":           integer,
+        "system_ms":        integer,
+        "guest_ms":         integer,
         "command":          string
       }
     ]
@@ -148,4 +151,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/pidstat_s.md

@@ -39,6 +39,7 @@ Schema:
       "percent_usr":      float,
       "percent_system":   float,
       "percent_guest":    float,
+      "percent_wait":     float,
       "percent_cpu":      float,
       "cpu":              integer,
       "minflt_s":         float,
@@ -53,6 +54,9 @@ Schema:
       "kb_ccwr_s":        float,
       "cswch_s":          float,
       "nvcswch_s":        float,
+      "usr_ms":           integer,
+      "system_ms":        integer,
+      "guest_ms":         integer,
       "command":          string,
 
       # below object only exists if using -qq or ignore_exceptions=True
@@ -107,4 +111,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

jc/parsers/pidstat.py

@@ -40,6 +40,9 @@ Schema:
         "kb_ccwr_s":        float,
         "cswch_s":          float,
         "nvcswch_s":        float,
+        "usr_ms":           integer,
+        "system_ms":        integer,
+        "guest_ms":         integer,
         "command":          string
       }
     ]
@@ -128,7 +131,7 @@ from jc.exceptions import ParseError
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.2'
+    version = '1.3'
     description = '`pidstat -H` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -152,11 +155,16 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
 
         List of Dictionaries. Structured to conform to the schema.
     """
-    int_list = {'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'}
+    int_list = {
+        'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref',
+        'usr_ms', 'system_ms', 'guest_ms'
+    }
 
-    float_list = {'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu',
+    float_list = {
-                  'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s',
+        'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu',
-                  'kb_ccwr_s', 'cswch_s', 'nvcswch_s', 'percent_wait'}
+        'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s',
+        'kb_ccwr_s', 'cswch_s', 'nvcswch_s', 'percent_wait'
+    }
 
     for entry in proc_data:
         for key in entry:
@@ -169,6 +177,14 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
     return proc_data
 
 
+def normalize_header(header: str) -> str:
+    return header.replace('#', ' ')\
+                 .replace('-', '_')\
+                 .replace('/', '_')\
+                 .replace('%', 'percent_')\
+                 .lower()
+
+
 def parse(
     data: str,
     raw: bool = False,
@@ -191,29 +207,28 @@ def parse(
     jc.utils.input_type_check(data)
 
     raw_output: List = []
+    table_list: List = []
+    header_found = False
 
     if jc.utils.has_data(data):
 
-        # check for line starting with # as the start of the table
         data_list = list(filter(None, data.splitlines()))
-        for line in data_list.copy():
-            if line.startswith('#'):
-                break
-            else:
-                data_list.pop(0)
 
-        if not data_list:
+        for line in data_list:
+            if line.startswith('#'):
+                header_found = True
+                if len(table_list) > 1:
+                    raw_output.extend(simple_table_parse(table_list))
+                table_list = [normalize_header(line)]
+                continue
+
+            if header_found:
+                table_list.append(line)
+
+        if len(table_list) > 1:
+            raw_output.extend(simple_table_parse(table_list))
+
+        if not header_found:
             raise ParseError('Could not parse pidstat output. Make sure to use "pidstat -h".')
 
-        # normalize header
-        data_list[0] = data_list[0].replace('#', ' ')\
-                                   .replace('/', '_')\
-                                   .replace('%', 'percent_')\
-                                   .lower()
-
-        # remove remaining header lines (e.g. pidstat -H 2 5)
-        data_list = [i for i in data_list if not i.startswith('#')]
-
-        raw_output = simple_table_parse(data_list)
-
     return raw_output if raw else _process(raw_output)

jc/parsers/pidstat_s.py

@@ -34,6 +34,7 @@ Schema:
       "percent_usr":      float,
       "percent_system":   float,
       "percent_guest":    float,
+      "percent_wait":     float,
       "percent_cpu":      float,
       "cpu":              integer,
       "minflt_s":         float,
@@ -48,6 +49,9 @@ Schema:
       "kb_ccwr_s":        float,
       "cswch_s":          float,
       "nvcswch_s":        float,
+      "usr_ms":           integer,
+      "system_ms":        integer,
+      "guest_ms":         integer,
       "command":          string,
 
       # below object only exists if using -qq or ignore_exceptions=True
@@ -72,7 +76,7 @@ Examples:
     {"time":"1646859134","uid":"0","pid":"9","percent_usr":"0.00","perc...}
     ...
 """
-from typing import Dict, Iterable, Union
+from typing import List, Dict, Iterable, Union
 import jc.utils
 from jc.streaming import (
     add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield
@@ -83,7 +87,7 @@ from jc.exceptions import ParseError
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.1'
+    version = '1.2'
     description = '`pidstat -H` command streaming parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -107,11 +111,16 @@ def _process(proc_data: Dict) -> Dict:
 
         Dictionary. Structured data to conform to the schema.
     """
-    int_list = {'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'}
+    int_list = {
+        'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref',
+        'usr_ms', 'system_ms', 'guest_ms'
+    }
 
-    float_list = {'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu',
+    float_list = {
-                  'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s',
+        'percent_usr', 'percent_system', 'percent_guest', 'percent_wait',
-                  'kb_ccwr_s', 'cswch_s', 'nvcswch_s'}
+        'percent_cpu', 'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s',
+        'kb_wr_s', 'kb_ccwr_s', 'cswch_s', 'nvcswch_s'
+    }
 
     for key in proc_data:
         if key in int_list:
@@ -123,6 +132,14 @@ def _process(proc_data: Dict) -> Dict:
     return proc_data
 
 
+def normalize_header(header: str) -> str:
+    return header.replace('#', ' ')\
+                 .replace('-', '_')\
+                 .replace('/', '_')\
+                 .replace('%', 'percent_')\
+                 .lower()
+
+
 @add_jc_meta
 def parse(
     data: Iterable[str],
@@ -149,8 +166,8 @@ def parse(
     jc.utils.compatibility(__name__, info.compatible, quiet)
     streaming_input_type_check(data)
 
-    found_first_hash = False
+    table_list: List = []
-    header = ''
+    header: str = ''
 
     for line in data:
         try:
@@ -161,29 +178,30 @@ def parse(
                 # skip blank lines
                 continue
 
-            if not line.startswith('#') and not found_first_hash:
+            if line.startswith('#'):
-                # skip preamble lines before header row
+                if len(table_list) > 1:
+                    output_line = simple_table_parse(table_list)[0]
+                    yield output_line if raw else _process(output_line)
+                    header = ''
+
+                header = normalize_header(line)
+                table_list = [header]
                 continue
 
-            if line.startswith('#') and not found_first_hash:
+            if header:
-                # normalize header
+                table_list.append(line)
-                header = line.replace('#', ' ')\
+                output_line = simple_table_parse(table_list)[0]
-                             .replace('/', '_')\
-                             .replace('%', 'percent_')\
-                             .lower()
-                found_first_hash = True
-                continue
-
-            if line.startswith('#') and found_first_hash:
-                # skip header lines after first one is found
-                continue
-
-            output_line = simple_table_parse([header, line])[0]
-
-            if output_line:
                 yield output_line if raw else _process(output_line)
-            else:
+                table_list = [header]
-                raise ParseError('Not pidstat data')
+                continue
 
         except Exception as e:
             yield raise_or_yield(ignore_exceptions, e, line)
+
+    try:
+        if len(table_list) > 1:
+            output_line = simple_table_parse(table_list)[0]
+            yield output_line if raw else _process(output_line)
+
+    except Exception as e:
+        yield raise_or_yield(ignore_exceptions, e, str(table_list))

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2023-09-30 1.23.5 "JSON Convert"
+.TH jc 1 2023-10-01 1.23.5 "JSON Convert"
 .SH NAME
 \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types,
 and strings

tests/fixtures/generic/pidstat-ht-streaming.json

@@ -0,0 +1 @@
+[{"time":1692579199,"uid":0,"pid":1,"percent_usr":0.0,"percent_system":0.07,"percent_guest":0.0,"percent_wait":0.08,"percent_cpu":0.08,"cpu":0,"command":"systemd"},{"time":1692579199,"uid":0,"pid":1,"usr_ms":2890,"system_ms":4260,"guest_ms":0,"command":"systemd"}]

tests/fixtures/generic/pidstat-ht.json

@@ -0,0 +1 @@
+[{"time":1692579199,"uid":0,"pid":1,"percent_usr":0.0,"percent_system":0.07,"percent_guest":0.0,"percent_wait":0.08,"percent_cpu":0.08,"cpu":0,"command":"systemd"},{"time":1692579199,"uid":0,"pid":1,"usr_ms":2890,"system_ms":4260,"guest_ms":0,"command":"systemd"}]

tests/fixtures/generic/pidstat-ht.out

@@ -0,0 +1,8 @@
+Linux 4.18.0-477.10.1.el8_8.x86_64 (localhost.localdomain) 	08/20/2023 	_x86_64_	(1 CPU)
+
+# Time        UID       PID    %usr %system  %guest   %wait    %CPU   CPU  Command
+1692579199      0         1    0.00    0.07    0.00    0.08    0.08     0  systemd
+
+# Time        UID       PID    usr-ms system-ms  guest-ms  Command
+1692579199      0         1      2890      4260         0  systemd
+

tests/test_pidstat.py

@@ -22,6 +22,9 @@ class MyTests(unittest.TestCase):
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out'), 'r', encoding='utf-8') as f:
         centos_7_7_pidstat_hdlrsuw_2_5 = f.read()
 
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht.out'), 'r', encoding='utf-8') as f:
+        generic_pidstat_ht = f.read()
+
     # output
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hl.json'), 'r', encoding='utf-8') as f:
         centos_7_7_pidstat_hl_json = json.loads(f.read())
@@ -32,6 +35,9 @@ class MyTests(unittest.TestCase):
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.json'), 'r', encoding='utf-8') as f:
         centos_7_7_pidstat_hdlrsuw_2_5_json = json.loads(f.read())
 
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht.json'), 'r', encoding='utf-8') as f:
+        generic_pidstat_ht_json = json.loads(f.read())
+
 
     def test_pidstat_nodata(self):
         """
@@ -63,6 +69,12 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(jc.parsers.pidstat.parse(self.centos_7_7_pidstat_hdlrsuw_2_5, quiet=True), self.centos_7_7_pidstat_hdlrsuw_2_5_json)
 
+    def test_pidstat_ht(self):
+        """
+        Test 'pidstat -hT'
+        """
+        self.assertEqual(jc.parsers.pidstat.parse(self.generic_pidstat_ht, quiet=True), self.generic_pidstat_ht_json)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_pidstat_s.py

@@ -24,6 +24,9 @@ class MyTests(unittest.TestCase):
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5.out'), 'r', encoding='utf-8') as f:
         centos_7_7_pidstat_hdlrsuw_2_5 = f.read()
 
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht.out'), 'r', encoding='utf-8') as f:
+        generic_pidstat_ht = f.read()
+
     # output
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hl-streaming.json'), 'r', encoding='utf-8') as f:
         centos_7_7_pidstat_hl_streaming_json = json.loads(f.read())
@@ -34,6 +37,9 @@ class MyTests(unittest.TestCase):
     with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/pidstat-hdlrsuw-2-5-streaming.json'), 'r', encoding='utf-8') as f:
         centos_7_7_pidstat_hdlrsuw_2_5_streaming_json = json.loads(f.read())
 
+    with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/pidstat-ht-streaming.json'), 'r', encoding='utf-8') as f:
+        generic_pidstat_ht_streaming_json = json.loads(f.read())
+
 
     def test_pidstat_s_nodata(self):
         """
@@ -65,6 +71,12 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(list(jc.parsers.pidstat_s.parse(self.centos_7_7_pidstat_hdlrsuw_2_5.splitlines(), quiet=True)), self.centos_7_7_pidstat_hdlrsuw_2_5_streaming_json)
 
+    def test_pidstat_s_ht(self):
+        """
+        Test 'pidstat -hT'
+        """
+        self.assertEqual(list(jc.parsers.pidstat_s.parse(self.generic_pidstat_ht.splitlines(), quiet=True)), self.generic_pidstat_ht_streaming_json)
+
 
 if __name__ == '__main__':
     unittest.main()