add simple summary support to rsync and rsync-s parsers

docs/parsers/rsync.md

@@ -159,4 +159,4 @@ Compatibility:  linux, darwin, freebsd
 
 Source: [`jc/parsers/rsync.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/rsync.py)
 
-Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/rsync_s.md

@@ -116,4 +116,4 @@ Compatibility:  linux, darwin, freebsd
 
 Source: [`jc/parsers/rsync_s.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/rsync_s.py)
 
-Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)

jc/parsers/rsync.py

@@ -131,13 +131,13 @@ Examples:
 """
 import re
 from copy import deepcopy
-from typing import List, Dict
+from typing import List, Dict, Union
 import jc.utils
 
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.1'
+    version = '1.2'
     description = '`rsync` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -171,10 +171,26 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
     for item in proc_data:
         for key in item['summary']:
             if key in int_list:
-                item['summary'][key] = jc.utils.convert_to_int(item['summary'][key])
+                item['summary'][key] = jc.utils.convert_size_to_int(item['summary'][key])
 
             if key in float_list:
-                item['summary'][key] = jc.utils.convert_to_float(item['summary'][key])
+                converted_val: Union[float, None] = None
+                val = item['summary'][key]
+                if any([
+                    'K' in val,
+                    'M' in val,
+                    'G' in val,
+                    'T' in val
+                ]):
+                    converted_int_val = jc.utils.convert_size_to_int(val)
+
+                    if not converted_int_val is None:
+                        converted_val = float(converted_int_val)
+
+                else:
+                    converted_val = jc.utils.convert_to_float(val)
+
+                item['summary'][key] = converted_val
 
         for entry in item['files']:
             for key in entry:
@@ -311,6 +327,9 @@ def parse(
     stat1_line_re = re.compile(r'(sent)\s+(?P<sent>[0-9,]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.]+)\s+(bytes/sec)')
     stat2_line_re = re.compile(r'(total size is)\s+(?P<total_size>[0-9,]+)\s+(speedup is)\s+(?P<speedup>[0-9,.]+)')
 
+    stat1_line_simple_re = re.compile(r'(sent)\s+(?P<sent>[0-9,.TGMK]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,.TGMK]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.TGMK]+)\s+(bytes/sec)')
+    stat2_line_simple_re = re.compile(r'(total\s+size\s+is)\s+(?P<total_size>[0-9,.TGMK]+)\s+(speedup\s+is)\s+(?P<speedup>[0-9,.TGMK]+)')
+
     file_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][u.+ ?][a.+ ?][x.+ ?]) (?P<name>.+)')
     file_line_log_mac_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][x.+ ?]) (?P<name>.+)')
     stat_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+sent\s+(?P<sent>[\d,]+)\s+bytes\s+received\s+(?P<received>[\d,]+)\s+bytes\s+total\s+size\s+(?P<total_size>[\d,]+)')
@@ -445,6 +464,21 @@ def parse(
                 rsync_run['summary']['speedup'] = stat2_line.group('speedup')
                 continue
 
+            stat1_line_simple = stat1_line_simple_re.match(line)
+            if stat1_line_simple:
+                rsync_run['summary'] = {
+                    'sent': stat1_line_simple.group('sent'),
+                    'received': stat1_line_simple.group('received'),
+                    'bytes_sec': stat1_line_simple.group('bytes_sec')
+                }
+                continue
+
+            stat2_line_simple = stat2_line_simple_re.match(line)
+            if stat2_line_simple:
+                rsync_run['summary']['total_size'] = stat2_line_simple.group('total_size')
+                rsync_run['summary']['speedup'] = stat2_line_simple.group('speedup')
+                continue
+
             stat_line_log = stat_line_log_re.match(line)
             if stat_line_log:
                 rsync_run['summary'] = {

jc/parsers/rsync_s.py

@@ -88,7 +88,7 @@ from jc.streaming import (
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.2'
+    version = '1.3'
     description = '`rsync` command streaming parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -121,10 +121,26 @@ def _process(proc_data: Dict) -> Dict:
 
     for key in proc_data.copy():
         if key in int_list:
-            proc_data[key] = jc.utils.convert_to_int(proc_data[key])
+            proc_data[key] = jc.utils.convert_size_to_int(proc_data[key])
 
         if key in float_list:
-            proc_data[key] = jc.utils.convert_to_float(proc_data[key])
+            converted_val: Union[float, None] = None
+            val = proc_data[key]
+            if any([
+                'K' in val,
+                'M' in val,
+                'G' in val,
+                'T' in val
+            ]):
+                converted_int_val = jc.utils.convert_size_to_int(val)
+
+                if not converted_int_val is None:
+                    converted_val = float(converted_int_val)
+
+            else:
+                converted_val = jc.utils.convert_to_float(val)
+
+            proc_data[key] = converted_val
 
         # add timestamp
         if 'date' in proc_data and 'time' in proc_data:
@@ -254,6 +270,9 @@ def parse(
     stat1_line_re = re.compile(r'(sent)\s+(?P<sent>[0-9,]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.]+)\s+(bytes/sec)')
     stat2_line_re = re.compile(r'(total size is)\s+(?P<total_size>[0-9,]+)\s+(speedup is)\s+(?P<speedup>[0-9,.]+)')
 
+    stat1_line_simple_re = re.compile(r'(sent)\s+(?P<sent>[0-9,.TGMK]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,.TGMK]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.TGMK]+)\s+(bytes/sec)')
+    stat2_line_simple_re = re.compile(r'(total\s+size\s+is)\s+(?P<total_size>[0-9,.TGMK]+)\s+(speedup\s+is)\s+(?P<speedup>[0-9,.TGMK]+)')
+
     file_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][u.+ ?][a.+ ?][x.+ ?]) (?P<name>.+)')
     file_line_log_mac_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][x.+ ?]) (?P<name>.+)')
     stat_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+sent\s+(?P<sent>[\d,]+)\s+bytes\s+received\s+(?P<received>[\d,]+)\s+bytes\s+total\s+size\s+(?P<total_size>[\d,]+)')
@@ -403,6 +422,22 @@ def parse(
                 summary['speedup'] = stat2_line.group('speedup')
                 continue
 
+            stat1_line_simple = stat1_line_simple_re.match(line)
+            if stat1_line_simple:
+                summary = {
+                    'type': 'summary',
+                    'sent': stat1_line_simple.group('sent'),
+                    'received': stat1_line_simple.group('received'),
+                    'bytes_sec': stat1_line_simple.group('bytes_sec')
+                }
+                continue
+
+            stat2_line_simple = stat2_line_simple_re.match(line)
+            if stat2_line_simple:
+                summary['total_size'] = stat2_line_simple.group('total_size')
+                summary['speedup'] = stat2_line_simple.group('speedup')
+                continue
+
             stat_line_log = stat_line_log_re.match(line)
             if stat_line_log:
                 summary = {

jc/utils.py

@@ -401,6 +401,9 @@ def convert_size_to_int(size: str, binary: bool = False) -> Optional[int]:
         >>> convert_size_to_int('1.5 GB', binary=True)
         1610612736
     """
+    # normalize input by removing commas
+    size = size.replace(',', '')
+
     def tokenize(text: str) -> List[str]:
         tokenized_input: List = []
         for token in re.split(r'(\d+(?:\.\d+)?)', text):

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2024-02-26 1.25.2 "JSON Convert"
+.TH jc 1 2024-02-27 1.25.2 "JSON Convert"
 .SH NAME
 \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types,
 and strings

tests/test_rsync.py

@@ -161,6 +161,18 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(jc.parsers.rsync.parse(self.osx_10_14_6_rsync_i_vvv_logfile_nochange, quiet=True), self.osx_10_14_6_rsync_i_vvv_logfile_nochange_json)
 
+    def test_rsync_simple_summary(self):
+        """
+        Test 'rsync avh' output with a simple summary
+        """
+        data = '''sending incremental file list
+
+sent 8.71M bytes  received 29.88K bytes  10.99K bytes/sec
+total size is 221.79G  speedup is 25,388.23
+'''
+        expected = [{"summary":{"sent":8710000,"received":29880,"bytes_sec":10990.0,"total_size":221790000000,"speedup":25388.23},"files":[]}]
+        self.assertEqual(jc.parsers.rsync.parse(data, quiet=True), expected)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_rsync_s.py

@@ -172,6 +172,18 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(list(jc.parsers.rsync_s.parse(self.osx_10_14_6_rsync_i_vvv_logfile_nochange.splitlines(), quiet=True)), self.osx_10_14_6_rsync_i_vvv_logfile_nochange_streaming_json)
 
+    def test_rsync_s_simple_summary(self):
+        """
+        Test 'rsync avh' output with a simple summary
+        """
+        data = '''sending incremental file list
+
+sent 8.71M bytes  received 29.88K bytes  10.99K bytes/sec
+total size is 221.79G  speedup is 25,388.23
+'''
+        expected = [{"type":"summary","sent":8710000,"received":29880,"bytes_sec":10990.0,"total_size":221790000000,"speedup":25388.23}]
+        self.assertEqual(list(jc.parsers.rsync_s.parse(data.splitlines(), quiet=True)), expected)
+
 
 
 if __name__ == '__main__':