From ea5011b616365b5c10c04416d43035d84e3dacba Mon Sep 17 00:00:00 2001
From: Kelly Brazil <kellyjonbrazil@gmail.com>
Date: Sat, 29 Jan 2022 17:24:43 -0800
Subject: [PATCH] updated schema. add log-file option support

---
 jc/parsers/rsync.py                           | 157 +++++++++++++-----
 tests/fixtures/centos-7.7/rsync-i-logfile.out |  59 +++++++
 2 files changed, 175 insertions(+), 41 deletions(-)
 create mode 100644 tests/fixtures/centos-7.7/rsync-i-logfile.out

diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py
index fbaf9373..02adbe03 100644
--- a/jc/parsers/rsync.py
+++ b/jc/parsers/rsync.py
@@ -25,25 +25,34 @@ Schema:
 
     [
       {
-        "type":                           string,      # 'item' or 'summary'
-        "filename":                       string,
-        "metadata":                       string,
-        "update_type":                    string/null,
-        "file_type":                      string/null,
-        "checksum_or_value_different":    bool/null,
-        "size_different":                 bool/null,
-        "modification_time_different":    bool/null,
-        "permissions_different":          bool/null,
-        "owner_different":                bool/null,
-        "group_different":                bool/null,
-        "future":                         null,
-        "acl_different":                  bool/null,
-        "extended_attribute_different":   bool/null,
-        "sent":                           integer,   # need to convert
-        "received":                       integer,   # need to convert
-        "bytes_sec":                      float,     # need to convert
-        "total_size":                     integer,   # need to convert
-        "speedup":                        float,     # need to convert
+        "summary": {
+          "date":                           string,
+          "time":                           string,
+          "process":                        integer,     # need to convert
+          "sent":                           integer,     # need to convert
+          "received":                       integer,     # need to convert
+          "total_size":                     integer      # need to convert
+        },
+        "items": [
+          "filename":                       string,
+          "metadata":                       string,
+          "update_type":                    string/null,
+          "file_type":                      string/null,
+          "checksum_or_value_different":    bool/null,
+          "size_different":                 bool/null,
+          "modification_time_different":    bool/null,
+          "permissions_different":          bool/null,
+          "owner_different":                bool/null,
+          "group_different":                bool/null,
+          "future":                         null,
+          "acl_different":                  bool/null,
+          "extended_attribute_different":   bool/null,
+          "sent":                           integer,     # need to convert
+          "received":                       integer,     # need to convert
+          "bytes_sec":                      float,       # need to convert
+          "total_size":                     integer,     # need to convert
+          "speedup":                        float,       # need to convert
+        ]
       }
     ]
 
@@ -85,14 +94,14 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
 
         List of Dictionaries. Structured to conform to the schema.
     """
-    int_list = ['sent', 'received', 'total_size']
-    float_list = ['bytes_sec', 'speedup']
-    for entry in proc_data:
-        for key in entry:
-            if key in int_list:
-                entry[key] = jc.utils.convert_to_int(entry[key])
-            if key in float_list:
-                entry[key] = jc.utils.convert_to_float(entry[key])
+    # int_list = ['sent', 'received', 'total_size']
+    # float_list = ['bytes_sec', 'speedup']
+    # for entry in proc_data:
+    #     for key in entry:
+    #         if key in int_list:
+    #             entry[key] = jc.utils.convert_to_int(entry[key])
+    #         if key in float_list:
+    #             entry[key] = jc.utils.convert_to_float(entry[key])
 
     return proc_data
 
@@ -119,6 +128,9 @@ def parse(
     jc.utils.input_type_check(data)
 
     raw_output: List = []
+    rsync_run: Dict = {}
+
+    last_process = ''
 
     update_type = {
         '<': 'file sent',
@@ -189,26 +201,35 @@ def parse(
         '+': None
     }
 
-    summary = {}
-
     if jc.utils.has_data(data):
+        rsync_run.update({
+            'summary': {},
+            'items': []
+        })
 
-        file_line_re = re.compile(r'(?P<meta>^[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P<name>.+)')
+        file_line_re = re.compile(r'(?P<meta>[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P<name>.+)')
         stat1_line_re = re.compile(r'(sent)\s+(?P<sent>[0-9,]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.]+)\s+(bytes/sec)')
         stat2_line_re = re.compile(r'(total size is)\s+(?P<total_size>[0-9,]+)\s+(speedup is)\s+(?P<speedup>[0-9,.]+)')
 
+        file_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+])\s+(?P<name>.+)')
+        stat_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+sent\s+(?P<sent>[\d,]+)\s+bytes\s+received\s+(?P<received>[\d,]+)\s+bytes\s+total\s+size\s+(?P<total_size>[\d,]+)')
+
+        stat1_line_log_v_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)]\s+total:\s+matches=(?P<matches>[\d,]+)\s+hash_hits=(?P<hash_hits>[\d,]+)\s+false_alarms=(?P<false_alarms>[\d,]+)\s+data=(?P<data>[\d,]+)')
+        stat2_line_log_v_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+sent\s+(?P<sent>[\d,]+)\s+bytes\s+received\s+(?P<received>[\d,]+)\s+bytes\s+(?P<bytes_sec>[\d,.]+)\s+bytes/sec')
+        stat3_line_log_v_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)]\s+total\s+size\s+is\s+(?P<total_size>[\d,]+)\s+speedup\s+is\s+(?P<speedup>[\d,.]+)')
+
         for line in filter(None, data.splitlines()):
 
-            file_line = file_line_re.match(line)
-            stat1_line = stat1_line_re.match(line)
-            stat2_line = stat2_line_re.match(line)
+            stat1_line_log_v = stat1_line_log_v_re.match(line)
+            stat2_line_log_v = stat2_line_log_v_re.match(line)
+            stat3_line_log_v = stat3_line_log_v_re.match(line)
 
+            file_line = file_line_re.match(line)
             if file_line:
-                meta = file_line.group('meta')
                 filename = file_line.group('name')
+                meta = file_line.group('meta')
 
                 output_line = {
-                    'type': 'item',
                     'filename': filename,
                     'metadata': meta,
                     'update_type': update_type[meta[0]],
@@ -224,22 +245,76 @@ def parse(
                     'extended_attribute_different': extended_attribute_different[meta[10]]
                 }
 
-                raw_output.append(output_line)
+                rsync_run['items'].append(output_line)
                 continue
 
+            file_line_log = file_line_log_re.match(line)
+            if file_line_log:
+                filename = file_line_log.group('name')
+                date = file_line_log.group('date')
+                time = file_line_log.group('time')
+                process = file_line_log.group('process')
+                meta = file_line_log.group('meta')
+
+                if process != last_process:
+                    if rsync_run:
+                        raw_output.append(rsync_run)
+                        rsync_run = {
+                            'summary': {},
+                            'items': []
+                        }
+                        last_process = process
+
+                output_line = {
+                    'filename': filename,
+                    'date': date,
+                    'time': time,
+                    'process': process,
+                    'metadata': meta,
+                    'update_type': update_type[meta[0]],
+                    'file_type': file_type[meta[1]],
+                    'checksum_or_value_different': checksum_or_value_different[meta[2]],
+                    'size_different': size_different[meta[3]],
+                    'modification_time_different': modification_time_different[meta[4]],
+                    'permissions_different': permissions_different[meta[5]],
+                    'owner_different': owner_different[meta[6]],
+                    'group_different': group_different[meta[7]],
+                    'future': future,
+                    'acl_different': acl_different[meta[9]],
+                    'extended_attribute_different': extended_attribute_different[meta[10]]
+                }
+
+                rsync_run['items'].append(output_line)
+                continue
+
+            stat1_line = stat1_line_re.match(line)
             if stat1_line:
-                summary = {
-                    'type': 'summary',
+                rsync_run['summary'] = {
                     'sent': stat1_line.group('sent'),
                     'received': stat1_line.group('received'),
                     'bytes_sec': stat1_line.group('bytes_sec')
                 }
                 continue
 
+            stat2_line = stat2_line_re.match(line)
             if stat2_line:
-                summary['total_size'] = stat2_line.group('total_size')
-                summary['speedup'] = stat2_line.group('speedup')
-                raw_output.append(summary)
+                rsync_run['summary']['total_size'] = stat2_line.group('total_size')
+                rsync_run['summary']['speedup'] = stat2_line.group('speedup')
                 continue
 
+            stat_line_log = stat_line_log_re.match(line)
+            if stat_line_log:
+                rsync_run['summary'] = {
+                    'date': stat_line_log.group('date'),
+                    'time': stat_line_log.group('time'),
+                    'process': stat_line_log.group('process'),
+                    'sent': stat_line_log.group('sent'),
+                    'received': stat_line_log.group('received'),
+                    'total_size': stat_line_log.group('total_size')
+                }
+                continue
+
+    if rsync_run:
+        raw_output.append(rsync_run)
+
     return raw_output if raw else _process(raw_output)
diff --git a/tests/fixtures/centos-7.7/rsync-i-logfile.out b/tests/fixtures/centos-7.7/rsync-i-logfile.out
new file mode 100644
index 00000000..ea01a589
--- /dev/null
+++ b/tests/fixtures/centos-7.7/rsync-i-logfile.out
@@ -0,0 +1,59 @@
+2022/01/28 03:53:52 [9190] building file list
+2022/01/28 03:53:52 [9190] .d..t...... ./
+2022/01/28 03:53:52 [9190] >f+++++++++ a.txt
+2022/01/28 03:53:52 [9190] >f+++++++++ b.txt
+2022/01/28 03:53:52 [9190] >f+++++++++ c.txt
+2022/01/28 03:53:52 [9190] >f+++++++++ d.txt
+2022/01/28 03:53:52 [9190] >f+++++++++ file with spaces.txt
+2022/01/28 03:53:52 [9190] cd+++++++++ folder/
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file1
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file10
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file11
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file12
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file13
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file14
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file15
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file16
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file17
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file18
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file19
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file2
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file20
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file3
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file4
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file5
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file6
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file7
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file8
+2022/01/28 03:53:52 [9190] >f+++++++++ folder/file9
+2022/01/28 03:53:52 [9190] sent 1713 bytes  received 507 bytes  total size 235
+2022/01/28 03:55:00 [9198] building file list
+2022/01/28 03:55:00 [9198] .d..t...... ./
+2022/01/28 03:55:00 [9198] >f+++++++++ a.txt
+2022/01/28 03:55:00 [9198] >f+++++++++ b.txt
+2022/01/28 03:55:00 [9198] >f+++++++++ c.txt
+2022/01/28 03:55:00 [9198] >f+++++++++ d.txt
+2022/01/28 03:55:00 [9198] >f+++++++++ file with spaces.txt
+2022/01/28 03:55:00 [9198] cd+++++++++ folder/
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file1
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file10
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file11
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file12
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file13
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file14
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file15
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file16
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file17
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file18
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file19
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file2
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file20
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file3
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file4
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file5
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file6
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file7
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file8
+2022/01/28 03:55:00 [9198] >f+++++++++ folder/file9
+2022/01/28 03:55:00 [9198] sent 1713 bytes  received 507 bytes  total size 235
+