python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-17 00:07:37 +02:00
Code Issues Releases Activity

add simple summary support to rsync and rsync-s parsers

Browse Source
This commit is contained in:
Kelly Brazil
2024-02-27 10:26:07 -08:00
parent f9bf889654
commit e2f97d7d8e
8 changed files with 106 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/parsers/rsync.md
View File

@ -159,4 +159,4 @@ Compatibility: linux, darwin, freebsd
Source: [`jc/parsers/rsync.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/rsync.py)
Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

2
docs/parsers/rsync_s.md
View File

@ -116,4 +116,4 @@ Compatibility: linux, darwin, freebsd
Source: [`jc/parsers/rsync_s.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/rsync_s.py)
Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)

42
jc/parsers/rsync.py
View File

@ -131,13 +131,13 @@ Examples:
"""
import re
from copy import deepcopy
from typing import List, Dict
from typing import List, Dict, Union
import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.1'
version = '1.2'
description = '`rsync` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -171,10 +171,26 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
for item in proc_data:
for key in item['summary']:
if key in int_list:
item['summary'][key] = jc.utils.convert_to_int(item['summary'][key])
item['summary'][key] = jc.utils.convert_size_to_int(item['summary'][key])
if key in float_list:
item['summary'][key] = jc.utils.convert_to_float(item['summary'][key])
converted_val: Union[float, None] = None
val = item['summary'][key]
if any([
'K' in val,
'M' in val,
'G' in val,
'T' in val
]):
converted_int_val = jc.utils.convert_size_to_int(val)
if not converted_int_val is None:
converted_val = float(converted_int_val)
else:
converted_val = jc.utils.convert_to_float(val)
item['summary'][key] = converted_val
for entry in item['files']:
for key in entry:
@ -311,6 +327,9 @@ def parse(
stat1_line_re = re.compile(r'(sent)\s+(?P<sent>[0-9,]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.]+)\s+(bytes/sec)')
stat2_line_re = re.compile(r'(total size is)\s+(?P<total_size>[0-9,]+)\s+(speedup is)\s+(?P<speedup>[0-9,.]+)')
stat1_line_simple_re = re.compile(r'(sent)\s+(?P<sent>[0-9,.TGMK]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,.TGMK]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.TGMK]+)\s+(bytes/sec)')
stat2_line_simple_re = re.compile(r'(total\s+size\s+is)\s+(?P<total_size>[0-9,.TGMK]+)\s+(speedup\s+is)\s+(?P<speedup>[0-9,.TGMK]+)')
file_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][u.+ ?][a.+ ?][x.+ ?]) (?P<name>.+)')
file_line_log_mac_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][x.+ ?]) (?P<name>.+)')
stat_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+sent\s+(?P<sent>[\d,]+)\s+bytes\s+received\s+(?P<received>[\d,]+)\s+bytes\s+total\s+size\s+(?P<total_size>[\d,]+)')
@ -445,6 +464,21 @@ def parse(
rsync_run['summary']['speedup'] = stat2_line.group('speedup')
continue
stat1_line_simple = stat1_line_simple_re.match(line)
if stat1_line_simple:
rsync_run['summary'] = {
'sent': stat1_line_simple.group('sent'),
'received': stat1_line_simple.group('received'),
'bytes_sec': stat1_line_simple.group('bytes_sec')
}
continue
stat2_line_simple = stat2_line_simple_re.match(line)
if stat2_line_simple:
rsync_run['summary']['total_size'] = stat2_line_simple.group('total_size')
rsync_run['summary']['speedup'] = stat2_line_simple.group('speedup')
continue
stat_line_log = stat_line_log_re.match(line)
if stat_line_log:
rsync_run['summary'] = {

41
jc/parsers/rsync_s.py
View File

@ -88,7 +88,7 @@ from jc.streaming import (
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.2'
version = '1.3'
description = '`rsync` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -121,10 +121,26 @@ def _process(proc_data: Dict) -> Dict:
for key in proc_data.copy():
if key in int_list:
proc_data[key] = jc.utils.convert_to_int(proc_data[key])
proc_data[key] = jc.utils.convert_size_to_int(proc_data[key])
if key in float_list:
proc_data[key] = jc.utils.convert_to_float(proc_data[key])
converted_val: Union[float, None] = None
val = proc_data[key]
if any([
'K' in val,
'M' in val,
'G' in val,
'T' in val
]):
converted_int_val = jc.utils.convert_size_to_int(val)
if not converted_int_val is None:
converted_val = float(converted_int_val)
else:
converted_val = jc.utils.convert_to_float(val)
proc_data[key] = converted_val
# add timestamp
if 'date' in proc_data and 'time' in proc_data:
@ -254,6 +270,9 @@ def parse(
stat1_line_re = re.compile(r'(sent)\s+(?P<sent>[0-9,]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.]+)\s+(bytes/sec)')
stat2_line_re = re.compile(r'(total size is)\s+(?P<total_size>[0-9,]+)\s+(speedup is)\s+(?P<speedup>[0-9,.]+)')
stat1_line_simple_re = re.compile(r'(sent)\s+(?P<sent>[0-9,.TGMK]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,.TGMK]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.TGMK]+)\s+(bytes/sec)')
stat2_line_simple_re = re.compile(r'(total\s+size\s+is)\s+(?P<total_size>[0-9,.TGMK]+)\s+(speedup\s+is)\s+(?P<speedup>[0-9,.TGMK]+)')
file_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][u.+ ?][a.+ ?][x.+ ?]) (?P<name>.+)')
file_line_log_mac_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][x.+ ?]) (?P<name>.+)')
stat_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+sent\s+(?P<sent>[\d,]+)\s+bytes\s+received\s+(?P<received>[\d,]+)\s+bytes\s+total\s+size\s+(?P<total_size>[\d,]+)')
@ -403,6 +422,22 @@ def parse(
summary['speedup'] = stat2_line.group('speedup')
continue
stat1_line_simple = stat1_line_simple_re.match(line)
if stat1_line_simple:
summary = {
'type': 'summary',
'sent': stat1_line_simple.group('sent'),
'received': stat1_line_simple.group('received'),
'bytes_sec': stat1_line_simple.group('bytes_sec')
}
continue
stat2_line_simple = stat2_line_simple_re.match(line)
if stat2_line_simple:
summary['total_size'] = stat2_line_simple.group('total_size')
summary['speedup'] = stat2_line_simple.group('speedup')
continue
stat_line_log = stat_line_log_re.match(line)
if stat_line_log:
summary = {

3
jc/utils.py
View File

@ -401,6 +401,9 @@ def convert_size_to_int(size: str, binary: bool = False) -> Optional[int]:
>>> convert_size_to_int('1.5 GB', binary=True)
1610612736
"""
# normalize input by removing commas
size = size.replace(',', '')
def tokenize(text: str) -> List[str]:
tokenized_input: List = []
for token in re.split(r'(\d+(?:\.\d+)?)', text):

2
man/jc.1
View File

@ -1,4 +1,4 @@
.TH jc 1 2024-02-26 1.25.2 "JSON Convert"
.TH jc 1 2024-02-27 1.25.2 "JSON Convert"
.SH NAME
\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types,
and strings

12
tests/test_rsync.py
View File

@ -161,6 +161,18 @@ class MyTests(unittest.TestCase):
"""
self.assertEqual(jc.parsers.rsync.parse(self.osx_10_14_6_rsync_i_vvv_logfile_nochange, quiet=True), self.osx_10_14_6_rsync_i_vvv_logfile_nochange_json)
def test_rsync_simple_summary(self):
"""
Test 'rsync avh' output with a simple summary
"""
data = '''sending incremental file list
sent 8.71M bytes received 29.88K bytes 10.99K bytes/sec
total size is 221.79G speedup is 25,388.23
'''
expected = [{"summary":{"sent":8710000,"received":29880,"bytes_sec":10990.0,"total_size":221790000000,"speedup":25388.23},"files":[]}]
self.assertEqual(jc.parsers.rsync.parse(data, quiet=True), expected)
if __name__ == '__main__':
unittest.main()

12
tests/test_rsync_s.py
View File

@ -172,6 +172,18 @@ class MyTests(unittest.TestCase):
"""
self.assertEqual(list(jc.parsers.rsync_s.parse(self.osx_10_14_6_rsync_i_vvv_logfile_nochange.splitlines(), quiet=True)), self.osx_10_14_6_rsync_i_vvv_logfile_nochange_streaming_json)
def test_rsync_s_simple_summary(self):
"""
Test 'rsync avh' output with a simple summary
"""
data = '''sending incremental file list
sent 8.71M bytes received 29.88K bytes 10.99K bytes/sec
total size is 221.79G speedup is 25,388.23
'''
expected = [{"type":"summary","sent":8710000,"received":29880,"bytes_sec":10990.0,"total_size":221790000000,"speedup":25388.23}]
self.assertEqual(list(jc.parsers.rsync_s.parse(data.splitlines(), quiet=True)), expected)
if __name__ == '__main__':