python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-23 00:29:59 +02:00
Code Issues Releases Activity

updated schema. add log-file option support

Browse Source
This commit is contained in:
Kelly Brazil
2022-01-29 17:24:43 -08:00
parent d6de81747f
commit ea5011b616
2 changed files with 175 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
jc/parsers/rsync.py
View File

@ -25,7 +25,15 @@ Schema:
[ [
{ {
"type": string, # 'item' or 'summary' "summary": {
"date": string,
"time": string,
"process": integer, # need to convert
"sent": integer, # need to convert
"received": integer, # need to convert
"total_size": integer # need to convert
},
"items": [
"filename": string, "filename": string,
"metadata": string, "metadata": string,
"update_type": string/null, "update_type": string/null,
@ -44,6 +52,7 @@ Schema:
"bytes_sec": float, # need to convert "bytes_sec": float, # need to convert
"total_size": integer, # need to convert "total_size": integer, # need to convert
"speedup": float, # need to convert "speedup": float, # need to convert
]
} }
] ]
@ -85,14 +94,14 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
List of Dictionaries. Structured to conform to the schema. List of Dictionaries. Structured to conform to the schema.
""" """
int_list = ['sent', 'received', 'total_size'] # int_list = ['sent', 'received', 'total_size']
float_list = ['bytes_sec', 'speedup'] # float_list = ['bytes_sec', 'speedup']
for entry in proc_data: # for entry in proc_data:
for key in entry: # for key in entry:
if key in int_list: # if key in int_list:
entry[key] = jc.utils.convert_to_int(entry[key]) # entry[key] = jc.utils.convert_to_int(entry[key])
if key in float_list: # if key in float_list:
entry[key] = jc.utils.convert_to_float(entry[key]) # entry[key] = jc.utils.convert_to_float(entry[key])
return proc_data return proc_data
@ -119,6 +128,9 @@ def parse(
jc.utils.input_type_check(data) jc.utils.input_type_check(data)
raw_output: List = [] raw_output: List = []
rsync_run: Dict = {}
last_process = ''
update_type = { update_type = {
'<': 'file sent', '<': 'file sent',
@ -189,26 +201,35 @@ def parse(
'+': None '+': None
} }
summary = {}
if jc.utils.has_data(data): if jc.utils.has_data(data):
rsync_run.update({
'summary': {},
'items': []
})
file_line_re = re.compile(r'(?P<meta>^[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P<name>.+)') file_line_re = re.compile(r'(?P<meta>[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P<name>.+)')
stat1_line_re = re.compile(r'(sent)\s+(?P<sent>[0-9,]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.]+)\s+(bytes/sec)') stat1_line_re = re.compile(r'(sent)\s+(?P<sent>[0-9,]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.]+)\s+(bytes/sec)')
stat2_line_re = re.compile(r'(total size is)\s+(?P<total_size>[0-9,]+)\s+(speedup is)\s+(?P<speedup>[0-9,.]+)') stat2_line_re = re.compile(r'(total size is)\s+(?P<total_size>[0-9,]+)\s+(speedup is)\s+(?P<speedup>[0-9,.]+)')
file_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+])\s+(?P<name>.+)')
stat_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+sent\s+(?P<sent>[\d,]+)\s+bytes\s+received\s+(?P<received>[\d,]+)\s+bytes\s+total\s+size\s+(?P<total_size>[\d,]+)')
stat1_line_log_v_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)]\s+total:\s+matches=(?P<matches>[\d,]+)\s+hash_hits=(?P<hash_hits>[\d,]+)\s+false_alarms=(?P<false_alarms>[\d,]+)\s+data=(?P<data>[\d,]+)')
stat2_line_log_v_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+sent\s+(?P<sent>[\d,]+)\s+bytes\s+received\s+(?P<received>[\d,]+)\s+bytes\s+(?P<bytes_sec>[\d,.]+)\s+bytes/sec')
stat3_line_log_v_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)]\s+total\s+size\s+is\s+(?P<total_size>[\d,]+)\s+speedup\s+is\s+(?P<speedup>[\d,.]+)')
for line in filter(None, data.splitlines()): for line in filter(None, data.splitlines()):
file_line = file_line_re.match(line) stat1_line_log_v = stat1_line_log_v_re.match(line)
stat1_line = stat1_line_re.match(line) stat2_line_log_v = stat2_line_log_v_re.match(line)
stat2_line = stat2_line_re.match(line) stat3_line_log_v = stat3_line_log_v_re.match(line)
file_line = file_line_re.match(line)
if file_line: if file_line:
meta = file_line.group('meta')
filename = file_line.group('name') filename = file_line.group('name')
meta = file_line.group('meta')
output_line = { output_line = {
'type': 'item',
'filename': filename, 'filename': filename,
'metadata': meta, 'metadata': meta,
'update_type': update_type[meta[0]], 'update_type': update_type[meta[0]],
@ -224,22 +245,76 @@ def parse(
'extended_attribute_different': extended_attribute_different[meta[10]] 'extended_attribute_different': extended_attribute_different[meta[10]]
} }
raw_output.append(output_line) rsync_run['items'].append(output_line)
continue continue
file_line_log = file_line_log_re.match(line)
if file_line_log:
filename = file_line_log.group('name')
date = file_line_log.group('date')
time = file_line_log.group('time')
process = file_line_log.group('process')
meta = file_line_log.group('meta')
if process != last_process:
if rsync_run:
raw_output.append(rsync_run)
rsync_run = {
'summary': {},
'items': []
}
last_process = process
output_line = {
'filename': filename,
'date': date,
'time': time,
'process': process,
'metadata': meta,
'update_type': update_type[meta[0]],
'file_type': file_type[meta[1]],
'checksum_or_value_different': checksum_or_value_different[meta[2]],
'size_different': size_different[meta[3]],
'modification_time_different': modification_time_different[meta[4]],
'permissions_different': permissions_different[meta[5]],
'owner_different': owner_different[meta[6]],
'group_different': group_different[meta[7]],
'future': future,
'acl_different': acl_different[meta[9]],
'extended_attribute_different': extended_attribute_different[meta[10]]
}
rsync_run['items'].append(output_line)
continue
stat1_line = stat1_line_re.match(line)
if stat1_line: if stat1_line:
summary = { rsync_run['summary'] = {
'type': 'summary',
'sent': stat1_line.group('sent'), 'sent': stat1_line.group('sent'),
'received': stat1_line.group('received'), 'received': stat1_line.group('received'),
'bytes_sec': stat1_line.group('bytes_sec') 'bytes_sec': stat1_line.group('bytes_sec')
} }
continue continue
stat2_line = stat2_line_re.match(line)
if stat2_line: if stat2_line:
summary['total_size'] = stat2_line.group('total_size') rsync_run['summary']['total_size'] = stat2_line.group('total_size')
summary['speedup'] = stat2_line.group('speedup') rsync_run['summary']['speedup'] = stat2_line.group('speedup')
raw_output.append(summary)
continue continue
stat_line_log = stat_line_log_re.match(line)
if stat_line_log:
rsync_run['summary'] = {
'date': stat_line_log.group('date'),
'time': stat_line_log.group('time'),
'process': stat_line_log.group('process'),
'sent': stat_line_log.group('sent'),
'received': stat_line_log.group('received'),
'total_size': stat_line_log.group('total_size')
}
continue
if rsync_run:
raw_output.append(rsync_run)
return raw_output if raw else _process(raw_output) return raw_output if raw else _process(raw_output)

59
tests/fixtures/centos-7.7/rsync-i-logfile.out vendored Normal file
View File

@ -0,0 +1,59 @@
2022/01/28 03:53:52 [9190] building file list
2022/01/28 03:53:52 [9190] .d..t...... ./
2022/01/28 03:53:52 [9190] >f+++++++++ a.txt
2022/01/28 03:53:52 [9190] >f+++++++++ b.txt
2022/01/28 03:53:52 [9190] >f+++++++++ c.txt
2022/01/28 03:53:52 [9190] >f+++++++++ d.txt
2022/01/28 03:53:52 [9190] >f+++++++++ file with spaces.txt
2022/01/28 03:53:52 [9190] cd+++++++++ folder/
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file1
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file10
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file11
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file12
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file13
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file14
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file15
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file16
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file17
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file18
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file19
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file2
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file20
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file3
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file4
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file5
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file6
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file7
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file8
2022/01/28 03:53:52 [9190] >f+++++++++ folder/file9
2022/01/28 03:53:52 [9190] sent 1713 bytes received 507 bytes total size 235
2022/01/28 03:55:00 [9198] building file list
2022/01/28 03:55:00 [9198] .d..t...... ./
2022/01/28 03:55:00 [9198] >f+++++++++ a.txt
2022/01/28 03:55:00 [9198] >f+++++++++ b.txt
2022/01/28 03:55:00 [9198] >f+++++++++ c.txt
2022/01/28 03:55:00 [9198] >f+++++++++ d.txt
2022/01/28 03:55:00 [9198] >f+++++++++ file with spaces.txt
2022/01/28 03:55:00 [9198] cd+++++++++ folder/
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file1
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file10
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file11
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file12
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file13
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file14
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file15
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file16
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file17
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file18
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file19
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file2
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file20
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file3
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file4
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file5
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file6
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file7
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file8
2022/01/28 03:55:00 [9198] >f+++++++++ folder/file9
2022/01/28 03:55:00 [9198] sent 1713 bytes received 507 bytes total size 235