python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-17 00:07:37 +02:00
Code Issues Releases Activity

update regexes for unchanged files

Browse Source
This commit is contained in:
Kelly Brazil
2022-02-01 13:54:25 -08:00
parent 26b8a1f167
commit 12f90c349c
2 changed files with 68 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
jc/parsers/rsync.py
View File

@ -246,58 +246,75 @@ def parse(
checksum_or_value_different = { checksum_or_value_different = {
'c': True, 'c': True,
'.': False, '.': False,
'+': None '+': None,
' ': None,
'?': None
} }
size_different = { size_different = {
's': True, 's': True,
'.': False, '.': False,
'+': None '+': None,
' ': None,
'?': None
} }
modification_time_different = { modification_time_different = {
't': True, 't': True,
'.': False, '.': False,
'+': None '+': None,
' ': None,
'?': None
} }
permissions_different = { permissions_different = {
'p': True, 'p': True,
'.': False, '.': False,
'+': None '+': None,
' ': None,
'?': None
} }
owner_different = { owner_different = {
'o': True, 'o': True,
'.': False, '.': False,
'+': None '+': None,
' ': None,
'?': None
} }
group_different = { group_different = {
'g': True, 'g': True,
'.': False, '.': False,
'+': None '+': None,
' ': None,
'?': None
} }
acl_different = { acl_different = {
'a': True, 'a': True,
'.': False, '.': False,
'+': None '+': None,
' ': None,
'?': None
} }
extended_attribute_different = { extended_attribute_different = {
'x': True, 'x': True,
'.': False, '.': False,
'+': None '+': None,
' ': None,
'?': None
} }
if jc.utils.has_data(data): if jc.utils.has_data(data):
file_line_re = re.compile(r'(?P<meta>[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P<name>.+)') file_line_re = re.compile(r'(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][u.+ ?][a.+ ?][x.+ ?]) (?P<name>.+)')
stat1_line_re = re.compile(r'(sent)\s+(?P<sent>[0-9,]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.]+)\s+(bytes/sec)') stat1_line_re = re.compile(r'(sent)\s+(?P<sent>[0-9,]+)\s+(bytes)\s+(received)\s+(?P<received>[0-9,]+)\s+(bytes)\s+(?P<bytes_sec>[0-9,.]+)\s+(bytes/sec)')
stat2_line_re = re.compile(r'(total size is)\s+(?P<total_size>[0-9,]+)\s+(speedup is)\s+(?P<speedup>[0-9,.]+)') stat2_line_re = re.compile(r'(total size is)\s+(?P<total_size>[0-9,]+)\s+(speedup is)\s+(?P<speedup>[0-9,.]+)')
file_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+])\s+(?P<name>.+)') file_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][u.+ ?][a.+ ?][x.+ ?]) (?P<name>.+)')
file_line_log_mac_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+(?P<meta>[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][x.+ ?]) (?P<name>.+)')
stat_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+sent\s+(?P<sent>[\d,]+)\s+bytes\s+received\s+(?P<received>[\d,]+)\s+bytes\s+total\s+size\s+(?P<total_size>[\d,]+)') stat_line_log_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)\]\s+sent\s+(?P<sent>[\d,]+)\s+bytes\s+received\s+(?P<received>[\d,]+)\s+bytes\s+total\s+size\s+(?P<total_size>[\d,]+)')
stat1_line_log_v_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)]\s+total:\s+matches=(?P<matches>[\d,]+)\s+hash_hits=(?P<hash_hits>[\d,]+)\s+false_alarms=(?P<false_alarms>[\d,]+)\s+data=(?P<data>[\d,]+)') stat1_line_log_v_re = re.compile(r'(?P<date>\d\d\d\d/\d\d/\d\d)\s+(?P<time>\d\d:\d\d:\d\d)\s+\[(?P<process>\d+)]\s+total:\s+matches=(?P<matches>[\d,]+)\s+hash_hits=(?P<hash_hits>[\d,]+)\s+false_alarms=(?P<false_alarms>[\d,]+)\s+data=(?P<data>[\d,]+)')
@ -362,6 +379,37 @@ def parse(
rsync_run['files'].append(output_line) rsync_run['files'].append(output_line)
continue continue
file_line_log_mac = file_line_log_mac_re.match(line)
if file_line_log_mac:
filename = file_line_log_mac.group('name')
date = file_line_log_mac.group('date')
time = file_line_log_mac.group('time')
process = file_line_log_mac.group('process')
meta = file_line_log_mac.group('meta')
if process != last_process:
raw_output.append(rsync_run)
rsync_run = deepcopy(rsync_run_new)
last_process = process
output_line = {
'filename': filename,
'date': date,
'time': time,
'process': process,
'metadata': meta,
'update_type': update_type[meta[0]],
'file_type': file_type[meta[1]],
'checksum_or_value_different': checksum_or_value_different[meta[2]],
'size_different': size_different[meta[3]],
'modification_time_different': modification_time_different[meta[4]],
'permissions_different': permissions_different[meta[5]],
'owner_different': owner_different[meta[6]],
'group_different': group_different[meta[7]]
}
rsync_run['files'].append(output_line)
continue
stat1_line = stat1_line_re.match(line) stat1_line = stat1_line_re.match(line)
if stat1_line: if stat1_line:
rsync_run['summary'] = { rsync_run['summary'] = {

20
tests/test_rsync.py
View File

@ -31,12 +31,8 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-i-vvv-logfile.out'), 'r', encoding='utf-8') as f: with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-i-vvv-logfile.out'), 'r', encoding='utf-8') as f:
self.centos_7_7_rsync_i_vvv_logfile = f.read() self.centos_7_7_rsync_i_vvv_logfile = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-i-vvv-logfile-nochange.out'), 'r', encoding='utf-8') as f:
self.centos_7_7_rsync_i_vvv_logfile_nochange = f.read()
# with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/rsync.out'), 'r', encoding='utf-8') as f:
# self.ubuntu_18_4_rsync = f.read()
# output # output
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-i.json'), 'r', encoding='utf-8') as f: with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-i.json'), 'r', encoding='utf-8') as f:
@ -60,10 +56,8 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-i-vvv-logfile.json'), 'r', encoding='utf-8') as f: with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-i-vvv-logfile.json'), 'r', encoding='utf-8') as f:
self.centos_7_7_rsync_i_vvv_logfile_json = json.loads(f.read()) self.centos_7_7_rsync_i_vvv_logfile_json = json.loads(f.read())
# with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/rsync.json'), 'r', encoding='utf-8') as f: with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-i-vvv-logfile-nochange.json'), 'r', encoding='utf-8') as f:
# self.ubuntu_18_4_rsync_json = json.loads(f.read()) self.centos_7_7_rsync_i_vvv_logfile_nochange_json = json.loads(f.read())
def test_rsync_nodata(self): def test_rsync_nodata(self):
""" """
@ -113,6 +107,12 @@ class MyTests(unittest.TestCase):
""" """
self.assertEqual(jc.parsers.rsync.parse(self.centos_7_7_rsync_i_vvv_logfile, quiet=True), self.centos_7_7_rsync_i_vvv_logfile_json) self.assertEqual(jc.parsers.rsync.parse(self.centos_7_7_rsync_i_vvv_logfile, quiet=True), self.centos_7_7_rsync_i_vvv_logfile_json)
def test_rsync_centos_7_7_rsync_i_vvv_logfile_nochange(self):
"""
Test 'rsync -ivvv --log-file=xxx' on Centos 7.7 with no file changes
"""
self.assertEqual(jc.parsers.rsync.parse(self.centos_7_7_rsync_i_vvv_logfile_nochange, quiet=True), self.centos_7_7_rsync_i_vvv_logfile_nochange_json)
# def test_rsync_ubuntu_18_4(self): # def test_rsync_ubuntu_18_4(self):
# """ # """
# Test 'rsync' on Ubuntu 18.4 # Test 'rsync' on Ubuntu 18.4