From 8f3b12e81e1b4f0d4394c33db5b3a352f4cfb391 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 1 Feb 2022 14:52:51 -0800 Subject: [PATCH] add fixes for mac -i output --- jc/parsers/rsync.py | 22 +++ .../osx-10.14.6/rsync-i-vvv-nochange.json | 1 + .../osx-10.14.6/rsync-i-vvv-nochange.out | 88 ++++++++++ tests/fixtures/osx-10.14.6/rsync-i-vvv.json | 1 + tests/fixtures/osx-10.14.6/rsync-i-vvv.out | 156 ++++++++++++++++++ tests/test_rsync.py | 31 ++++ 6 files changed, 299 insertions(+) create mode 100644 tests/fixtures/osx-10.14.6/rsync-i-vvv-nochange.json create mode 100644 tests/fixtures/osx-10.14.6/rsync-i-vvv-nochange.out create mode 100644 tests/fixtures/osx-10.14.6/rsync-i-vvv.json create mode 100644 tests/fixtures/osx-10.14.6/rsync-i-vvv.out diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py index c1621e85..197f5cc5 100644 --- a/jc/parsers/rsync.py +++ b/jc/parsers/rsync.py @@ -310,6 +310,7 @@ def parse( if jc.utils.has_data(data): file_line_re = re.compile(r'(?P[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][u.+ ?][a.+ ?][x.+ ?]) (?P.+)') + file_line_mac_re = re.compile(r'(?P[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][x.+ ?]) (?P.+)') stat1_line_re = re.compile(r'(sent)\s+(?P[0-9,]+)\s+(bytes)\s+(received)\s+(?P[0-9,]+)\s+(bytes)\s+(?P[0-9,.]+)\s+(bytes/sec)') stat2_line_re = re.compile(r'(total size is)\s+(?P[0-9,]+)\s+(speedup is)\s+(?P[0-9,.]+)') @@ -346,6 +347,27 @@ def parse( rsync_run['files'].append(output_line) continue + file_line_mac = file_line_mac_re.match(line) + if file_line_mac: + filename = file_line_mac.group('name') + meta = file_line_mac.group('meta') + + output_line = { + 'filename': filename, + 'metadata': meta, + 'update_type': update_type[meta[0]], + 'file_type': file_type[meta[1]], + 'checksum_or_value_different': checksum_or_value_different[meta[2]], + 'size_different': size_different[meta[3]], + 'modification_time_different': modification_time_different[meta[4]], + 'permissions_different': permissions_different[meta[5]], + 'owner_different': owner_different[meta[6]], + 'group_different': group_different[meta[7]] + } + + rsync_run['files'].append(output_line) + continue + file_line_log = file_line_log_re.match(line) if file_line_log: filename = file_line_log.group('name') diff --git a/tests/fixtures/osx-10.14.6/rsync-i-vvv-nochange.json b/tests/fixtures/osx-10.14.6/rsync-i-vvv-nochange.json new file mode 100644 index 00000000..637aed96 --- /dev/null +++ b/tests/fixtures/osx-10.14.6/rsync-i-vvv-nochange.json @@ -0,0 +1 @@ +[{"summary":{"sent":284,"received":80,"bytes_sec":728.0,"total_size":320,"speedup":0.88},"files":[{"filename":"./","metadata":".d ","update_type":"not updated","file_type":"directory","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"a.txt","metadata":".f ","update_type":"not updated","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"b.txt","metadata":".f ","update_type":"not updated","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"c.txt","metadata":".f ","update_type":"not updated","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"d.txt","metadata":".f ","update_type":"not updated","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"folder/","metadata":".d ","update_type":"not updated","file_type":"directory","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"folder/a.txt","metadata":".f ","update_type":"not updated","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"folder/b.txt","metadata":".f ","update_type":"not updated","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"folder/c.txt","metadata":".f ","update_type":"not updated","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"folder/d.txt","metadata":".f ","update_type":"not updated","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null}]}] diff --git a/tests/fixtures/osx-10.14.6/rsync-i-vvv-nochange.out b/tests/fixtures/osx-10.14.6/rsync-i-vvv-nochange.out new file mode 100644 index 00000000..f795c9b0 --- /dev/null +++ b/tests/fixtures/osx-10.14.6/rsync-i-vvv-nochange.out @@ -0,0 +1,88 @@ +building file list ... +[sender] make_file(.,*,2) +[sender] make_file(folder,*,2) +[sender] make_file(c.txt,*,2) +[sender] make_file(b.txt,*,2) +[sender] make_file(a.txt,*,2) +[sender] make_file(d.txt,*,2) +[sender] make_file(folder/c.txt,*,2) +[sender] make_file(folder/b.txt,*,2) +[sender] make_file(folder/a.txt,*,2) +[sender] make_file(folder/d.txt,*,2) +done +server_recv(2) starting pid=13455 +send_file_list done +send_files starting +recv_file_name(.) +recv_file_name(folder) +recv_file_name(c.txt) +recv_file_name(b.txt) +recv_file_name(a.txt) +recv_file_name(d.txt) +recv_file_name(folder/c.txt) +recv_file_name(folder/b.txt) +recv_file_name(folder/a.txt) +recv_file_name(folder/d.txt) +received 10 names +recv_file_list done +get_local_name count=10 dest +generator starting pid=13455 count=10 +delta-transmission disabled for local transfer or --whole-file +recv_generator(.,0) +send_files(0, source/.) +.d ./ +recv_generator(a.txt,1) +send_files(1, source/a.txt) +.f a.txt +recv_generator(b.txt,2) +send_files(2, source/b.txt) +.f b.txt +recv_generator(c.txt,3) +send_files(3, source/c.txt) +.f c.txt +recv_generator(d.txt,4) +send_files(4, source/d.txt) +.f d.txt +recv_generator(folder,5) +send_files(5, source/folder) +.d folder/ +recv_generator(folder/a.txt,6) +send_files(6, source/folder/a.txt) +.f folder/a.txt +recv_generator(folder/b.txt,7) +send_files(7, source/folder/b.txt) +.f folder/b.txt +recv_generator(folder/c.txt,8) +send_files(8, source/folder/c.txt) +.f folder/c.txt +recv_generator(folder/d.txt,9) +send_files(9, source/folder/d.txt) +.f folder/d.txt +generate_files phase=1 +send_files phase=1 +recv_files(10) starting +recv_files(.) +recv_files(a.txt) +recv_files(b.txt) +recv_files(c.txt) +recv_files(d.txt) +recv_files(folder) +recv_files(folder/a.txt) +recv_files(folder/b.txt) +recv_files(folder/c.txt) +recv_files(folder/d.txt) +recv_files phase=1 +generate_files phase=2 +send_files phase=2 +send files finished +total: matches=0 hash_hits=0 false_alarms=0 data=0 +recv_files phase=2 +generate_files phase=3 +recv_files finished +recv_generator(.,0) +recv_generator(folder,5) +generate_files finished + +sent 284 bytes received 80 bytes 728.00 bytes/sec +total size is 320 speedup is 0.88 +_exit_cleanup(code=0, file=/BuildRoot/Library/Caches/com.apple.xbs/Sources/rsync/rsync-52.200.2/rsync/main.c, line=996): about to call exit(0) diff --git a/tests/fixtures/osx-10.14.6/rsync-i-vvv.json b/tests/fixtures/osx-10.14.6/rsync-i-vvv.json new file mode 100644 index 00000000..2b9c0b81 --- /dev/null +++ b/tests/fixtures/osx-10.14.6/rsync-i-vvv.json @@ -0,0 +1 @@ +[{"summary":{"sent":924,"received":208,"bytes_sec":2264.0,"total_size":320,"speedup":0.28},"files":[{"filename":"./","metadata":".d..t....","update_type":"not updated","file_type":"directory","checksum_or_value_different":false,"size_different":false,"modification_time_different":true,"permissions_different":false,"owner_different":false,"group_different":false},{"filename":"a.txt","metadata":">f+++++++","update_type":"file received","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"b.txt","metadata":">f+++++++","update_type":"file received","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"c.txt","metadata":">f+++++++","update_type":"file received","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"d.txt","metadata":">f+++++++","update_type":"file received","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"folder/","metadata":"cd+++++++","update_type":"local change or creation","file_type":"directory","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"folder/a.txt","metadata":">f+++++++","update_type":"file received","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"folder/b.txt","metadata":">f+++++++","update_type":"file received","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"folder/c.txt","metadata":">f+++++++","update_type":"file received","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null},{"filename":"folder/d.txt","metadata":">f+++++++","update_type":"file received","file_type":"file","checksum_or_value_different":null,"size_different":null,"modification_time_different":null,"permissions_different":null,"owner_different":null,"group_different":null}]}] diff --git a/tests/fixtures/osx-10.14.6/rsync-i-vvv.out b/tests/fixtures/osx-10.14.6/rsync-i-vvv.out new file mode 100644 index 00000000..80bd869f --- /dev/null +++ b/tests/fixtures/osx-10.14.6/rsync-i-vvv.out @@ -0,0 +1,156 @@ +building file list ... +[sender] make_file(.,*,2) +[sender] make_file(folder,*,2) +[sender] make_file(c.txt,*,2) +[sender] make_file(b.txt,*,2) +[sender] make_file(a.txt,*,2) +[sender] make_file(d.txt,*,2) +[sender] make_file(folder/c.txt,*,2) +[sender] make_file(folder/b.txt,*,2) +[sender] make_file(folder/a.txt,*,2) +[sender] make_file(folder/d.txt,*,2) +done +server_recv(2) starting pid=13404 +send_file_list done +send_files starting +recv_file_name(.) +recv_file_name(folder) +recv_file_name(c.txt) +recv_file_name(b.txt) +recv_file_name(a.txt) +recv_file_name(d.txt) +recv_file_name(folder/c.txt) +recv_file_name(folder/b.txt) +recv_file_name(folder/a.txt) +recv_file_name(folder/d.txt) +received 10 names +recv_file_list done +get_local_name count=10 dest +generator starting pid=13404 count=10 +delta-transmission disabled for local transfer or --whole-file +recv_generator(.,0) +recv_files(10) starting +send_files(0, source/.) +.d..t.... ./ +set modtime of . to (1643731675) Tue Feb 1 08:07:55 2022 +recv_generator(a.txt,1) +send_files(1, source/a.txt) +send_files mapped source/a.txt of size 40 +calling match_sums source/a.txt +>f+++++++ a.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/a.txt +recv_generator(b.txt,2) +recv_files(.) +send_files(2, source/b.txt) +send_files mapped source/b.txt of size 40 +calling match_sums source/b.txt +>f+++++++ b.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/b.txt +recv_generator(c.txt,3) +send_files(3, source/c.txt) +send_files mapped source/c.txt of size 40 +calling match_sums source/c.txt +>f+++++++ c.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/c.txt +recv_generator(d.txt,4) +send_files(4, source/d.txt) +send_files mapped source/d.txt of size 40 +calling match_sums source/d.txt +>f+++++++ d.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/d.txt +recv_generator(folder,5) +recv_files(a.txt) +send_files(5, source/folder) +cd+++++++ folder/ +set modtime of folder to (1643731693) Tue Feb 1 08:08:13 2022 +recv_generator(folder/a.txt,6) +send_files(6, source/folder/a.txt) +send_files mapped source/folder/a.txt of size 40 +calling match_sums source/folder/a.txt +>f+++++++ folder/a.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/a.txt +recv_generator(folder/b.txt,7) +send_files(7, source/folder/b.txt) +send_files mapped source/folder/b.txt of size 40 +calling match_sums source/folder/b.txt +>f+++++++ folder/b.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/b.txt +recv_generator(folder/c.txt,8) +send_files(8, source/folder/c.txt) +send_files mapped source/folder/c.txt of size 40 +calling match_sums source/folder/c.txt +>f+++++++ folder/c.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/c.txt +recv_generator(folder/d.txt,9) +send_files(9, source/folder/d.txt) +send_files mapped source/folder/d.txt of size 40 +calling match_sums source/folder/d.txt +>f+++++++ folder/d.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/d.txt +generate_files phase=1 +send_files phase=1 +got file_sum +set modtime of .a.txt.oj9IpV to (1643731665) Tue Feb 1 08:07:45 2022 +renaming .a.txt.oj9IpV to a.txt +recv_files(b.txt) +got file_sum +set modtime of .b.txt.cscHph to (1643731668) Tue Feb 1 08:07:48 2022 +renaming .b.txt.cscHph to b.txt +recv_files(c.txt) +got file_sum +set modtime of .c.txt.7G49mz to (1643731671) Tue Feb 1 08:07:51 2022 +renaming .c.txt.7G49mz to c.txt +recv_files(d.txt) +got file_sum +set modtime of .d.txt.fqafla to (1643731675) Tue Feb 1 08:07:55 2022 +renaming .d.txt.fqafla to d.txt +recv_files(folder) +recv_files(folder/a.txt) +got file_sum +set modtime of folder/.a.txt.s7JSpw to (1643731685) Tue Feb 1 08:08:05 2022 +renaming folder/.a.txt.s7JSpw to folder/a.txt +recv_files(folder/b.txt) +got file_sum +set modtime of folder/.b.txt.91fm9W to (1643731687) Tue Feb 1 08:08:07 2022 +renaming folder/.b.txt.91fm9W to folder/b.txt +recv_files(folder/c.txt) +got file_sum +set modtime of folder/.c.txt.I0phi8 to (1643731690) Tue Feb 1 08:08:10 2022 +renaming folder/.c.txt.I0phi8 to folder/c.txt +recv_files(folder/d.txt) +got file_sum +set modtime of folder/.d.txt.2HWHBz to (1643731693) Tue Feb 1 08:08:13 2022 +renaming folder/.d.txt.2HWHBz to folder/d.txt +recv_files phase=1 +generate_files phase=2 +send_files phase=2 +send files finished +total: matches=0 hash_hits=0 false_alarms=0 data=320 +recv_files phase=2 +recv_files finished +generate_files phase=3 +recv_generator(.,0) +set modtime of . to (1643731675) Tue Feb 1 08:07:55 2022 +recv_generator(folder,5) +set modtime of folder to (1643731693) Tue Feb 1 08:08:13 2022 +generate_files finished + +sent 924 bytes received 208 bytes 2264.00 bytes/sec +total size is 320 speedup is 0.28 +_exit_cleanup(code=0, file=/BuildRoot/Library/Caches/com.apple.xbs/Sources/rsync/rsync-52.200.2/rsync/main.c, line=996): about to call exit(0) diff --git a/tests/test_rsync.py b/tests/test_rsync.py index 7bde0298..92764f53 100644 --- a/tests/test_rsync.py +++ b/tests/test_rsync.py @@ -19,6 +19,14 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-ivvv.out'), 'r', encoding='utf-8') as f: self.centos_7_7_rsync_ivvv = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/rsync-i-vvv.out'), 'r', encoding='utf-8') as f: + self.osx_10_14_6_rsync_ivvv = f.read() + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/rsync-i-vvv-nochange.out'), 'r', encoding='utf-8') as f: + self.osx_10_14_6_rsync_ivvv_nochange = f.read() + + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-i-logfile.out'), 'r', encoding='utf-8') as f: self.centos_7_7_rsync_i_logfile = f.read() @@ -47,6 +55,16 @@ class MyTests(unittest.TestCase): with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-ivvv.json'), 'r', encoding='utf-8') as f: self.centos_7_7_rsync_ivvv_json = json.loads(f.read()) + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/rsync-i-vvv.json'), 'r', encoding='utf-8') as f: + self.osx_10_14_6_rsync_ivvv_json = json.loads(f.read()) + + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/rsync-i-vvv-nochange.json'), 'r', encoding='utf-8') as f: + self.osx_10_14_6_rsync_ivvv_nochange_json = json.loads(f.read()) + + + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/rsync-i-logfile.json'), 'r', encoding='utf-8') as f: self.centos_7_7_rsync_i_logfile_json = json.loads(f.read()) @@ -89,6 +107,19 @@ class MyTests(unittest.TestCase): """ self.assertEqual(jc.parsers.rsync.parse(self.centos_7_7_rsync_ivvv, quiet=True), self.centos_7_7_rsync_ivvv_json) + + def test_rsync_osx_10_14_6_rsync_ivvv(self): + """ + Test 'rsync -ivvv' on OSX 10.14.6 + """ + self.assertEqual(jc.parsers.rsync.parse(self.osx_10_14_6_rsync_ivvv, quiet=True), self.osx_10_14_6_rsync_ivvv_json) + + def test_rsync_osx_10_14_6_rsync_ivvv_nochange(self): + """ + Test 'rsync -ivvv' on OSX 10.14.6 with no file changes + """ + self.assertEqual(jc.parsers.rsync.parse(self.osx_10_14_6_rsync_ivvv_nochange, quiet=True), self.osx_10_14_6_rsync_ivvv_nochange_json) + def test_rsync_centos_7_7_rsync_i_logfile(self): """ Test 'rsync -i --log-file=xxx' on Centos 7.7