diff --git a/jc/lib.py b/jc/lib.py index ab6b6b38..9128b0db 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -9,7 +9,7 @@ import importlib from typing import Dict, List, Iterable, Union, Iterator, Optional from jc import appdirs -__version__ = '1.18.2' +__version__ = '1.18.3' parsers = [ 'acpi', @@ -68,6 +68,7 @@ parsers = [ 'pip-show', 'ps', 'route', + 'rsync', 'rpm-qi', 'sfdisk', 'shadow', diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py new file mode 100644 index 00000000..46c2534e --- /dev/null +++ b/jc/parsers/rsync.py @@ -0,0 +1,212 @@ +"""jc - JSON CLI output utility `rsync` command output parser + +Supports the `-i` or `--itemize-changes` option with all levels of +verbosity. + +Usage (cli): + + $ rsync -i -a source/ dest | jc --rsync + + or + + $ jc rsync -i -a source/ dest + +Usage (module): + + import jc + result = jc.parse('rsync', rsync_command_output) + + or + + import jc.parsers.rsync + result = jc.parsers.rsync.parse(rsync_command_output) + +Schema: + + [ + { + "filename": string, + "metadata": string, + "update_type": string/null, + "file_type": string/null, + "checksum_or_value_different": bool/null, + "size_different": bool/null, + "modification_time_different": bool/null, + "permissions_different": bool/null, + "owner_different": bool/null, + "group_different": bool/null, + "future": null, + "acl_different": bool/null, + "extended_attribute_different": bool/null + } + ] + +Examples: + + $ rsync | jc --rsync -p + [] + + $ rsync | jc --rsync -p -r + [] +""" +import re +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`rsync` command parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'freebsd'] + magic_commands = ['rsync -i', 'rsync --itemize-changes'] + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + + # process the data here + # rebuild output for added semantic information + # use helper functions in jc.utils for int, float, bool + # conversions and timestamps + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + update_type = { + '<': 'file sent', + '>': 'file received', + 'c': 'local change or creation', + 'h': 'hard link', + '.': 'not updated', + '*': 'message', + '+': None + } + + file_type = { + 'f': 'file', + 'd': 'directory', + 'L': 'symlink', + 'D': 'device', + 'S': 'special file', + '+': None + } + + checksum_or_value_different = { + 'c': True, + '.': False, + '+': None + } + + size_different = { + 's': True, + '.': False, + '+': None + } + + modification_time_different = { + 't': True, + '.': False, + '+': None + } + + permissions_different = { + 'p': True, + '.': False, + '+': None + } + + owner_different = { + 'o': True, + '.': False, + '+': None + } + + group_different = { + 'g': True, + '.': False, + '+': None + } + + future = None + + acl_different = { + 'a': True, + '.': False, + '+': None + } + + extended_attribute_different = { + 'x': True, + '.': False, + '+': None + } + + if jc.utils.has_data(data): + + file_line_re = re.compile(r'(?P^[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P.+)') + + for line in filter(None, data.splitlines()): + + file_line = file_line_re.match(line) + if file_line: + meta = file_line.group('meta') + filename = file_line.group('name') + + output_line = { + 'filename': filename, + 'metadata': meta, + 'update_type': update_type[meta[0]], + 'file_type': file_type[meta[1]], + 'checksum_or_value_different': checksum_or_value_different[meta[2]], + 'size_different': size_different[meta[3]], + 'modification_time_different': modification_time_different[meta[4]], + 'permissions_different': permissions_different[meta[5]], + 'owner_different': owner_different[meta[6]], + 'group_different': group_different[meta[7]], + 'future': future, + 'acl_different': acl_different[meta[9]], + 'extended_attribute_different': extended_attribute_different[meta[10]] + } + + raw_output.append(output_line) + + return raw_output if raw else _process(raw_output) diff --git a/setup.py b/setup.py index 73eb28e5..f3dcc6b6 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='1.18.2', + version='1.18.3', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='Converts the output of popular command-line tools and file-types to JSON.', diff --git a/tests/fixtures/centos-7.7/rsync-ivvv.out b/tests/fixtures/centos-7.7/rsync-ivvv.out new file mode 100644 index 00000000..8ce768e9 --- /dev/null +++ b/tests/fixtures/centos-7.7/rsync-ivvv.out @@ -0,0 +1,422 @@ +sending incremental file list +[sender] make_file(.,*,0) +[sender] pushing local filters for /home/kbrazil/rsynctest/source/ +[sender] make_file(a.txt,*,2) +[sender] make_file(b.txt,*,2) +[sender] make_file(c.txt,*,2) +[sender] make_file(d.txt,*,2) +[sender] make_file(file with spaces.txt,*,2) +[sender] make_file(folder,*,2) +send_file_list done +send_files starting +[sender] pushing local filters for /home/kbrazil/rsynctest/source/folder/ +[sender] make_file(folder/file1,*,2) +[sender] make_file(folder/file2,*,2) +[sender] make_file(folder/file3,*,2) +[sender] make_file(folder/file4,*,2) +[sender] make_file(folder/file5,*,2) +[sender] make_file(folder/file6,*,2) +[sender] make_file(folder/file7,*,2) +[sender] make_file(folder/file8,*,2) +[sender] make_file(folder/file9,*,2) +[sender] make_file(folder/file10,*,2) +[sender] make_file(folder/file11,*,2) +[sender] make_file(folder/file12,*,2) +[sender] make_file(folder/file13,*,2) +[sender] make_file(folder/file14,*,2) +[sender] make_file(folder/file15,*,2) +[sender] make_file(folder/file16,*,2) +[sender] make_file(folder/file17,*,2) +[sender] make_file(folder/file18,*,2) +[sender] make_file(folder/file19,*,2) +[sender] make_file(folder/file20,*,2) +server_recv(2) starting pid=7804 +recv_file_name(.) +recv_file_name(a.txt) +recv_file_name(b.txt) +recv_file_name(c.txt) +recv_file_name(d.txt) +recv_file_name(file with spaces.txt) +recv_file_name(folder) +received 7 names +recv_file_list done +get_local_name count=7 dest +generator starting pid=7804 +delta-transmission disabled for local transfer or --whole-file +recv_generator(.,0) +set modtime of . to (1643343337) Thu Jan 27 20:15:37 2022 +recv_generator(.,1) +recv_generator(a.txt,2) +recv_generator(b.txt,3) +recv_generator(c.txt,4) +recv_generator(d.txt,5) +recv_generator(file with spaces.txt,6) +recv_generator(folder,7) +send_files(0, source/.) +.d..t...... ./ +send_files(2, source/a.txt) +send_files mapped source/a.txt of size 47 +calling match_sums source/a.txt +>f+++++++++ a.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/a.txt +send_files(3, source/b.txt) +send_files mapped source/b.txt of size 47 +calling match_sums source/b.txt +>f+++++++++ b.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/b.txt +send_files(4, source/c.txt) +send_files mapped source/c.txt of size 47 +calling match_sums source/c.txt +>f+++++++++ c.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/c.txt +send_files(5, source/d.txt) +send_files mapped source/d.txt of size 47 +calling match_sums source/d.txt +>f+++++++++ d.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/d.txt +send_files(6, source/file with spaces.txt) +send_files mapped source/file with spaces.txt of size 47 +calling match_sums source/file with spaces.txt +>f+++++++++ file with spaces.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/file with spaces.txt +recv_files(7) starting +[receiver] receiving flist for dir 1 +recv_file_name(folder/file1) +recv_file_name(folder/file2) +recv_file_name(folder/file3) +recv_file_name(folder/file4) +recv_file_name(folder/file5) +recv_file_name(folder/file6) +recv_file_name(folder/file7) +recv_file_name(folder/file8) +recv_file_name(folder/file9) +recv_file_name(folder/file10) +recv_file_name(folder/file11) +recv_file_name(folder/file12) +recv_file_name(folder/file13) +recv_file_name(folder/file14) +recv_file_name(folder/file15) +recv_file_name(folder/file16) +recv_file_name(folder/file17) +recv_file_name(folder/file18) +recv_file_name(folder/file19) +recv_file_name(folder/file20) +received 20 names +recv_file_list done +recv_files(.) +recv_files(a.txt) +got file_sum +set modtime of .a.txt.WA5SfS to (1643342949) Thu Jan 27 20:09:09 2022 +renaming .a.txt.WA5SfS to a.txt +recv_files(b.txt) +got file_sum +set modtime of .b.txt.b6U9z7 to (1643342953) Thu Jan 27 20:09:13 2022 +renaming .b.txt.b6U9z7 to b.txt +recv_files(c.txt) +got file_sum +set modtime of .c.txt.0ZRrUm to (1643342956) Thu Jan 27 20:09:16 2022 +renaming .c.txt.0ZRrUm to c.txt +recv_files(d.txt) +got file_sum +set modtime of .d.txt.N3zKeC to (1643342959) Thu Jan 27 20:09:19 2022 +renaming .d.txt.N3zKeC to d.txt +recv_files(file with spaces.txt) +got file_sum +set modtime of .file with spaces.txt.KN33yR to (1643342980) Thu Jan 27 20:09:40 2022 +renaming .file with spaces.txt.KN33yR to file with spaces.txt +[generator] receiving flist for dir 1 +recv_file_name(folder/file1) +recv_file_name(folder/file2) +recv_file_name(folder/file3) +recv_file_name(folder/file4) +recv_file_name(folder/file5) +recv_file_name(folder/file6) +recv_file_name(folder/file7) +recv_file_name(folder/file8) +recv_file_name(folder/file9) +recv_file_name(folder/file10) +recv_file_name(folder/file11) +recv_file_name(folder/file12) +recv_file_name(folder/file13) +recv_file_name(folder/file14) +recv_file_name(folder/file15) +recv_file_name(folder/file16) +recv_file_name(folder/file17) +recv_file_name(folder/file18) +recv_file_name(folder/file19) +recv_file_name(folder/file20) +received 20 names +recv_file_list done +recv_generator(folder,8) +set modtime of folder to (1643343369) Thu Jan 27 20:16:09 2022 +recv_generator(folder/file1,9) +set modtime of . to (1643343337) Thu Jan 27 20:15:37 2022 +recv_generator(folder/file10,10) +recv_generator(folder/file11,11) +recv_generator(folder/file12,12) +recv_generator(folder/file13,13) +recv_generator(folder/file14,14) +recv_generator(folder/file15,15) +recv_generator(folder/file16,16) +recv_generator(folder/file17,17) +recv_generator(folder/file18,18) +recv_generator(folder/file19,19) +recv_generator(folder/file2,20) +recv_generator(folder/file20,21) +recv_generator(folder/file3,22) +recv_generator(folder/file4,23) +recv_generator(folder/file5,24) +recv_generator(folder/file6,25) +recv_generator(folder/file7,26) +recv_generator(folder/file8,27) +recv_generator(folder/file9,28) +generate_files phase=1 +send_files(8, source/folder) +cd+++++++++ folder/ +send_files(9, source/folder/file1) +send_files mapped source/folder/file1 of size 0 +calling match_sums source/folder/file1 +>f+++++++++ folder/file1 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file1 +send_files(10, source/folder/file10) +send_files mapped source/folder/file10 of size 0 +calling match_sums source/folder/file10 +>f+++++++++ folder/file10 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file10 +send_files(11, source/folder/file11) +send_files mapped source/folder/file11 of size 0 +calling match_sums source/folder/file11 +>f+++++++++ folder/file11 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file11 +send_files(12, source/folder/file12) +send_files mapped source/folder/file12 of size 0 +calling match_sums source/folder/file12 +>f+++++++++ folder/file12 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file12 +send_files(13, source/folder/file13) +send_files mapped source/folder/file13 of size 0 +calling match_sums source/folder/file13 +>f+++++++++ folder/file13 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file13 +send_files(14, source/folder/file14) +send_files mapped source/folder/file14 of size 0 +calling match_sums source/folder/file14 +>f+++++++++ folder/file14 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file14 +send_files(15, source/folder/file15) +send_files mapped source/folder/file15 of size 0 +calling match_sums source/folder/file15 +>f+++++++++ folder/file15 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file15 +send_files(16, source/folder/file16) +send_files mapped source/folder/file16 of size 0 +calling match_sums source/folder/file16 +>f+++++++++ folder/file16 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file16 +send_files(17, source/folder/file17) +send_files mapped source/folder/file17 of size 0 +calling match_sums source/folder/file17 +>f+++++++++ folder/file17 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file17 +send_files(18, source/folder/file18) +send_files mapped source/folder/file18 of size 0 +calling match_sums source/folder/file18 +>f+++++++++ folder/file18 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file18 +send_files(19, source/folder/file19) +send_files mapped source/folder/file19 of size 0 +calling match_sums source/folder/file19 +>f+++++++++ folder/file19 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file19 +send_files(20, source/folder/file2) +send_files mapped source/folder/file2 of size 0 +calling match_sums source/folder/file2 +>f+++++++++ folder/file2 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file2 +send_files(21, source/folder/file20) +send_files mapped source/folder/file20 of size 0 +calling match_sums source/folder/file20 +>f+++++++++ folder/file20 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file20 +send_files(22, source/folder/file3) +send_files mapped source/folder/file3 of size 0 +calling match_sums source/folder/file3 +>f+++++++++ folder/file3 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file3 +send_files(23, source/folder/file4) +send_files mapped source/folder/file4 of size 0 +calling match_sums source/folder/file4 +>f+++++++++ folder/file4 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file4 +send_files(24, source/folder/file5) +send_files mapped source/folder/file5 of size 0 +calling match_sums source/folder/file5 +>f+++++++++ folder/file5 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file5 +send_files(25, source/folder/file6) +send_files mapped source/folder/file6 of size 0 +calling match_sums source/folder/file6 +>f+++++++++ folder/file6 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file6 +send_files(26, source/folder/file7) +send_files mapped source/folder/file7 of size 0 +calling match_sums source/folder/file7 +>f+++++++++ folder/file7 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file7 +send_files(27, source/folder/file8) +send_files mapped source/folder/file8 of size 0 +calling match_sums source/folder/file8 +>f+++++++++ folder/file8 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file8 +send_files(28, source/folder/file9) +send_files mapped source/folder/file9 of size 0 +calling match_sums source/folder/file9 +>f+++++++++ folder/file9 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file9 +recv_files(folder) +recv_files(folder/file1) +got file_sum +set modtime of folder/.file1.iqSNT6 to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file1.iqSNT6 to folder/file1 +recv_files(folder/file10) +got file_sum +set modtime of folder/.file10.AlTyem to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file10.AlTyem to folder/file10 +recv_files(folder/file11) +got file_sum +set modtime of folder/.file11.McEkzB to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file11.McEkzB to folder/file11 +recv_files(folder/file12) +got file_sum +set modtime of folder/.file12.6i46TQ to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file12.6i46TQ to folder/file12 +recv_files(folder/file13) +got file_sum +set modtime of folder/.file13.GE3Te6 to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file13.GE3Te6 to folder/file13 +recv_files(folder/file14) +got file_sum +set modtime of folder/.file14.iHFHzl to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file14.iHFHzl to folder/file14 +recv_files(folder/file15) +got file_sum +set modtime of folder/.file15.QXUvUA to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file15.QXUvUA to folder/file15 +recv_files(folder/file16) +got file_sum +set modtime of folder/.file16.avHkfQ to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file16.avHkfQ to folder/file16 +recv_files(folder/file17) +got file_sum +set modtime of folder/.file17.wH89z5 to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file17.wH89z5 to folder/file17 +recv_files(folder/file18) +got file_sum +set modtime of folder/.file18.Kpj0Uk to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file18.Kpj0Uk to folder/file18 +recv_files(folder/file19) +got file_sum +set modtime of folder/.file19.885QfA to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file19.885QfA to folder/file19 +recv_files(folder/file2) +got file_sum +set modtime of folder/.file2.SQrIAP to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file2.SQrIAP to folder/file2 +recv_files(folder/file20) +got file_sum +set modtime of folder/.file20.kXpAV4 to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file20.kXpAV4 to folder/file20 +recv_files(folder/file3) +got file_sum +set modtime of folder/.file3.GRdtgk to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file3.GRdtgk to folder/file3 +recv_files(folder/file4) +got file_sum +set modtime of folder/.file4.uJGmBz to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file4.uJGmBz to folder/file4 +recv_files(folder/file5) +got file_sum +set modtime of folder/.file5.OPPgWO to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file5.OPPgWO to folder/file5 +recv_files(folder/file6) +got file_sum +set modtime of folder/.file6.abEbh4 to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file6.abEbh4 to folder/file6 +recv_files(folder/file7) +got file_sum +set modtime of folder/.file7.4T46Bj to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file7.4T46Bj to folder/file7 +recv_files(folder/file8) +got file_sum +set modtime of folder/.file8.CA62Wy to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file8.CA62Wy to folder/file8 +recv_files(folder/file9) +got file_sum +set modtime of folder/.file9.Yu80hO to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file9.Yu80hO to folder/file9 +set modtime of folder to (1643343369) Thu Jan 27 20:16:09 2022 +send_files phase=1 +recv_files phase=1 +generate_files phase=2 +send_files phase=2 +send files finished +total: matches=0 hash_hits=0 false_alarms=0 data=235 +recv_files phase=2 +recv_files finished +generate_files phase=3 +generate_files finished + +sent 1,708 bytes received 8,209 bytes 19,834.00 bytes/sec +total size is 235 speedup is 0.02 +[sender] _exit_cleanup(code=0, file=main.c, line=1178): about to call exit(0) +