From 1f96586a5e883709f436526de6dcb21ff44bf97b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 28 Jan 2022 14:08:21 -0800 Subject: [PATCH 01/74] reorder imports --- jc/parsers/foo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/foo.py b/jc/parsers/foo.py index bd077f3f..afd56deb 100644 --- a/jc/parsers/foo.py +++ b/jc/parsers/foo.py @@ -38,8 +38,8 @@ Examples: $ foo | jc --foo -p -r [] """ -import jc.utils from typing import List, Dict +import jc.utils class info(): From 84169e1a913d035744f71472be81fe3a7d774b97 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 28 Jan 2022 14:08:43 -0800 Subject: [PATCH 02/74] add rsync parser --- jc/lib.py | 3 +- jc/parsers/rsync.py | 212 ++++++++++++ setup.py | 2 +- tests/fixtures/centos-7.7/rsync-ivvv.out | 422 +++++++++++++++++++++++ 4 files changed, 637 insertions(+), 2 deletions(-) create mode 100644 jc/parsers/rsync.py create mode 100644 tests/fixtures/centos-7.7/rsync-ivvv.out diff --git a/jc/lib.py b/jc/lib.py index ab6b6b38..9128b0db 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -9,7 +9,7 @@ import importlib from typing import Dict, List, Iterable, Union, Iterator, Optional from jc import appdirs -__version__ = '1.18.2' +__version__ = '1.18.3' parsers = [ 'acpi', @@ -68,6 +68,7 @@ parsers = [ 'pip-show', 'ps', 'route', + 'rsync', 'rpm-qi', 'sfdisk', 'shadow', diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py new file mode 100644 index 00000000..46c2534e --- /dev/null +++ b/jc/parsers/rsync.py @@ -0,0 +1,212 @@ +"""jc - JSON CLI output utility `rsync` command output parser + +Supports the `-i` or `--itemize-changes` option with all levels of +verbosity. + +Usage (cli): + + $ rsync -i -a source/ dest | jc --rsync + + or + + $ jc rsync -i -a source/ dest + +Usage (module): + + import jc + result = jc.parse('rsync', rsync_command_output) + + or + + import jc.parsers.rsync + result = jc.parsers.rsync.parse(rsync_command_output) + +Schema: + + [ + { + "filename": string, + "metadata": string, + "update_type": string/null, + "file_type": string/null, + "checksum_or_value_different": bool/null, + "size_different": bool/null, + "modification_time_different": bool/null, + "permissions_different": bool/null, + "owner_different": bool/null, + "group_different": bool/null, + "future": null, + "acl_different": bool/null, + "extended_attribute_different": bool/null + } + ] + +Examples: + + $ rsync | jc --rsync -p + [] + + $ rsync | jc --rsync -p -r + [] +""" +import re +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`rsync` command parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'freebsd'] + magic_commands = ['rsync -i', 'rsync --itemize-changes'] + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + + # process the data here + # rebuild output for added semantic information + # use helper functions in jc.utils for int, float, bool + # conversions and timestamps + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + update_type = { + '<': 'file sent', + '>': 'file received', + 'c': 'local change or creation', + 'h': 'hard link', + '.': 'not updated', + '*': 'message', + '+': None + } + + file_type = { + 'f': 'file', + 'd': 'directory', + 'L': 'symlink', + 'D': 'device', + 'S': 'special file', + '+': None + } + + checksum_or_value_different = { + 'c': True, + '.': False, + '+': None + } + + size_different = { + 's': True, + '.': False, + '+': None + } + + modification_time_different = { + 't': True, + '.': False, + '+': None + } + + permissions_different = { + 'p': True, + '.': False, + '+': None + } + + owner_different = { + 'o': True, + '.': False, + '+': None + } + + group_different = { + 'g': True, + '.': False, + '+': None + } + + future = None + + acl_different = { + 'a': True, + '.': False, + '+': None + } + + extended_attribute_different = { + 'x': True, + '.': False, + '+': None + } + + if jc.utils.has_data(data): + + file_line_re = re.compile(r'(?P^[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P.+)') + + for line in filter(None, data.splitlines()): + + file_line = file_line_re.match(line) + if file_line: + meta = file_line.group('meta') + filename = file_line.group('name') + + output_line = { + 'filename': filename, + 'metadata': meta, + 'update_type': update_type[meta[0]], + 'file_type': file_type[meta[1]], + 'checksum_or_value_different': checksum_or_value_different[meta[2]], + 'size_different': size_different[meta[3]], + 'modification_time_different': modification_time_different[meta[4]], + 'permissions_different': permissions_different[meta[5]], + 'owner_different': owner_different[meta[6]], + 'group_different': group_different[meta[7]], + 'future': future, + 'acl_different': acl_different[meta[9]], + 'extended_attribute_different': extended_attribute_different[meta[10]] + } + + raw_output.append(output_line) + + return raw_output if raw else _process(raw_output) diff --git a/setup.py b/setup.py index 73eb28e5..f3dcc6b6 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='1.18.2', + version='1.18.3', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='Converts the output of popular command-line tools and file-types to JSON.', diff --git a/tests/fixtures/centos-7.7/rsync-ivvv.out b/tests/fixtures/centos-7.7/rsync-ivvv.out new file mode 100644 index 00000000..8ce768e9 --- /dev/null +++ b/tests/fixtures/centos-7.7/rsync-ivvv.out @@ -0,0 +1,422 @@ +sending incremental file list +[sender] make_file(.,*,0) +[sender] pushing local filters for /home/kbrazil/rsynctest/source/ +[sender] make_file(a.txt,*,2) +[sender] make_file(b.txt,*,2) +[sender] make_file(c.txt,*,2) +[sender] make_file(d.txt,*,2) +[sender] make_file(file with spaces.txt,*,2) +[sender] make_file(folder,*,2) +send_file_list done +send_files starting +[sender] pushing local filters for /home/kbrazil/rsynctest/source/folder/ +[sender] make_file(folder/file1,*,2) +[sender] make_file(folder/file2,*,2) +[sender] make_file(folder/file3,*,2) +[sender] make_file(folder/file4,*,2) +[sender] make_file(folder/file5,*,2) +[sender] make_file(folder/file6,*,2) +[sender] make_file(folder/file7,*,2) +[sender] make_file(folder/file8,*,2) +[sender] make_file(folder/file9,*,2) +[sender] make_file(folder/file10,*,2) +[sender] make_file(folder/file11,*,2) +[sender] make_file(folder/file12,*,2) +[sender] make_file(folder/file13,*,2) +[sender] make_file(folder/file14,*,2) +[sender] make_file(folder/file15,*,2) +[sender] make_file(folder/file16,*,2) +[sender] make_file(folder/file17,*,2) +[sender] make_file(folder/file18,*,2) +[sender] make_file(folder/file19,*,2) +[sender] make_file(folder/file20,*,2) +server_recv(2) starting pid=7804 +recv_file_name(.) +recv_file_name(a.txt) +recv_file_name(b.txt) +recv_file_name(c.txt) +recv_file_name(d.txt) +recv_file_name(file with spaces.txt) +recv_file_name(folder) +received 7 names +recv_file_list done +get_local_name count=7 dest +generator starting pid=7804 +delta-transmission disabled for local transfer or --whole-file +recv_generator(.,0) +set modtime of . to (1643343337) Thu Jan 27 20:15:37 2022 +recv_generator(.,1) +recv_generator(a.txt,2) +recv_generator(b.txt,3) +recv_generator(c.txt,4) +recv_generator(d.txt,5) +recv_generator(file with spaces.txt,6) +recv_generator(folder,7) +send_files(0, source/.) +.d..t...... ./ +send_files(2, source/a.txt) +send_files mapped source/a.txt of size 47 +calling match_sums source/a.txt +>f+++++++++ a.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/a.txt +send_files(3, source/b.txt) +send_files mapped source/b.txt of size 47 +calling match_sums source/b.txt +>f+++++++++ b.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/b.txt +send_files(4, source/c.txt) +send_files mapped source/c.txt of size 47 +calling match_sums source/c.txt +>f+++++++++ c.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/c.txt +send_files(5, source/d.txt) +send_files mapped source/d.txt of size 47 +calling match_sums source/d.txt +>f+++++++++ d.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/d.txt +send_files(6, source/file with spaces.txt) +send_files mapped source/file with spaces.txt of size 47 +calling match_sums source/file with spaces.txt +>f+++++++++ file with spaces.txt +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/file with spaces.txt +recv_files(7) starting +[receiver] receiving flist for dir 1 +recv_file_name(folder/file1) +recv_file_name(folder/file2) +recv_file_name(folder/file3) +recv_file_name(folder/file4) +recv_file_name(folder/file5) +recv_file_name(folder/file6) +recv_file_name(folder/file7) +recv_file_name(folder/file8) +recv_file_name(folder/file9) +recv_file_name(folder/file10) +recv_file_name(folder/file11) +recv_file_name(folder/file12) +recv_file_name(folder/file13) +recv_file_name(folder/file14) +recv_file_name(folder/file15) +recv_file_name(folder/file16) +recv_file_name(folder/file17) +recv_file_name(folder/file18) +recv_file_name(folder/file19) +recv_file_name(folder/file20) +received 20 names +recv_file_list done +recv_files(.) +recv_files(a.txt) +got file_sum +set modtime of .a.txt.WA5SfS to (1643342949) Thu Jan 27 20:09:09 2022 +renaming .a.txt.WA5SfS to a.txt +recv_files(b.txt) +got file_sum +set modtime of .b.txt.b6U9z7 to (1643342953) Thu Jan 27 20:09:13 2022 +renaming .b.txt.b6U9z7 to b.txt +recv_files(c.txt) +got file_sum +set modtime of .c.txt.0ZRrUm to (1643342956) Thu Jan 27 20:09:16 2022 +renaming .c.txt.0ZRrUm to c.txt +recv_files(d.txt) +got file_sum +set modtime of .d.txt.N3zKeC to (1643342959) Thu Jan 27 20:09:19 2022 +renaming .d.txt.N3zKeC to d.txt +recv_files(file with spaces.txt) +got file_sum +set modtime of .file with spaces.txt.KN33yR to (1643342980) Thu Jan 27 20:09:40 2022 +renaming .file with spaces.txt.KN33yR to file with spaces.txt +[generator] receiving flist for dir 1 +recv_file_name(folder/file1) +recv_file_name(folder/file2) +recv_file_name(folder/file3) +recv_file_name(folder/file4) +recv_file_name(folder/file5) +recv_file_name(folder/file6) +recv_file_name(folder/file7) +recv_file_name(folder/file8) +recv_file_name(folder/file9) +recv_file_name(folder/file10) +recv_file_name(folder/file11) +recv_file_name(folder/file12) +recv_file_name(folder/file13) +recv_file_name(folder/file14) +recv_file_name(folder/file15) +recv_file_name(folder/file16) +recv_file_name(folder/file17) +recv_file_name(folder/file18) +recv_file_name(folder/file19) +recv_file_name(folder/file20) +received 20 names +recv_file_list done +recv_generator(folder,8) +set modtime of folder to (1643343369) Thu Jan 27 20:16:09 2022 +recv_generator(folder/file1,9) +set modtime of . to (1643343337) Thu Jan 27 20:15:37 2022 +recv_generator(folder/file10,10) +recv_generator(folder/file11,11) +recv_generator(folder/file12,12) +recv_generator(folder/file13,13) +recv_generator(folder/file14,14) +recv_generator(folder/file15,15) +recv_generator(folder/file16,16) +recv_generator(folder/file17,17) +recv_generator(folder/file18,18) +recv_generator(folder/file19,19) +recv_generator(folder/file2,20) +recv_generator(folder/file20,21) +recv_generator(folder/file3,22) +recv_generator(folder/file4,23) +recv_generator(folder/file5,24) +recv_generator(folder/file6,25) +recv_generator(folder/file7,26) +recv_generator(folder/file8,27) +recv_generator(folder/file9,28) +generate_files phase=1 +send_files(8, source/folder) +cd+++++++++ folder/ +send_files(9, source/folder/file1) +send_files mapped source/folder/file1 of size 0 +calling match_sums source/folder/file1 +>f+++++++++ folder/file1 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file1 +send_files(10, source/folder/file10) +send_files mapped source/folder/file10 of size 0 +calling match_sums source/folder/file10 +>f+++++++++ folder/file10 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file10 +send_files(11, source/folder/file11) +send_files mapped source/folder/file11 of size 0 +calling match_sums source/folder/file11 +>f+++++++++ folder/file11 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file11 +send_files(12, source/folder/file12) +send_files mapped source/folder/file12 of size 0 +calling match_sums source/folder/file12 +>f+++++++++ folder/file12 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file12 +send_files(13, source/folder/file13) +send_files mapped source/folder/file13 of size 0 +calling match_sums source/folder/file13 +>f+++++++++ folder/file13 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file13 +send_files(14, source/folder/file14) +send_files mapped source/folder/file14 of size 0 +calling match_sums source/folder/file14 +>f+++++++++ folder/file14 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file14 +send_files(15, source/folder/file15) +send_files mapped source/folder/file15 of size 0 +calling match_sums source/folder/file15 +>f+++++++++ folder/file15 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file15 +send_files(16, source/folder/file16) +send_files mapped source/folder/file16 of size 0 +calling match_sums source/folder/file16 +>f+++++++++ folder/file16 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file16 +send_files(17, source/folder/file17) +send_files mapped source/folder/file17 of size 0 +calling match_sums source/folder/file17 +>f+++++++++ folder/file17 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file17 +send_files(18, source/folder/file18) +send_files mapped source/folder/file18 of size 0 +calling match_sums source/folder/file18 +>f+++++++++ folder/file18 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file18 +send_files(19, source/folder/file19) +send_files mapped source/folder/file19 of size 0 +calling match_sums source/folder/file19 +>f+++++++++ folder/file19 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file19 +send_files(20, source/folder/file2) +send_files mapped source/folder/file2 of size 0 +calling match_sums source/folder/file2 +>f+++++++++ folder/file2 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file2 +send_files(21, source/folder/file20) +send_files mapped source/folder/file20 of size 0 +calling match_sums source/folder/file20 +>f+++++++++ folder/file20 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file20 +send_files(22, source/folder/file3) +send_files mapped source/folder/file3 of size 0 +calling match_sums source/folder/file3 +>f+++++++++ folder/file3 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file3 +send_files(23, source/folder/file4) +send_files mapped source/folder/file4 of size 0 +calling match_sums source/folder/file4 +>f+++++++++ folder/file4 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file4 +send_files(24, source/folder/file5) +send_files mapped source/folder/file5 of size 0 +calling match_sums source/folder/file5 +>f+++++++++ folder/file5 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file5 +send_files(25, source/folder/file6) +send_files mapped source/folder/file6 of size 0 +calling match_sums source/folder/file6 +>f+++++++++ folder/file6 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file6 +send_files(26, source/folder/file7) +send_files mapped source/folder/file7 of size 0 +calling match_sums source/folder/file7 +>f+++++++++ folder/file7 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file7 +send_files(27, source/folder/file8) +send_files mapped source/folder/file8 of size 0 +calling match_sums source/folder/file8 +>f+++++++++ folder/file8 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file8 +send_files(28, source/folder/file9) +send_files mapped source/folder/file9 of size 0 +calling match_sums source/folder/file9 +>f+++++++++ folder/file9 +sending file_sum +false_alarms=0 hash_hits=0 matches=0 +sender finished source/folder/file9 +recv_files(folder) +recv_files(folder/file1) +got file_sum +set modtime of folder/.file1.iqSNT6 to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file1.iqSNT6 to folder/file1 +recv_files(folder/file10) +got file_sum +set modtime of folder/.file10.AlTyem to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file10.AlTyem to folder/file10 +recv_files(folder/file11) +got file_sum +set modtime of folder/.file11.McEkzB to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file11.McEkzB to folder/file11 +recv_files(folder/file12) +got file_sum +set modtime of folder/.file12.6i46TQ to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file12.6i46TQ to folder/file12 +recv_files(folder/file13) +got file_sum +set modtime of folder/.file13.GE3Te6 to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file13.GE3Te6 to folder/file13 +recv_files(folder/file14) +got file_sum +set modtime of folder/.file14.iHFHzl to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file14.iHFHzl to folder/file14 +recv_files(folder/file15) +got file_sum +set modtime of folder/.file15.QXUvUA to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file15.QXUvUA to folder/file15 +recv_files(folder/file16) +got file_sum +set modtime of folder/.file16.avHkfQ to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file16.avHkfQ to folder/file16 +recv_files(folder/file17) +got file_sum +set modtime of folder/.file17.wH89z5 to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file17.wH89z5 to folder/file17 +recv_files(folder/file18) +got file_sum +set modtime of folder/.file18.Kpj0Uk to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file18.Kpj0Uk to folder/file18 +recv_files(folder/file19) +got file_sum +set modtime of folder/.file19.885QfA to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file19.885QfA to folder/file19 +recv_files(folder/file2) +got file_sum +set modtime of folder/.file2.SQrIAP to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file2.SQrIAP to folder/file2 +recv_files(folder/file20) +got file_sum +set modtime of folder/.file20.kXpAV4 to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file20.kXpAV4 to folder/file20 +recv_files(folder/file3) +got file_sum +set modtime of folder/.file3.GRdtgk to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file3.GRdtgk to folder/file3 +recv_files(folder/file4) +got file_sum +set modtime of folder/.file4.uJGmBz to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file4.uJGmBz to folder/file4 +recv_files(folder/file5) +got file_sum +set modtime of folder/.file5.OPPgWO to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file5.OPPgWO to folder/file5 +recv_files(folder/file6) +got file_sum +set modtime of folder/.file6.abEbh4 to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file6.abEbh4 to folder/file6 +recv_files(folder/file7) +got file_sum +set modtime of folder/.file7.4T46Bj to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file7.4T46Bj to folder/file7 +recv_files(folder/file8) +got file_sum +set modtime of folder/.file8.CA62Wy to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file8.CA62Wy to folder/file8 +recv_files(folder/file9) +got file_sum +set modtime of folder/.file9.Yu80hO to (1643343369) Thu Jan 27 20:16:09 2022 +renaming folder/.file9.Yu80hO to folder/file9 +set modtime of folder to (1643343369) Thu Jan 27 20:16:09 2022 +send_files phase=1 +recv_files phase=1 +generate_files phase=2 +send_files phase=2 +send files finished +total: matches=0 hash_hits=0 false_alarms=0 data=235 +recv_files phase=2 +recv_files finished +generate_files phase=3 +generate_files finished + +sent 1,708 bytes received 8,209 bytes 19,834.00 bytes/sec +total size is 235 speedup is 0.02 +[sender] _exit_cleanup(code=0, file=main.c, line=1178): about to call exit(0) + From 38f04b1c9699246d85332b20df5b6a91ae0c48d9 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 28 Jan 2022 14:45:01 -0800 Subject: [PATCH 03/74] add summary --- jc/parsers/rsync.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py index 46c2534e..9aaa2df6 100644 --- a/jc/parsers/rsync.py +++ b/jc/parsers/rsync.py @@ -25,6 +25,7 @@ Schema: [ { + "type": string, # 'file' or 'summary' "filename": string, "metadata": string, "update_type": string/null, @@ -37,7 +38,12 @@ Schema: "group_different": bool/null, "future": null, "acl_different": bool/null, - "extended_attribute_different": bool/null + "extended_attribute_different": bool/null, + "sent": integer, # need to convert + "received": integer, # need to convert + "bytes_sec": float, # need to convert + "total_size": integer, # need to convert + "speedup": float, # need to convert } ] @@ -79,12 +85,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - - # process the data here - # rebuild output for added semantic information - # use helper functions in jc.utils for int, float, bool - # conversions and timestamps - + # no further processing needed return proc_data @@ -180,18 +181,26 @@ def parse( '+': None } + summary = {} + if jc.utils.has_data(data): file_line_re = re.compile(r'(?P^[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P.+)') + stat1_line_re = re.compile(r'(sent)\s+(?P[0-9,]+)\s+(bytes)\s+(received)\s+(?P[0-9,]+)\s+(bytes)\s+(?P[0-9,.]+)\s+(bytes/sec)') + stat2_line_re = re.compile(r'(total size is)\s+(?P[0-9,]+)\s+(speedup is)\s+(?P[0-9,.]+)') for line in filter(None, data.splitlines()): file_line = file_line_re.match(line) + stat1_line = stat1_line_re.match(line) + stat2_line = stat2_line_re.match(line) + if file_line: meta = file_line.group('meta') filename = file_line.group('name') output_line = { + 'type': 'file', 'filename': filename, 'metadata': meta, 'update_type': update_type[meta[0]], @@ -208,5 +217,21 @@ def parse( } raw_output.append(output_line) + continue + + if stat1_line: + summary = { + 'type': 'summary', + 'sent': stat1_line.group('sent'), + 'received': stat1_line.group('received'), + 'bytes_sec': stat1_line.group('bytes_sec') + } + continue + + if stat2_line: + summary['total_size'] = stat2_line.group('total_size') + summary['speedup'] = stat2_line.group('speedup') + raw_output.append(summary) + continue return raw_output if raw else _process(raw_output) From d6de81747fd92642dbd5cb1fb097e375aa38be35 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 28 Jan 2022 15:00:05 -0800 Subject: [PATCH 04/74] add int/float conversions --- jc/parsers/rsync.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py index 9aaa2df6..fbaf9373 100644 --- a/jc/parsers/rsync.py +++ b/jc/parsers/rsync.py @@ -25,7 +25,7 @@ Schema: [ { - "type": string, # 'file' or 'summary' + "type": string, # 'item' or 'summary' "filename": string, "metadata": string, "update_type": string/null, @@ -85,7 +85,15 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - # no further processing needed + int_list = ['sent', 'received', 'total_size'] + float_list = ['bytes_sec', 'speedup'] + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + if key in float_list: + entry[key] = jc.utils.convert_to_float(entry[key]) + return proc_data @@ -200,7 +208,7 @@ def parse( filename = file_line.group('name') output_line = { - 'type': 'file', + 'type': 'item', 'filename': filename, 'metadata': meta, 'update_type': update_type[meta[0]], From ea5011b616365b5c10c04416d43035d84e3dacba Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 29 Jan 2022 17:24:43 -0800 Subject: [PATCH 05/74] updated schema. add log-file option support --- jc/parsers/rsync.py | 157 +++++++++++++----- tests/fixtures/centos-7.7/rsync-i-logfile.out | 59 +++++++ 2 files changed, 175 insertions(+), 41 deletions(-) create mode 100644 tests/fixtures/centos-7.7/rsync-i-logfile.out diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py index fbaf9373..02adbe03 100644 --- a/jc/parsers/rsync.py +++ b/jc/parsers/rsync.py @@ -25,25 +25,34 @@ Schema: [ { - "type": string, # 'item' or 'summary' - "filename": string, - "metadata": string, - "update_type": string/null, - "file_type": string/null, - "checksum_or_value_different": bool/null, - "size_different": bool/null, - "modification_time_different": bool/null, - "permissions_different": bool/null, - "owner_different": bool/null, - "group_different": bool/null, - "future": null, - "acl_different": bool/null, - "extended_attribute_different": bool/null, - "sent": integer, # need to convert - "received": integer, # need to convert - "bytes_sec": float, # need to convert - "total_size": integer, # need to convert - "speedup": float, # need to convert + "summary": { + "date": string, + "time": string, + "process": integer, # need to convert + "sent": integer, # need to convert + "received": integer, # need to convert + "total_size": integer # need to convert + }, + "items": [ + "filename": string, + "metadata": string, + "update_type": string/null, + "file_type": string/null, + "checksum_or_value_different": bool/null, + "size_different": bool/null, + "modification_time_different": bool/null, + "permissions_different": bool/null, + "owner_different": bool/null, + "group_different": bool/null, + "future": null, + "acl_different": bool/null, + "extended_attribute_different": bool/null, + "sent": integer, # need to convert + "received": integer, # need to convert + "bytes_sec": float, # need to convert + "total_size": integer, # need to convert + "speedup": float, # need to convert + ] } ] @@ -85,14 +94,14 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = ['sent', 'received', 'total_size'] - float_list = ['bytes_sec', 'speedup'] - for entry in proc_data: - for key in entry: - if key in int_list: - entry[key] = jc.utils.convert_to_int(entry[key]) - if key in float_list: - entry[key] = jc.utils.convert_to_float(entry[key]) + # int_list = ['sent', 'received', 'total_size'] + # float_list = ['bytes_sec', 'speedup'] + # for entry in proc_data: + # for key in entry: + # if key in int_list: + # entry[key] = jc.utils.convert_to_int(entry[key]) + # if key in float_list: + # entry[key] = jc.utils.convert_to_float(entry[key]) return proc_data @@ -119,6 +128,9 @@ def parse( jc.utils.input_type_check(data) raw_output: List = [] + rsync_run: Dict = {} + + last_process = '' update_type = { '<': 'file sent', @@ -189,26 +201,35 @@ def parse( '+': None } - summary = {} - if jc.utils.has_data(data): + rsync_run.update({ + 'summary': {}, + 'items': [] + }) - file_line_re = re.compile(r'(?P^[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P.+)') + file_line_re = re.compile(r'(?P[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P.+)') stat1_line_re = re.compile(r'(sent)\s+(?P[0-9,]+)\s+(bytes)\s+(received)\s+(?P[0-9,]+)\s+(bytes)\s+(?P[0-9,.]+)\s+(bytes/sec)') stat2_line_re = re.compile(r'(total size is)\s+(?P[0-9,]+)\s+(speedup is)\s+(?P[0-9,.]+)') + file_line_log_re = re.compile(r'(?P\d\d\d\d/\d\d/\d\d)\s+(?P