python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-17 00:07:37 +02:00
Code Issues Releases Activity

add rsync parser

Browse Source
This commit is contained in:
Kelly Brazil
2022-01-28 14:08:43 -08:00
parent 1f96586a5e
commit 84169e1a91
4 changed files with 637 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
jc/lib.py
View File

@ -9,7 +9,7 @@ import importlib
from typing import Dict, List, Iterable, Union, Iterator, Optional
from jc import appdirs
__version__ = '1.18.2'
__version__ = '1.18.3'
parsers = [
'acpi',
@ -68,6 +68,7 @@ parsers = [
'pip-show',
'ps',
'route',
'rsync',
'rpm-qi',
'sfdisk',
'shadow',

212
jc/parsers/rsync.py Normal file
View File

@ -0,0 +1,212 @@
"""jc - JSON CLI output utility `rsync` command output parser
Supports the `-i` or `--itemize-changes` option with all levels of
verbosity.
Usage (cli):
$ rsync -i -a source/ dest | jc --rsync
or
$ jc rsync -i -a source/ dest
Usage (module):
import jc
result = jc.parse('rsync', rsync_command_output)
or
import jc.parsers.rsync
result = jc.parsers.rsync.parse(rsync_command_output)
Schema:
[
{
"filename": string,
"metadata": string,
"update_type": string/null,
"file_type": string/null,
"checksum_or_value_different": bool/null,
"size_different": bool/null,
"modification_time_different": bool/null,
"permissions_different": bool/null,
"owner_different": bool/null,
"group_different": bool/null,
"future": null,
"acl_different": bool/null,
"extended_attribute_different": bool/null
}
]
Examples:
$ rsync | jc --rsync -p
[]
$ rsync | jc --rsync -p -r
[]
"""
import re
from typing import List, Dict
import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.0'
description = '`rsync` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
compatible = ['linux', 'darwin', 'cygwin', 'freebsd']
magic_commands = ['rsync -i', 'rsync --itemize-changes']
__version__ = info.version
def _process(proc_data: List[Dict]) -> List[Dict]:
"""
Final processing to conform to the schema.
Parameters:
proc_data: (List of Dictionaries) raw structured data to process
Returns:
List of Dictionaries. Structured to conform to the schema.
"""
# process the data here
# rebuild output for added semantic information
# use helper functions in jc.utils for int, float, bool
# conversions and timestamps
return proc_data
def parse(
data: str,
raw: bool = False,
quiet: bool = False
) -> List[Dict]:
"""
Main text parsing function
Parameters:
data: (string) text data to parse
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
Returns:
List of Dictionaries. Raw or processed structured data.
"""
jc.utils.compatibility(__name__, info.compatible, quiet)
jc.utils.input_type_check(data)
raw_output: List = []
update_type = {
'<': 'file sent',
'>': 'file received',
'c': 'local change or creation',
'h': 'hard link',
'.': 'not updated',
'*': 'message',
'+': None
}
file_type = {
'f': 'file',
'd': 'directory',
'L': 'symlink',
'D': 'device',
'S': 'special file',
'+': None
}
checksum_or_value_different = {
'c': True,
'.': False,
'+': None
}
size_different = {
's': True,
'.': False,
'+': None
}
modification_time_different = {
't': True,
'.': False,
'+': None
}
permissions_different = {
'p': True,
'.': False,
'+': None
}
owner_different = {
'o': True,
'.': False,
'+': None
}
group_different = {
'g': True,
'.': False,
'+': None
}
future = None
acl_different = {
'a': True,
'.': False,
'+': None
}
extended_attribute_different = {
'x': True,
'.': False,
'+': None
}
if jc.utils.has_data(data):
file_line_re = re.compile(r'(?P<meta>^[<>ch.*][fdlDS][c.+][s.+][t.+][p.+][o.+][g.+][u.+][a.+][x.+]) (?P<name>.+)')
for line in filter(None, data.splitlines()):
file_line = file_line_re.match(line)
if file_line:
meta = file_line.group('meta')
filename = file_line.group('name')
output_line = {
'filename': filename,
'metadata': meta,
'update_type': update_type[meta[0]],
'file_type': file_type[meta[1]],
'checksum_or_value_different': checksum_or_value_different[meta[2]],
'size_different': size_different[meta[3]],
'modification_time_different': modification_time_different[meta[4]],
'permissions_different': permissions_different[meta[5]],
'owner_different': owner_different[meta[6]],
'group_different': group_different[meta[7]],
'future': future,
'acl_different': acl_different[meta[9]],
'extended_attribute_different': extended_attribute_different[meta[10]]
}
raw_output.append(output_line)
return raw_output if raw else _process(raw_output)

2
setup.py
View File

@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
setuptools.setup(
name='jc',
version='1.18.2',
version='1.18.3',
author='Kelly Brazil',
author_email='kellyjonbrazil@gmail.com',
description='Converts the output of popular command-line tools and file-types to JSON.',

422
tests/fixtures/centos-7.7/rsync-ivvv.out vendored Normal file
View File

@ -0,0 +1,422 @@
sending incremental file list
[sender] make_file(.,*,0)
[sender] pushing local filters for /home/kbrazil/rsynctest/source/
[sender] make_file(a.txt,*,2)
[sender] make_file(b.txt,*,2)
[sender] make_file(c.txt,*,2)
[sender] make_file(d.txt,*,2)
[sender] make_file(file with spaces.txt,*,2)
[sender] make_file(folder,*,2)
send_file_list done
send_files starting
[sender] pushing local filters for /home/kbrazil/rsynctest/source/folder/
[sender] make_file(folder/file1,*,2)
[sender] make_file(folder/file2,*,2)
[sender] make_file(folder/file3,*,2)
[sender] make_file(folder/file4,*,2)
[sender] make_file(folder/file5,*,2)
[sender] make_file(folder/file6,*,2)
[sender] make_file(folder/file7,*,2)
[sender] make_file(folder/file8,*,2)
[sender] make_file(folder/file9,*,2)
[sender] make_file(folder/file10,*,2)
[sender] make_file(folder/file11,*,2)
[sender] make_file(folder/file12,*,2)
[sender] make_file(folder/file13,*,2)
[sender] make_file(folder/file14,*,2)
[sender] make_file(folder/file15,*,2)
[sender] make_file(folder/file16,*,2)
[sender] make_file(folder/file17,*,2)
[sender] make_file(folder/file18,*,2)
[sender] make_file(folder/file19,*,2)
[sender] make_file(folder/file20,*,2)
server_recv(2) starting pid=7804
recv_file_name(.)
recv_file_name(a.txt)
recv_file_name(b.txt)
recv_file_name(c.txt)
recv_file_name(d.txt)
recv_file_name(file with spaces.txt)
recv_file_name(folder)
received 7 names
recv_file_list done
get_local_name count=7 dest
generator starting pid=7804
delta-transmission disabled for local transfer or --whole-file
recv_generator(.,0)
set modtime of . to (1643343337) Thu Jan 27 20:15:37 2022
recv_generator(.,1)
recv_generator(a.txt,2)
recv_generator(b.txt,3)
recv_generator(c.txt,4)
recv_generator(d.txt,5)
recv_generator(file with spaces.txt,6)
recv_generator(folder,7)
send_files(0, source/.)
.d..t...... ./
send_files(2, source/a.txt)
send_files mapped source/a.txt of size 47
calling match_sums source/a.txt
>f+++++++++ a.txt
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/a.txt
send_files(3, source/b.txt)
send_files mapped source/b.txt of size 47
calling match_sums source/b.txt
>f+++++++++ b.txt
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/b.txt
send_files(4, source/c.txt)
send_files mapped source/c.txt of size 47
calling match_sums source/c.txt
>f+++++++++ c.txt
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/c.txt
send_files(5, source/d.txt)
send_files mapped source/d.txt of size 47
calling match_sums source/d.txt
>f+++++++++ d.txt
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/d.txt
send_files(6, source/file with spaces.txt)
send_files mapped source/file with spaces.txt of size 47
calling match_sums source/file with spaces.txt
>f+++++++++ file with spaces.txt
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/file with spaces.txt
recv_files(7) starting
[receiver] receiving flist for dir 1
recv_file_name(folder/file1)
recv_file_name(folder/file2)
recv_file_name(folder/file3)
recv_file_name(folder/file4)
recv_file_name(folder/file5)
recv_file_name(folder/file6)
recv_file_name(folder/file7)
recv_file_name(folder/file8)
recv_file_name(folder/file9)
recv_file_name(folder/file10)
recv_file_name(folder/file11)
recv_file_name(folder/file12)
recv_file_name(folder/file13)
recv_file_name(folder/file14)
recv_file_name(folder/file15)
recv_file_name(folder/file16)
recv_file_name(folder/file17)
recv_file_name(folder/file18)
recv_file_name(folder/file19)
recv_file_name(folder/file20)
received 20 names
recv_file_list done
recv_files(.)
recv_files(a.txt)
got file_sum
set modtime of .a.txt.WA5SfS to (1643342949) Thu Jan 27 20:09:09 2022
renaming .a.txt.WA5SfS to a.txt
recv_files(b.txt)
got file_sum
set modtime of .b.txt.b6U9z7 to (1643342953) Thu Jan 27 20:09:13 2022
renaming .b.txt.b6U9z7 to b.txt
recv_files(c.txt)
got file_sum
set modtime of .c.txt.0ZRrUm to (1643342956) Thu Jan 27 20:09:16 2022
renaming .c.txt.0ZRrUm to c.txt
recv_files(d.txt)
got file_sum
set modtime of .d.txt.N3zKeC to (1643342959) Thu Jan 27 20:09:19 2022
renaming .d.txt.N3zKeC to d.txt
recv_files(file with spaces.txt)
got file_sum
set modtime of .file with spaces.txt.KN33yR to (1643342980) Thu Jan 27 20:09:40 2022
renaming .file with spaces.txt.KN33yR to file with spaces.txt
[generator] receiving flist for dir 1
recv_file_name(folder/file1)
recv_file_name(folder/file2)
recv_file_name(folder/file3)
recv_file_name(folder/file4)
recv_file_name(folder/file5)
recv_file_name(folder/file6)
recv_file_name(folder/file7)
recv_file_name(folder/file8)
recv_file_name(folder/file9)
recv_file_name(folder/file10)
recv_file_name(folder/file11)
recv_file_name(folder/file12)
recv_file_name(folder/file13)
recv_file_name(folder/file14)
recv_file_name(folder/file15)
recv_file_name(folder/file16)
recv_file_name(folder/file17)
recv_file_name(folder/file18)
recv_file_name(folder/file19)
recv_file_name(folder/file20)
received 20 names
recv_file_list done
recv_generator(folder,8)
set modtime of folder to (1643343369) Thu Jan 27 20:16:09 2022
recv_generator(folder/file1,9)
set modtime of . to (1643343337) Thu Jan 27 20:15:37 2022
recv_generator(folder/file10,10)
recv_generator(folder/file11,11)
recv_generator(folder/file12,12)
recv_generator(folder/file13,13)
recv_generator(folder/file14,14)
recv_generator(folder/file15,15)
recv_generator(folder/file16,16)
recv_generator(folder/file17,17)
recv_generator(folder/file18,18)
recv_generator(folder/file19,19)
recv_generator(folder/file2,20)
recv_generator(folder/file20,21)
recv_generator(folder/file3,22)
recv_generator(folder/file4,23)
recv_generator(folder/file5,24)
recv_generator(folder/file6,25)
recv_generator(folder/file7,26)
recv_generator(folder/file8,27)
recv_generator(folder/file9,28)
generate_files phase=1
send_files(8, source/folder)
cd+++++++++ folder/
send_files(9, source/folder/file1)
send_files mapped source/folder/file1 of size 0
calling match_sums source/folder/file1
>f+++++++++ folder/file1
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file1
send_files(10, source/folder/file10)
send_files mapped source/folder/file10 of size 0
calling match_sums source/folder/file10
>f+++++++++ folder/file10
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file10
send_files(11, source/folder/file11)
send_files mapped source/folder/file11 of size 0
calling match_sums source/folder/file11
>f+++++++++ folder/file11
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file11
send_files(12, source/folder/file12)
send_files mapped source/folder/file12 of size 0
calling match_sums source/folder/file12
>f+++++++++ folder/file12
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file12
send_files(13, source/folder/file13)
send_files mapped source/folder/file13 of size 0
calling match_sums source/folder/file13
>f+++++++++ folder/file13
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file13
send_files(14, source/folder/file14)
send_files mapped source/folder/file14 of size 0
calling match_sums source/folder/file14
>f+++++++++ folder/file14
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file14
send_files(15, source/folder/file15)
send_files mapped source/folder/file15 of size 0
calling match_sums source/folder/file15
>f+++++++++ folder/file15
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file15
send_files(16, source/folder/file16)
send_files mapped source/folder/file16 of size 0
calling match_sums source/folder/file16
>f+++++++++ folder/file16
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file16
send_files(17, source/folder/file17)
send_files mapped source/folder/file17 of size 0
calling match_sums source/folder/file17
>f+++++++++ folder/file17
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file17
send_files(18, source/folder/file18)
send_files mapped source/folder/file18 of size 0
calling match_sums source/folder/file18
>f+++++++++ folder/file18
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file18
send_files(19, source/folder/file19)
send_files mapped source/folder/file19 of size 0
calling match_sums source/folder/file19
>f+++++++++ folder/file19
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file19
send_files(20, source/folder/file2)
send_files mapped source/folder/file2 of size 0
calling match_sums source/folder/file2
>f+++++++++ folder/file2
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file2
send_files(21, source/folder/file20)
send_files mapped source/folder/file20 of size 0
calling match_sums source/folder/file20
>f+++++++++ folder/file20
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file20
send_files(22, source/folder/file3)
send_files mapped source/folder/file3 of size 0
calling match_sums source/folder/file3
>f+++++++++ folder/file3
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file3
send_files(23, source/folder/file4)
send_files mapped source/folder/file4 of size 0
calling match_sums source/folder/file4
>f+++++++++ folder/file4
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file4
send_files(24, source/folder/file5)
send_files mapped source/folder/file5 of size 0
calling match_sums source/folder/file5
>f+++++++++ folder/file5
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file5
send_files(25, source/folder/file6)
send_files mapped source/folder/file6 of size 0
calling match_sums source/folder/file6
>f+++++++++ folder/file6
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file6
send_files(26, source/folder/file7)
send_files mapped source/folder/file7 of size 0
calling match_sums source/folder/file7
>f+++++++++ folder/file7
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file7
send_files(27, source/folder/file8)
send_files mapped source/folder/file8 of size 0
calling match_sums source/folder/file8
>f+++++++++ folder/file8
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file8
send_files(28, source/folder/file9)
send_files mapped source/folder/file9 of size 0
calling match_sums source/folder/file9
>f+++++++++ folder/file9
sending file_sum
false_alarms=0 hash_hits=0 matches=0
sender finished source/folder/file9
recv_files(folder)
recv_files(folder/file1)
got file_sum
set modtime of folder/.file1.iqSNT6 to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file1.iqSNT6 to folder/file1
recv_files(folder/file10)
got file_sum
set modtime of folder/.file10.AlTyem to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file10.AlTyem to folder/file10
recv_files(folder/file11)
got file_sum
set modtime of folder/.file11.McEkzB to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file11.McEkzB to folder/file11
recv_files(folder/file12)
got file_sum
set modtime of folder/.file12.6i46TQ to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file12.6i46TQ to folder/file12
recv_files(folder/file13)
got file_sum
set modtime of folder/.file13.GE3Te6 to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file13.GE3Te6 to folder/file13
recv_files(folder/file14)
got file_sum
set modtime of folder/.file14.iHFHzl to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file14.iHFHzl to folder/file14
recv_files(folder/file15)
got file_sum
set modtime of folder/.file15.QXUvUA to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file15.QXUvUA to folder/file15
recv_files(folder/file16)
got file_sum
set modtime of folder/.file16.avHkfQ to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file16.avHkfQ to folder/file16
recv_files(folder/file17)
got file_sum
set modtime of folder/.file17.wH89z5 to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file17.wH89z5 to folder/file17
recv_files(folder/file18)
got file_sum
set modtime of folder/.file18.Kpj0Uk to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file18.Kpj0Uk to folder/file18
recv_files(folder/file19)
got file_sum
set modtime of folder/.file19.885QfA to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file19.885QfA to folder/file19
recv_files(folder/file2)
got file_sum
set modtime of folder/.file2.SQrIAP to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file2.SQrIAP to folder/file2
recv_files(folder/file20)
got file_sum
set modtime of folder/.file20.kXpAV4 to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file20.kXpAV4 to folder/file20
recv_files(folder/file3)
got file_sum
set modtime of folder/.file3.GRdtgk to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file3.GRdtgk to folder/file3
recv_files(folder/file4)
got file_sum
set modtime of folder/.file4.uJGmBz to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file4.uJGmBz to folder/file4
recv_files(folder/file5)
got file_sum
set modtime of folder/.file5.OPPgWO to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file5.OPPgWO to folder/file5
recv_files(folder/file6)
got file_sum
set modtime of folder/.file6.abEbh4 to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file6.abEbh4 to folder/file6
recv_files(folder/file7)
got file_sum
set modtime of folder/.file7.4T46Bj to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file7.4T46Bj to folder/file7
recv_files(folder/file8)
got file_sum
set modtime of folder/.file8.CA62Wy to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file8.CA62Wy to folder/file8
recv_files(folder/file9)
got file_sum
set modtime of folder/.file9.Yu80hO to (1643343369) Thu Jan 27 20:16:09 2022
renaming folder/.file9.Yu80hO to folder/file9
set modtime of folder to (1643343369) Thu Jan 27 20:16:09 2022
send_files phase=1
recv_files phase=1
generate_files phase=2
send_files phase=2
send files finished
total: matches=0 hash_hits=0 false_alarms=0 data=235
recv_files phase=2
recv_files finished
generate_files phase=3
generate_files finished
sent 1,708 bytes received 8,209 bytes 19,834.00 bytes/sec
total size is 235 speedup is 0.02
[sender] _exit_cleanup(code=0, file=main.c, line=1178): about to call exit(0)