diff --git a/CHANGELOG b/CHANGELOG index 32520170..89997433 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,13 @@ jc changelog +20220214 v1.18.3 +- Add rsync command and log file parser tested on linux and macOS +- Add rsync command and log file streaming parser tested on linux and macOS +- Add xrandr command parser tested on linux +- Enhance timestamp performance with caching and format hints +- Refactor ignore_exceptions functionality in streaming parsers +- Fix man page in packages + 20220127 v1.18.2 - Fix for plugin parsers with underscores in the name - Add type hints to public API functions diff --git a/EXAMPLES.md b/EXAMPLES.md index e058b995..3bd6cfee 100644 --- a/EXAMPLES.md +++ b/EXAMPLES.md @@ -2743,6 +2743,39 @@ rpm_qia | jc --rpm_qi -p # or: jc -p rpm -qia } ] ``` +### rsync +```bash +rsync -i -a source/ dest | jc --rsync -p # or jc -p rsync -i -a source/ dest +``` +```json +[ + { + "summary": { + "sent": 1708, + "received": 8209, + "bytes_sec": 19834.0, + "total_size": 235, + "speedup": 0.02 + }, + "files": [ + { + "filename": "./", + "metadata": ".d..t......", + "update_type": "not updated", + "file_type": "directory", + "checksum_or_value_different": false, + "size_different": false, + "modification_time_different": true, + "permissions_different": false, + "owner_different": false, + "group_different": false, + "acl_different": false, + "extended_attribute_different": false + } + ] + } +] +``` ### sfdisk ```bash sfdisk -l | jc --sfdisk -p # or jc -p sfdisk -l diff --git a/README.md b/README.md index 7f48230d..f9743e0d 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ pip3 install jc > For more OS Packages, see https://repology.org/project/jc/versions. -### Binaries and Packages +### Binaries For precompiled binaries, see [Releases](https://github.com/kellyjonbrazil/jc/releases) on Github. @@ -208,6 +208,8 @@ option. - `--ps` enables the `ps` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ps)) - `--route` enables the `route` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/route)) - `--rpm-qi` enables the `rpm -qi` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/rpm_qi)) +- `--rsync` enables the `rsync` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/rsync)) +- `--rsync-s` enables the `rsync` command streaming parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/rsync_s)) - `--sfdisk` enables the `sfdisk` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/sfdisk)) - `--shadow` enables the `/etc/shadow` file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/shadow)) - `--ss` enables the `ss` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ss)) @@ -234,6 +236,7 @@ option. - `--wc` enables the `wc` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/wc)) - `--who` enables the `who` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/who)) - `--xml` enables the XML file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/xml)) +- `--xrandr` enables the `xrandr` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/xrandr)) - `--yaml` enables the YAML file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/yaml)) - `--zipinfo` enables the `zipinfo` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/zipinfo)) diff --git a/docgen.sh b/docgen.sh index 13cd6fec..38495775 100755 --- a/docgen.sh +++ b/docgen.sh @@ -85,6 +85,9 @@ pydoc-markdown -m jc.lib "${toc_config}" > ../docs/lib.md echo Building docs for: utils pydoc-markdown -m jc.utils "${toc_config}" > ../docs/utils.md +echo Building docs for: streaming +pydoc-markdown -m jc.streaming "${toc_config}" > ../docs/streaming.md + echo Building docs for: universal parser pydoc-markdown -m jc.parsers.universal "${toc_config}" > ../docs/parsers/universal.md diff --git a/docs/lib.md b/docs/lib.md index 8d2b7c2b..6b79fe1d 100644 --- a/docs/lib.md +++ b/docs/lib.md @@ -105,7 +105,7 @@ subset of `parser_mod_list()`. ### parser\_info ```python -def parser_info(parser_mod_name: str) -> Union[Dict, None] +def parser_info(parser_mod_name: str) -> Dict ``` Returns a dictionary that includes the module metadata. @@ -118,10 +118,10 @@ This function will accept **module_name**, **cli-name**, and ### all\_parser\_info ```python -def all_parser_info() -> List[Optional[Dict]] +def all_parser_info() -> List[Dict] ``` -Returns a list of dictionaris that includes metadata for all modules. +Returns a list of dictionaries that includes metadata for all modules. diff --git a/docs/parsers/csv_s.md b/docs/parsers/csv_s.md index eee33fae..6362a7a1 100644 --- a/docs/parsers/csv_s.md +++ b/docs/parsers/csv_s.md @@ -73,6 +73,7 @@ Examples: ### parse ```python +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False) ``` @@ -93,9 +94,9 @@ Yields: Returns: - Iterator object + Iterator object (generator) ### Parser Information Compatibility: linux, darwin, cygwin, win32, aix, freebsd -Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/date.md b/docs/parsers/date.md index 5e2f4d42..e4966ba9 100644 --- a/docs/parsers/date.md +++ b/docs/parsers/date.md @@ -105,4 +105,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, freebsd -Version 2.2 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 2.3 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/dig.md b/docs/parsers/dig.md index fa3d492d..d1459682 100644 --- a/docs/parsers/dig.md +++ b/docs/parsers/dig.md @@ -350,4 +350,4 @@ Returns: ### Parser Information Compatibility: linux, aix, freebsd, darwin, win32, cygwin -Version 2.2 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 2.3 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/dir.md b/docs/parsers/dir.md index db4651c9..02e08ac5 100644 --- a/docs/parsers/dir.md +++ b/docs/parsers/dir.md @@ -148,4 +148,4 @@ Returns: ### Parser Information Compatibility: win32 -Version 1.4 by Rasheed Elsaleh (rasheed@rebelliondefense.com) +Version 1.5 by Rasheed Elsaleh (rasheed@rebelliondefense.com) diff --git a/docs/parsers/iostat_s.md b/docs/parsers/iostat_s.md index fd116a85..1fc68dac 100644 --- a/docs/parsers/iostat_s.md +++ b/docs/parsers/iostat_s.md @@ -110,6 +110,7 @@ Examples: ### parse ```python +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False) ``` @@ -130,9 +131,9 @@ Yields: Returns: - Iterator object + Iterator object (generator) ### Parser Information Compatibility: linux -Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/ls.md b/docs/parsers/ls.md index f9acc3df..2ae42823 100644 --- a/docs/parsers/ls.md +++ b/docs/parsers/ls.md @@ -144,4 +144,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, cygwin, aix, freebsd -Version 1.10 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.11 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/ls_s.md b/docs/parsers/ls_s.md index 1016aafa..bd0e5979 100644 --- a/docs/parsers/ls_s.md +++ b/docs/parsers/ls_s.md @@ -87,6 +87,7 @@ Examples: ### parse ```python +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False) ``` @@ -107,9 +108,9 @@ Yields: Returns: - Iterator object + Iterator object (generator) ### Parser Information Compatibility: linux, darwin, cygwin, aix, freebsd -Version 0.6 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/ping_s.md b/docs/parsers/ping_s.md index f31a2c68..b904e2b7 100644 --- a/docs/parsers/ping_s.md +++ b/docs/parsers/ping_s.md @@ -93,6 +93,7 @@ Examples: ### parse ```python +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False) ``` @@ -113,9 +114,9 @@ Yields: Returns: - Iterator object + Iterator object (generator) ### Parser Information Compatibility: linux, darwin, freebsd -Version 0.6 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/rpm_qi.md b/docs/parsers/rpm_qi.md index d3c4f71e..3d316d03 100644 --- a/docs/parsers/rpm_qi.md +++ b/docs/parsers/rpm_qi.md @@ -189,4 +189,4 @@ Returns: ### Parser Information Compatibility: linux -Version 1.4 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/rsync.md b/docs/parsers/rsync.md new file mode 100644 index 00000000..04f8c664 --- /dev/null +++ b/docs/parsers/rsync.md @@ -0,0 +1,165 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.rsync + +jc - JSON CLI output utility `rsync` command output parser + +Supports the `-i` or `--itemize-changes` options with all levels of +verbosity. This parser will process the STDOUT output or a log file +generated with the `--log-file` option. + +Usage (cli): + + $ rsync -i -a source/ dest | jc --rsync + + or + + $ jc rsync -i -a source/ dest + + or + + $ cat rsync-backup.log | jc --rsync + +Usage (module): + + import jc + result = jc.parse('rsync', rsync_command_output) + + or + + import jc.parsers.rsync + result = jc.parsers.rsync.parse(rsync_command_output) + +Schema: + + [ + { + "summary": { + "date": string, + "time": string, + "process": integer, + "sent": integer, + "received": integer, + "total_size": integer, + "matches": integer, + "hash_hits": integer, + "false_alarms": integer, + "data": integer, + "bytes_sec": float, + "speedup": float + }, + "files": [ + { + "filename": string, + "date": string, + "time": string, + "process": integer, + "metadata": string, + "update_type": string/null, [0] + "file_type": string/null, [1] + "checksum_or_value_different": bool/null, + "size_different": bool/null, + "modification_time_different": bool/null, + "permissions_different": bool/null, + "owner_different": bool/null, + "group_different": bool/null, + "acl_different": bool/null, + "extended_attribute_different": bool/null, + "epoch": integer, [2] + } + ] + } + ] + + [0] 'file sent', 'file received', 'local change or creation', + 'hard link', 'not updated', 'message' + [1] 'file', 'directory', 'symlink', 'device', 'special file' + [2] naive timestamp if time and date fields exist and can be converted. + +Examples: + + $ rsync -i -a source/ dest | jc --rsync -p + [ + { + "summary": { + "sent": 1708, + "received": 8209, + "bytes_sec": 19834.0, + "total_size": 235, + "speedup": 0.02 + }, + "files": [ + { + "filename": "./", + "metadata": ".d..t......", + "update_type": "not updated", + "file_type": "directory", + "checksum_or_value_different": false, + "size_different": false, + "modification_time_different": true, + "permissions_different": false, + "owner_different": false, + "group_different": false, + "acl_different": false, + "extended_attribute_different": false + }, + ... + ] + } + ] + + $ rsync | jc --rsync -p -r + [ + { + "summary": { + "sent": "1,708", + "received": "8,209", + "bytes_sec": "19,834.00", + "total_size": "235", + "speedup": "0.02" + }, + "files": [ + { + "filename": "./", + "metadata": ".d..t......", + "update_type": "not updated", + "file_type": "directory", + "checksum_or_value_different": false, + "size_different": false, + "modification_time_different": true, + "permissions_different": false, + "owner_different": false, + "group_different": false, + "acl_different": false, + "extended_attribute_different": false + }, + ... + ] + } + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/rsync_s.md b/docs/parsers/rsync_s.md new file mode 100644 index 00000000..27b2c9a7 --- /dev/null +++ b/docs/parsers/rsync_s.md @@ -0,0 +1,127 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.rsync\_s + +jc - JSON CLI output utility `rsync` command output streaming parser + +> This streaming parser outputs JSON Lines + +Supports the `-i` or `--itemize-changes` options with all levels of +verbosity. This parser will process the STDOUT output or a log file +generated with the `--log-file` option. + +Usage (cli): + + $ rsync -i -a source/ dest | jc --rsync-s + + or + + $ cat rsync-backup.log | jc --rsync-s + +Usage (module): + + import jc + # result is an iterable object (generator) + result = jc.parse('rsync_s', rsync_command_output.splitlines()) + for item in result: + # do something + + or + + import jc.parsers.rsync_s + # result is an iterable object (generator) + result = jc.parsers.rsync_s.parse(rsync_command_output.splitlines()) + for item in result: + # do something + +Schema: + + { + "type": string, # 'file' or 'summary' + "date": string, + "time": string, + "process": integer, + "sent": integer, + "received": integer, + "total_size": integer, + "matches": integer, + "hash_hits": integer, + "false_alarms": integer, + "data": integer, + "bytes_sec": float, + "speedup": float, + "filename": string, + "date": string, + "time": string, + "process": integer, + "metadata": string, + "update_type": string/null, [0] + "file_type": string/null, [1] + "checksum_or_value_different": bool/null, + "size_different": bool/null, + "modification_time_different": bool/null, + "permissions_different": bool/null, + "owner_different": bool/null, + "group_different": bool/null, + "acl_different": bool/null, + "extended_attribute_different": bool/null, + "epoch": integer, [2] + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": + { + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + + [0] 'file sent', 'file received', 'local change or creation', + 'hard link', 'not updated', 'message' + [1] 'file', 'directory', 'symlink', 'device', 'special file' + [2] naive timestamp if time and date fields exist and can be converted. + +Examples: + + $ rsync -i -a source/ dest | jc --rsync-s + {"type":"file","filename":"./","metadata":".d..t......","update_...} + ... + + $ cat rsync_backup.log | jc --rsync-s + {"type":"file","filename":"./","date":"2022/01/28","time":"03:53...} + ... + + + +### parse + +```python +@add_jc_meta +def parse(data: Iterable[str], raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple] +``` + +Main text parsing generator function. Returns an iterator object. + +Parameters: + + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + ignore_exceptions: (boolean) ignore parsing exceptions if True + +Yields: + + Dictionary. Raw or processed structured data. + +Returns: + + Iterator object (generator) + +### Parser Information +Compatibility: linux, darwin, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/stat.md b/docs/parsers/stat.md index 72566c41..b5db81dd 100644 --- a/docs/parsers/stat.md +++ b/docs/parsers/stat.md @@ -198,4 +198,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, freebsd -Version 1.10 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.11 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/stat_s.md b/docs/parsers/stat_s.md index 4f560f7c..4a59f84d 100644 --- a/docs/parsers/stat_s.md +++ b/docs/parsers/stat_s.md @@ -91,6 +91,7 @@ Examples: ### parse ```python +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False) ``` @@ -111,9 +112,9 @@ Yields: Returns: - Iterator object + Iterator object (generator) ### Parser Information Compatibility: linux, darwin, freebsd -Version 0.5 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/systeminfo.md b/docs/parsers/systeminfo.md index fa77289c..bd85b45f 100644 --- a/docs/parsers/systeminfo.md +++ b/docs/parsers/systeminfo.md @@ -239,4 +239,4 @@ Returns: ### Parser Information Compatibility: win32 -Version 1.1 by Jon Smith (jon@rebelliondefense.com) +Version 1.2 by Jon Smith (jon@rebelliondefense.com) diff --git a/docs/parsers/timedatectl.md b/docs/parsers/timedatectl.md index 8c4b1539..0e40adbe 100644 --- a/docs/parsers/timedatectl.md +++ b/docs/parsers/timedatectl.md @@ -92,4 +92,4 @@ Returns: ### Parser Information Compatibility: linux -Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/universal.md b/docs/parsers/universal.md index f360f875..00dca1d3 100644 --- a/docs/parsers/universal.md +++ b/docs/parsers/universal.md @@ -40,7 +40,7 @@ Returns: ### sparse\_table\_parse ```python -def sparse_table_parse(data: List[str], delim: Optional[str] = '\u2063') -> List[Dict] +def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict] ``` Parse tables with missing column data or with spaces in column data. diff --git a/docs/parsers/upower.md b/docs/parsers/upower.md index ffcbb334..be0384ad 100644 --- a/docs/parsers/upower.md +++ b/docs/parsers/upower.md @@ -226,4 +226,4 @@ Returns: ### Parser Information Compatibility: linux -Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.4 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/vmstat.md b/docs/parsers/vmstat.md index 2291fb79..c6e76a38 100644 --- a/docs/parsers/vmstat.md +++ b/docs/parsers/vmstat.md @@ -154,4 +154,4 @@ Returns: ### Parser Information Compatibility: linux -Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/vmstat_s.md b/docs/parsers/vmstat_s.md index 6f85dce7..2445f1b6 100644 --- a/docs/parsers/vmstat_s.md +++ b/docs/parsers/vmstat_s.md @@ -110,6 +110,7 @@ Examples: ### parse ```python +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False) ``` @@ -130,9 +131,9 @@ Yields: Returns: - Iterator object + Iterator object (generator) ### Parser Information Compatibility: linux -Version 0.6 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/who.md b/docs/parsers/who.md index b1d5996f..94953475 100644 --- a/docs/parsers/who.md +++ b/docs/parsers/who.md @@ -163,4 +163,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, cygwin, aix, freebsd -Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/xrandr.md b/docs/parsers/xrandr.md new file mode 100644 index 00000000..dcc7fea2 --- /dev/null +++ b/docs/parsers/xrandr.md @@ -0,0 +1,166 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.xrandr + +jc - JSON CLI output utility `xrandr` command output parser + +Usage (cli): + + $ xrandr | jc --xrandr + + or + + $ jc xrandr + +Usage (module): + + import jc + result = jc.parse('xrandr', xrandr_command_output) + + or + + import jc.parsers.xrandr + result = jc.parsers.xrandr.parse(xrandr_command_output) + +Schema: + + { + "screens": [ + { + "screen_number": integer, + "minimum_width": integer, + "minimum_height": integer, + "current_width": integer, + "current_height": integer, + "maximum_width": integer, + "maximum_height": integer, + "associated_device": { + "associated_modes": [ + { + "resolution_width": integer, + "resolution_height": integer, + "is_high_resolution": boolean, + "frequencies": [ + { + "frequency": float, + "is_current": boolean, + "is_preferred": boolean + } + ], + "is_connected": boolean, + "is_primary": boolean, + "device_name": string, + "resolution_width": integer, + "resolution_height": integer, + "offset_width": integer, + "offset_height": integer, + "dimension_width": integer, + "dimension_height": integer + } + } + ], + "unassociated_devices": [ + { + "associated_modes": [ + { + "resolution_width": integer, + "resolution_height": integer, + "is_high_resolution": boolean, + "frequencies": [ + { + "frequency": float, + "is_current": boolean, + "is_preferred": boolean + } + ] + } + ] + } + ] + } + +Examples: + + $ xrandr | jc --xrandr -p + { + "screens": [ + { + "screen_number": 0, + "minimum_width": 8, + "minimum_height": 8, + "current_width": 1920, + "current_height": 1080, + "maximum_width": 32767, + "maximum_height": 32767, + "associated_device": { + "associated_modes": [ + { + "resolution_width": 1920, + "resolution_height": 1080, + "is_high_resolution": false, + "frequencies": [ + { + "frequency": 60.03, + "is_current": true, + "is_preferred": true + }, + { + "frequency": 59.93, + "is_current": false, + "is_preferred": false + } + ] + }, + { + "resolution_width": 1680, + "resolution_height": 1050, + "is_high_resolution": false, + "frequencies": [ + { + "frequency": 59.88, + "is_current": false, + "is_preferred": false + } + ] + } + ], + "is_connected": true, + "is_primary": true, + "device_name": "eDP1", + "resolution_width": 1920, + "resolution_height": 1080, + "offset_width": 0, + "offset_height": 0, + "dimension_width": 310, + "dimension_height": 170 + } + } + ], + "unassociated_devices": [] + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux, darwin, cygwin, aix, freebsd + +Version 1.0 by Kevin Lyter (lyter_git at sent.com) diff --git a/docs/parsers/zipinfo.md b/docs/parsers/zipinfo.md index 49a4712a..665a1a31 100644 --- a/docs/parsers/zipinfo.md +++ b/docs/parsers/zipinfo.md @@ -44,7 +44,7 @@ Schema: { "flags": string, "zipversion": string, - "zipunder": string + "zipunder": string, "filesize": integer, "type": string, "method": string, diff --git a/docs/readme.md b/docs/readme.md index 073af8bc..7410dd4f 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -14,6 +14,8 @@ and file-types to dictionaries and lists of dictionaries. >>> help('jc') >>> help('jc.lib') >>> help('jc.utils') + >>> help('jc.streaming') + >>> help('jc.parsers.universal') >>> jc.get_help('parser_module_name') ## Online Documentation diff --git a/docs/streaming.md b/docs/streaming.md new file mode 100644 index 00000000..b8971010 --- /dev/null +++ b/docs/streaming.md @@ -0,0 +1,114 @@ +# Table of Contents + +* [jc.streaming](#jc.streaming) + * [streaming\_input\_type\_check](#jc.streaming.streaming_input_type_check) + * [streaming\_line\_input\_type\_check](#jc.streaming.streaming_line_input_type_check) + * [stream\_success](#jc.streaming.stream_success) + * [stream\_error](#jc.streaming.stream_error) + * [add\_jc\_meta](#jc.streaming.add_jc_meta) + * [raise\_or\_yield](#jc.streaming.raise_or_yield) + + + +# jc.streaming + +jc - JSON CLI output utility streaming utils + + + +### streaming\_input\_type\_check + +```python +def streaming_input_type_check(data: Iterable) -> None +``` + +Ensure input data is an iterable, but not a string or bytes. Raises +`TypeError` if not. + + + +### streaming\_line\_input\_type\_check + +```python +def streaming_line_input_type_check(line: str) -> None +``` + +Ensure each line is a string. Raises `TypeError` if not. + + + +### stream\_success + +```python +def stream_success(output_line: Dict, ignore_exceptions: bool) -> Dict +``` + +Add `_jc_meta` object to output line if `ignore_exceptions=True` + + + +### stream\_error + +```python +def stream_error(e: BaseException, line: str) -> Dict +``` + +Return an error `_jc_meta` field. + + + +### add\_jc\_meta + +```python +def add_jc_meta(func) +``` + +Decorator for streaming parsers to add stream_success and stream_error +objects. This simplifies the yield lines in the streaming parsers. + +With the decorator on parse(): + + # successfully parsed line: + yield output_line if raw else _process(output_line) + + # unsuccessfully parsed line: + except Exception as e: + yield raise_or_yield(ignore_exceptions, e, line) + +Without the decorator on parse(): + + # successfully parsed line: + if raw: + yield stream_success(output_line, ignore_exceptions) + else: + stream_success(_process(output_line), ignore_exceptions) + + # unsuccessfully parsed line: + except Exception as e: + yield stream_error(raise_or_yield(ignore_exceptions, e, line)) + +In all cases above: + + output_line: (Dict) successfully parsed line yielded as a dict + + e: (BaseException) exception object as the first value + of the tuple if the line was not successfully parsed. + + line: (str) string of the original line that did not + successfully parse. + + ignore_exceptions: (bool) continue processing lines and ignore + exceptions if True. + + + +### raise\_or\_yield + +```python +def raise_or_yield(ignore_exceptions: bool, e: BaseException, line: str) -> tuple +``` + +Return the exception object and line string if ignore_exceptions is +True. Otherwise, re-raise the exception from the exception object with +an annotation. + diff --git a/docs/utils.md b/docs/utils.md index 2b914cac..c8c24180 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -8,11 +8,7 @@ * [convert\_to\_int](#jc.utils.convert_to_int) * [convert\_to\_float](#jc.utils.convert_to_float) * [convert\_to\_bool](#jc.utils.convert_to_bool) - * [stream\_success](#jc.utils.stream_success) - * [stream\_error](#jc.utils.stream_error) * [input\_type\_check](#jc.utils.input_type_check) - * [streaming\_input\_type\_check](#jc.utils.streaming_input_type_check) - * [streaming\_line\_input\_type\_check](#jc.utils.streaming_line_input_type_check) * [timestamp](#jc.utils.timestamp) * [\_\_init\_\_](#jc.utils.timestamp.__init__) @@ -67,7 +63,7 @@ Returns: ### compatibility ```python -def compatibility(mod_name: str, compatible: List, quiet: Optional[bool] = False) -> None +def compatibility(mod_name: str, compatible: List, quiet: bool = False) -> None ``` Checks for the parser's compatibility with the running OS @@ -112,7 +108,7 @@ Returns: ### convert\_to\_int ```python -def convert_to_int(value: Union[str, float]) -> Union[int, None] +def convert_to_int(value: Union[str, float]) -> Optional[int] ``` Converts string and float input to int. Strips all non-numeric @@ -131,7 +127,7 @@ Returns: ### convert\_to\_float ```python -def convert_to_float(value: Union[str, int]) -> Union[float, None] +def convert_to_float(value: Union[str, int]) -> Optional[float] ``` Converts string and int input to float. Strips all non-numeric @@ -165,27 +161,6 @@ Returns: True/False False unless a 'truthy' number or string is found ('y', 'yes', 'true', '1', 1, -1, etc.) - - -### stream\_success - -```python -def stream_success(output_line: Dict, ignore_exceptions: bool) -> Dict -``` - -Add `_jc_meta` object to output line if `ignore_exceptions=True` - - - -### stream\_error - -```python -def stream_error(e: BaseException, ignore_exceptions: bool, line: str) -> Dict -``` - -Reraise the stream exception with annotation or print an error -`_jc_meta` field if `ignore_exceptions=True`. - ### input\_type\_check @@ -196,27 +171,6 @@ def input_type_check(data: str) -> None Ensure input data is a string. Raises `TypeError` if not. - - -### streaming\_input\_type\_check - -```python -def streaming_input_type_check(data: Iterable) -> None -``` - -Ensure input data is an iterable, but not a string or bytes. Raises -`TypeError` if not. - - - -### streaming\_line\_input\_type\_check - -```python -def streaming_line_input_type_check(line: str) -> None -``` - -Ensure each line is a string. Raises `TypeError` if not. - ### timestamp Objects @@ -230,29 +184,34 @@ class timestamp() ### \_\_init\_\_ ```python -def __init__(datetime_string: str) -> None +def __init__(datetime_string: str, format_hint: Union[List, Tuple, None] = None) -> None ``` -Input a date-time text string of several formats and convert to a +Input a datetime text string of several formats and convert to a naive or timezone-aware epoch timestamp in UTC. Parameters: - datetime_string: (str) a string representation of a - date-time in several supported formats + datetime_string (str): a string representation of a + datetime in several supported formats -Attributes: + format_hint (list | tuple): an optional list of format ID + integers to instruct the timestamp object to try those + formats first in the order given. Other formats will be + tried after the format hint list is exhausted. This can + speed up timestamp conversion so several different formats + don't have to be tried in brute-force fashion. - string (str) the input datetime string +Returns a timestamp object with the following attributes: - format (int) the format rule that was used to - decode the datetime string. None if - conversion fails + string (str): the input datetime string - naive (int) timestamp based on locally configured - timezone. None if conversion fails + format (int | None): the format rule that was used to decode + the datetime string. None if conversion fails. - utc (int) aware timestamp only if UTC timezone - detected in datetime string. None if - conversion fails + naive (int | None): timestamp based on locally configured + timezone. None if conversion fails. + + utc (int | None): aware timestamp only if UTC timezone + detected in datetime string. None if conversion fails. diff --git a/jc/__init__.py b/jc/__init__.py index d5a02709..a59ba5b2 100644 --- a/jc/__init__.py +++ b/jc/__init__.py @@ -10,6 +10,8 @@ and file-types to dictionaries and lists of dictionaries. >>> help('jc') >>> help('jc.lib') >>> help('jc.utils') + >>> help('jc.streaming') + >>> help('jc.parsers.universal') >>> jc.get_help('parser_module_name') ## Online Documentation diff --git a/jc/cli.py b/jc/cli.py index 20425df1..f1fd5660 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -89,13 +89,15 @@ def set_env_colors(env_colors=None): """ Return a dictionary to be used in Pygments custom style class. - Grab custom colors from JC_COLORS environment variable. JC_COLORS env variable takes 4 comma - separated string values and should be in the format of: + Grab custom colors from JC_COLORS environment variable. JC_COLORS env + variable takes 4 comma separated string values and should be in the + format of: JC_COLORS=,,, - Where colors are: black, red, green, yellow, blue, magenta, cyan, gray, brightblack, brightred, - brightgreen, brightyellow, brightblue, brightmagenta, brightcyan, white, default + Where colors are: black, red, green, yellow, blue, magenta, cyan, gray, + brightblack, brightred, brightgreen, brightyellow, + brightblue, brightmagenta, brightcyan, white, default Default colors: @@ -132,8 +134,10 @@ def set_env_colors(env_colors=None): def piped_output(force_color): - """Return False if stdout is a TTY. True if output is being piped to another program - and foce_color is True. This allows forcing of ANSI color codes even when using pipes. + """ + Return False if stdout is a TTY. True if output is being piped to + another program and foce_color is True. This allows forcing of ANSI + color codes even when using pipes. """ return not sys.stdout.isatty() and not force_color @@ -224,8 +228,8 @@ def helptext(): def help_doc(options): """ - Returns the parser documentation if a parser is found in the arguments, otherwise - the general help text is returned. + Returns the parser documentation if a parser is found in the arguments, + otherwise the general help text is returned. """ for arg in options: parser_name = parser_shortname(arg) @@ -253,7 +257,10 @@ def versiontext(): def json_out(data, pretty=False, env_colors=None, mono=False, piped_out=False): - """Return a JSON formatted string. String may include color codes or be pretty printed.""" + """ + Return a JSON formatted string. String may include color codes or be + pretty printed. + """ separators = (',', ':') indent = None @@ -277,10 +284,10 @@ def magic_parser(args): Parse command arguments for magic syntax: jc -p ls -al Return a tuple: - valid_command (bool) is this a valid command? (exists in magic dict) - run_command (list) list of the user's command to run. None if no command. - jc_parser (str) parser to use for this user's command. - jc_options (list) list of jc options + valid_command (bool) is this a valid cmd? (exists in magic dict) + run_command (list) list of the user's cmd to run. None if no cmd. + jc_parser (str) parser to use for this user's cmd. + jc_options (list) list of jc options """ # bail immediately if there are no args or a parser is defined if len(args) <= 1 or args[1].startswith('--'): @@ -335,12 +342,15 @@ def magic_parser(args): def run_user_command(command): - """Use subprocess to run the user's command. Returns the STDOUT, STDERR, and the Exit Code as a tuple.""" + """ + Use subprocess to run the user's command. Returns the STDOUT, STDERR, + and the Exit Code as a tuple. + """ proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - close_fds=False, # Allows inheriting file descriptors. Useful for process substitution - universal_newlines=True) + close_fds=False, # Allows inheriting file descriptors; + universal_newlines=True) # useful for process substitution stdout, stderr = proc.communicate() return ( @@ -441,14 +451,18 @@ def main(): raise error_msg = os.strerror(e.errno) - utils.error_message([f'"{run_command_str}" command could not be run: {error_msg}. For details use the -d or -dd option.']) + utils.error_message([ + f'"{run_command_str}" command could not be run: {error_msg}. For details use the -d or -dd option.' + ]) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) except Exception: if debug: raise - utils.error_message([f'"{run_command_str}" command could not be run. For details use the -d or -dd option.']) + utils.error_message([ + f'"{run_command_str}" command could not be run. For details use the -d or -dd option.' + ]) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) elif run_command is not None: @@ -489,7 +503,10 @@ def main(): # streaming if getattr(parser.info, 'streaming', None): - result = parser.parse(sys.stdin, raw=raw, quiet=quiet, ignore_exceptions=ignore_exceptions) + result = parser.parse(sys.stdin, + raw=raw, + quiet=quiet, + ignore_exceptions=ignore_exceptions) for line in result: print(json_out(line, pretty=pretty, @@ -503,7 +520,9 @@ def main(): # regular else: data = magic_stdout or sys.stdin.read() - result = parser.parse(data, raw=raw, quiet=quiet) + result = parser.parse(data, + raw=raw, + quiet=quiet) print(json_out(result, pretty=pretty, env_colors=jc_colors, @@ -517,10 +536,11 @@ def main(): if debug: raise - utils.error_message([f'Parser issue with {parser_name}:', - f'{e.__class__.__name__}: {e}', - 'If this is the correct parser, try setting the locale to C (LANG=C).', - 'For details use the -d or -dd option. Use "jc -h" for help.']) + utils.error_message([ + f'Parser issue with {parser_name}:', f'{e.__class__.__name__}: {e}', + 'If this is the correct parser, try setting the locale to C (LANG=C).', + 'For details use the -d or -dd option. Use "jc -h" for help.' + ]) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) except json.JSONDecodeError: diff --git a/jc/lib.py b/jc/lib.py index ab6b6b38..bbf3a43b 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -6,10 +6,10 @@ import sys import os import re import importlib -from typing import Dict, List, Iterable, Union, Iterator, Optional +from typing import Dict, List, Iterable, Union, Iterator from jc import appdirs -__version__ = '1.18.2' +__version__ = '1.18.3' parsers = [ 'acpi', @@ -69,6 +69,8 @@ parsers = [ 'ps', 'route', 'rpm-qi', + 'rsync', + 'rsync-s', 'sfdisk', 'shadow', 'ss', @@ -95,6 +97,7 @@ parsers = [ 'wc', 'who', 'xml', + 'xrandr', 'yaml', 'zipinfo' ] @@ -224,7 +227,7 @@ def plugin_parser_mod_list() -> List[str]: """ return [_cliname_to_modname(p) for p in local_parsers] -def parser_info(parser_mod_name: str) -> Union[Dict, None]: +def parser_info(parser_mod_name: str) -> Dict: """ Returns a dictionary that includes the module metadata. @@ -235,9 +238,9 @@ def parser_info(parser_mod_name: str) -> Union[Dict, None]: parser_mod_name = _cliname_to_modname(parser_mod_name) parser_mod = _get_parser(parser_mod_name) + info_dict: Dict = {} if hasattr(parser_mod, 'info'): - info_dict: Dict = {} info_dict['name'] = parser_mod_name info_dict['argument'] = _parser_argument(parser_mod_name) parser_entry = vars(parser_mod.info) @@ -249,15 +252,13 @@ def parser_info(parser_mod_name: str) -> Union[Dict, None]: if _modname_to_cliname(parser_mod_name) in local_parsers: info_dict['plugin'] = True - return info_dict + return info_dict - return None - -def all_parser_info() -> List[Optional[Dict]]: +def all_parser_info() -> List[Dict]: """ - Returns a list of dictionaris that includes metadata for all modules. + Returns a list of dictionaries that includes metadata for all modules. """ - return [parser_info(_cliname_to_modname(p)) for p in parsers] + return [parser_info(p) for p in parsers] def get_help(parser_mod_name: str) -> None: """ diff --git a/jc/parsers/csv_s.py b/jc/parsers/csv_s.py index c692dde8..e4c573c4 100644 --- a/jc/parsers/csv_s.py +++ b/jc/parsers/csv_s.py @@ -66,13 +66,13 @@ Examples: import itertools import csv import jc.utils -from jc.utils import stream_success, stream_error +from jc.streaming import streaming_input_type_check, add_jc_meta, raise_or_yield from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.2' + version = '1.3' description = 'CSV file streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -101,6 +101,7 @@ def _process(proc_data): return proc_data +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False): """ Main text parsing generator function. Returns an iterator object. @@ -120,10 +121,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Returns: - Iterator object + Iterator object (generator) """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) # convert data to an iterable in case a sequence like a list is used as input. # this allows the exhaustion of the input so we don't double-process later. @@ -155,6 +156,6 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): for row in reader: try: - yield stream_success(row, ignore_exceptions) if raw else stream_success(_process(row), ignore_exceptions) + yield row if raw else _process(row) except Exception as e: - yield stream_error(e, ignore_exceptions, row) + yield raise_or_yield(ignore_exceptions, e, str(row)) diff --git a/jc/parsers/date.py b/jc/parsers/date.py index d09eed5d..df710f69 100644 --- a/jc/parsers/date.py +++ b/jc/parsers/date.py @@ -83,7 +83,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '2.2' + version = '2.3' description = '`date` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -168,7 +168,7 @@ def parse(data, raw=False, quiet=False): dt = None dt_utc = None - timestamp = jc.utils.timestamp(data) + timestamp = jc.utils.timestamp(data, format_hint=(1000, 6000, 7000)) if timestamp.naive: dt = datetime.fromtimestamp(timestamp.naive) if timestamp.utc: diff --git a/jc/parsers/dig.py b/jc/parsers/dig.py index b7d32d50..0af225b1 100644 --- a/jc/parsers/dig.py +++ b/jc/parsers/dig.py @@ -327,7 +327,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '2.2' + version = '2.3' description = '`dig` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -384,7 +384,7 @@ def _process(proc_data): auth['ttl'] = jc.utils.convert_to_int(auth['ttl']) if 'when' in entry: - ts = jc.utils.timestamp(entry['when']) + ts = jc.utils.timestamp(entry['when'], format_hint=(1000, 7000)) entry['when_epoch'] = ts.naive entry['when_epoch_utc'] = ts.utc diff --git a/jc/parsers/dir.py b/jc/parsers/dir.py index 5c65750d..f9f168b1 100644 --- a/jc/parsers/dir.py +++ b/jc/parsers/dir.py @@ -126,7 +126,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.4' + version = '1.5' description = '`dir` command parser' author = 'Rasheed Elsaleh' author_email = 'rasheed@rebelliondefense.com' @@ -152,7 +152,7 @@ def _process(proc_data): # add timestamps if 'date' in entry and 'time' in entry: dt = entry['date'] + ' ' + entry['time'] - timestamp = jc.utils.timestamp(dt) + timestamp = jc.utils.timestamp(dt, format_hint=(1600,)) entry['epoch'] = timestamp.naive # add ints diff --git a/jc/parsers/foo.py b/jc/parsers/foo.py index bd077f3f..afd56deb 100644 --- a/jc/parsers/foo.py +++ b/jc/parsers/foo.py @@ -38,8 +38,8 @@ Examples: $ foo | jc --foo -p -r [] """ -import jc.utils from typing import List, Dict +import jc.utils class info(): diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index 44e0ee31..eadfc202 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -49,9 +49,11 @@ Examples: {example output} ... """ -from typing import Dict, Iterable +from typing import Dict, Iterable, Union import jc.utils -from jc.utils import stream_success, stream_error +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) from jc.exceptions import ParseError @@ -63,7 +65,7 @@ class info(): author_email = 'johndoe@gmail.com' # compatible options: linux, darwin, cygwin, win32, aix, freebsd - compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] streaming = True @@ -91,12 +93,13 @@ def _process(proc_data: Dict) -> Dict: return proc_data +@add_jc_meta def parse( data: Iterable[str], raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Iterable[Dict]: +) -> Union[Iterable[Dict], tuple]: """ Main text parsing generator function. Returns an iterator object. @@ -115,24 +118,24 @@ def parse( Returns: - Iterator object + Iterator object (generator) """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) for line in data: - output_line: Dict = {} try: - jc.utils.streaming_line_input_type_check(line) + streaming_line_input_type_check(line) + output_line: Dict = {} # parse the content here # check out helper functions in jc.utils # and jc.parsers.universal if output_line: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) else: raise ParseError('Not foo data') except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/jc/parsers/iostat_s.py b/jc/parsers/iostat_s.py index d6ac5b4b..d98f58bc 100644 --- a/jc/parsers/iostat_s.py +++ b/jc/parsers/iostat_s.py @@ -101,14 +101,16 @@ Examples: ... """ import jc.utils -from jc.utils import stream_success, stream_error +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) from jc.exceptions import ParseError import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`iostat` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -159,6 +161,8 @@ def _create_obj_list(section_list, section_name): item['type'] = section_name return output_list + +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False): """ Main text parsing generator function. Returns an iterator object. @@ -178,10 +182,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Returns: - Iterator object + Iterator object (generator) """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) section = '' # either 'cpu' or 'device' headers = '' @@ -189,9 +193,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): device_list = [] for line in data: - output_line = {} try: - jc.utils.streaming_line_input_type_check(line) + streaming_line_input_type_check(line) + output_line = {} # ignore blank lines and header line if line == '\n' or line == '' or line.startswith('Linux'): @@ -223,9 +227,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): device_list = [] if output_line: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) else: raise ParseError('Not iostat data') except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index e497c194..fba2c2a4 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -122,7 +122,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.10' + version = '1.11' description = '`ls` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -154,7 +154,7 @@ def _process(proc_data): if 'date' in entry: # to speed up processing only try to convert the date if it's not the default format if not re.match(r'[a-zA-Z]{3}\s{1,2}\d{1,2}\s{1,2}[0-9:]{4,5}', entry['date']): - ts = jc.utils.timestamp(entry['date']) + ts = jc.utils.timestamp(entry['date'], format_hint=(7200,)) entry['epoch'] = ts.naive entry['epoch_utc'] = ts.utc diff --git a/jc/parsers/ls_s.py b/jc/parsers/ls_s.py index 7de75ce9..0ca3261c 100644 --- a/jc/parsers/ls_s.py +++ b/jc/parsers/ls_s.py @@ -79,13 +79,15 @@ Examples: """ import re import jc.utils -from jc.utils import stream_success, stream_error +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '0.6' + version = '1.0' description = '`ls` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -116,13 +118,14 @@ def _process(proc_data): if 'date' in proc_data: # to speed up processing only try to convert the date if it's not the default format if not re.match(r'[a-zA-Z]{3}\s{1,2}\d{1,2}\s{1,2}[0-9:]{4,5}', proc_data['date']): - ts = jc.utils.timestamp(proc_data['date']) + ts = jc.utils.timestamp(proc_data['date'], format_hint=(7200,)) proc_data['epoch'] = ts.naive proc_data['epoch_utc'] = ts.utc return proc_data +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False): """ Main text parsing generator function. Returns an iterator object. @@ -142,16 +145,16 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Returns: - Iterator object + Iterator object (generator) """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) parent = '' for line in data: try: - jc.utils.streaming_line_input_type_check(line) + streaming_line_input_type_check(line) # skip line if it starts with 'total 1234' if re.match(r'total [0-9]+', line): @@ -163,7 +166,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): # Look for parent line if glob or -R is used if not re.match(r'[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', line) \ - and line.strip().endswith(':'): + and line.strip().endswith(':'): parent = line.strip()[:-1] continue @@ -196,7 +199,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): output_line['size'] = parsed_line[4] output_line['date'] = ' '.join(parsed_line[5:8]) - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/jc/parsers/ping_s.py b/jc/parsers/ping_s.py index 8536c3f5..ab80d11b 100644 --- a/jc/parsers/ping_s.py +++ b/jc/parsers/ping_s.py @@ -86,13 +86,15 @@ Examples: import string import ipaddress import jc.utils +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) from jc.exceptions import ParseError -from jc.utils import stream_success, stream_error class info(): """Provides parser metadata (version, author, etc.)""" - version = '0.6' + version = '1.0' description = '`ping` and `ping6` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -469,6 +471,7 @@ def _linux_parse(line, s): return output_line +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False): """ Main text parsing generator function. Returns an iterator object. @@ -488,17 +491,17 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Returns: - Iterator object + Iterator object (generator) """ + jc.utils.compatibility(__name__, info.compatible, quiet) + streaming_input_type_check(data) + s = _state() - jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) - for line in data: - output_line = {} try: - jc.utils.streaming_line_input_type_check(line) + streaming_line_input_type_check(line) + output_line = {} # skip blank lines if line.strip() == '': @@ -542,9 +545,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): # yield the output line if it has data if output_line: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) else: continue except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/jc/parsers/rpm_qi.py b/jc/parsers/rpm_qi.py index a0e6e689..2a6a81dd 100644 --- a/jc/parsers/rpm_qi.py +++ b/jc/parsers/rpm_qi.py @@ -166,7 +166,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.4' + version = '1.5' description = '`rpm -qi` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -197,12 +197,12 @@ def _process(proc_data): entry[key] = jc.utils.convert_to_int(entry[key]) if 'build_date' in entry: - timestamp = jc.utils.timestamp(entry['build_date']) + timestamp = jc.utils.timestamp(entry['build_date'], format_hint=(3000,)) entry['build_epoch'] = timestamp.naive entry['build_epoch_utc'] = timestamp.utc if 'install_date' in entry: - timestamp = jc.utils.timestamp(entry['install_date']) + timestamp = jc.utils.timestamp(entry['install_date'], format_hint=(3000,)) entry['install_date_epoch'] = timestamp.naive entry['install_date_epoch_utc'] = timestamp.utc diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py new file mode 100644 index 00000000..8b56761a --- /dev/null +++ b/jc/parsers/rsync.py @@ -0,0 +1,493 @@ +"""jc - JSON CLI output utility `rsync` command output parser + +Supports the `-i` or `--itemize-changes` options with all levels of +verbosity. This parser will process the STDOUT output or a log file +generated with the `--log-file` option. + +Usage (cli): + + $ rsync -i -a source/ dest | jc --rsync + + or + + $ jc rsync -i -a source/ dest + + or + + $ cat rsync-backup.log | jc --rsync + +Usage (module): + + import jc + result = jc.parse('rsync', rsync_command_output) + + or + + import jc.parsers.rsync + result = jc.parsers.rsync.parse(rsync_command_output) + +Schema: + + [ + { + "summary": { + "date": string, + "time": string, + "process": integer, + "sent": integer, + "received": integer, + "total_size": integer, + "matches": integer, + "hash_hits": integer, + "false_alarms": integer, + "data": integer, + "bytes_sec": float, + "speedup": float + }, + "files": [ + { + "filename": string, + "date": string, + "time": string, + "process": integer, + "metadata": string, + "update_type": string/null, [0] + "file_type": string/null, [1] + "checksum_or_value_different": bool/null, + "size_different": bool/null, + "modification_time_different": bool/null, + "permissions_different": bool/null, + "owner_different": bool/null, + "group_different": bool/null, + "acl_different": bool/null, + "extended_attribute_different": bool/null, + "epoch": integer, [2] + } + ] + } + ] + + [0] 'file sent', 'file received', 'local change or creation', + 'hard link', 'not updated', 'message' + [1] 'file', 'directory', 'symlink', 'device', 'special file' + [2] naive timestamp if time and date fields exist and can be converted. + +Examples: + + $ rsync -i -a source/ dest | jc --rsync -p + [ + { + "summary": { + "sent": 1708, + "received": 8209, + "bytes_sec": 19834.0, + "total_size": 235, + "speedup": 0.02 + }, + "files": [ + { + "filename": "./", + "metadata": ".d..t......", + "update_type": "not updated", + "file_type": "directory", + "checksum_or_value_different": false, + "size_different": false, + "modification_time_different": true, + "permissions_different": false, + "owner_different": false, + "group_different": false, + "acl_different": false, + "extended_attribute_different": false + }, + ... + ] + } + ] + + $ rsync | jc --rsync -p -r + [ + { + "summary": { + "sent": "1,708", + "received": "8,209", + "bytes_sec": "19,834.00", + "total_size": "235", + "speedup": "0.02" + }, + "files": [ + { + "filename": "./", + "metadata": ".d..t......", + "update_type": "not updated", + "file_type": "directory", + "checksum_or_value_different": false, + "size_different": false, + "modification_time_different": true, + "permissions_different": false, + "owner_different": false, + "group_different": false, + "acl_different": false, + "extended_attribute_different": false + }, + ... + ] + } + ] +""" +import re +from copy import deepcopy +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`rsync` command parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'freebsd'] + magic_commands = ['rsync -i', 'rsync --itemize-changes'] + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = [ + 'process', 'sent', 'received', 'total_size', 'matches', 'hash_hits', + 'false_alarms', 'data' + ] + float_list = ['bytes_sec', 'speedup'] + + for item in proc_data: + for key in item['summary']: + if key in int_list: + item['summary'][key] = jc.utils.convert_to_int(item['summary'][key]) + if key in float_list: + item['summary'][key] = jc.utils.convert_to_float(item['summary'][key]) + + for entry in item['files']: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + + # add timestamp + if 'date' in entry and 'time' in entry: + date = entry['date'].replace('/', '-') + date_time = f'{date} {entry["time"]}' + ts = jc.utils.timestamp(date_time, format_hint=(7250,)) + entry['epoch'] = ts.naive + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + rsync_run_new: Dict = { + 'summary': {}, + 'files': [] + } + + rsync_run = deepcopy(rsync_run_new) + + last_process = '' + + update_type = { + '<': 'file sent', + '>': 'file received', + 'c': 'local change or creation', + 'h': 'hard link', + '.': 'not updated', + '*': 'message', + '+': None + } + + file_type = { + 'f': 'file', + 'd': 'directory', + 'L': 'symlink', + 'D': 'device', + 'S': 'special file', + '+': None + } + + checksum_or_value_different = { + 'c': True, + '.': False, + '+': None, + ' ': None, + '?': None + } + + size_different = { + 's': True, + '.': False, + '+': None, + ' ': None, + '?': None + } + + modification_time_different = { + 't': True, + '.': False, + '+': None, + ' ': None, + '?': None + } + + permissions_different = { + 'p': True, + '.': False, + '+': None, + ' ': None, + '?': None + } + + owner_different = { + 'o': True, + '.': False, + '+': None, + ' ': None, + '?': None + } + + group_different = { + 'g': True, + '.': False, + '+': None, + ' ': None, + '?': None + } + + acl_different = { + 'a': True, + '.': False, + '+': None, + ' ': None, + '?': None + } + + extended_attribute_different = { + 'x': True, + '.': False, + '+': None, + ' ': None, + '?': None + } + + file_line_re = re.compile(r'(?P[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][u.+ ?][a.+ ?][x.+ ?]) (?P.+)') + file_line_mac_re = re.compile(r'(?P[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][x.+ ?]) (?P.+)') + stat1_line_re = re.compile(r'(sent)\s+(?P[0-9,]+)\s+(bytes)\s+(received)\s+(?P[0-9,]+)\s+(bytes)\s+(?P[0-9,.]+)\s+(bytes/sec)') + stat2_line_re = re.compile(r'(total size is)\s+(?P[0-9,]+)\s+(speedup is)\s+(?P[0-9,.]+)') + + file_line_log_re = re.compile(r'(?P\d\d\d\d/\d\d/\d\d)\s+(?P