diff --git a/CHANGELOG b/CHANGELOG
index 32520170..89997433 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,5 +1,13 @@
jc changelog
+20220214 v1.18.3
+- Add rsync command and log file parser tested on linux and macOS
+- Add rsync command and log file streaming parser tested on linux and macOS
+- Add xrandr command parser tested on linux
+- Enhance timestamp performance with caching and format hints
+- Refactor ignore_exceptions functionality in streaming parsers
+- Fix man page in packages
+
20220127 v1.18.2
- Fix for plugin parsers with underscores in the name
- Add type hints to public API functions
diff --git a/EXAMPLES.md b/EXAMPLES.md
index e058b995..3bd6cfee 100644
--- a/EXAMPLES.md
+++ b/EXAMPLES.md
@@ -2743,6 +2743,39 @@ rpm_qia | jc --rpm_qi -p # or: jc -p rpm -qia
}
]
```
+### rsync
+```bash
+rsync -i -a source/ dest | jc --rsync -p # or jc -p rsync -i -a source/ dest
+```
+```json
+[
+ {
+ "summary": {
+ "sent": 1708,
+ "received": 8209,
+ "bytes_sec": 19834.0,
+ "total_size": 235,
+ "speedup": 0.02
+ },
+ "files": [
+ {
+ "filename": "./",
+ "metadata": ".d..t......",
+ "update_type": "not updated",
+ "file_type": "directory",
+ "checksum_or_value_different": false,
+ "size_different": false,
+ "modification_time_different": true,
+ "permissions_different": false,
+ "owner_different": false,
+ "group_different": false,
+ "acl_different": false,
+ "extended_attribute_different": false
+ }
+ ]
+ }
+]
+```
### sfdisk
```bash
sfdisk -l | jc --sfdisk -p # or jc -p sfdisk -l
diff --git a/README.md b/README.md
index 7f48230d..f9743e0d 100644
--- a/README.md
+++ b/README.md
@@ -128,7 +128,7 @@ pip3 install jc
> For more OS Packages, see https://repology.org/project/jc/versions.
-### Binaries and Packages
+### Binaries
For precompiled binaries, see [Releases](https://github.com/kellyjonbrazil/jc/releases)
on Github.
@@ -208,6 +208,8 @@ option.
- `--ps` enables the `ps` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ps))
- `--route` enables the `route` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/route))
- `--rpm-qi` enables the `rpm -qi` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/rpm_qi))
+- `--rsync` enables the `rsync` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/rsync))
+- `--rsync-s` enables the `rsync` command streaming parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/rsync_s))
- `--sfdisk` enables the `sfdisk` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/sfdisk))
- `--shadow` enables the `/etc/shadow` file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/shadow))
- `--ss` enables the `ss` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ss))
@@ -234,6 +236,7 @@ option.
- `--wc` enables the `wc` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/wc))
- `--who` enables the `who` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/who))
- `--xml` enables the XML file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/xml))
+- `--xrandr` enables the `xrandr` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/xrandr))
- `--yaml` enables the YAML file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/yaml))
- `--zipinfo` enables the `zipinfo` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/zipinfo))
diff --git a/docgen.sh b/docgen.sh
index 13cd6fec..38495775 100755
--- a/docgen.sh
+++ b/docgen.sh
@@ -85,6 +85,9 @@ pydoc-markdown -m jc.lib "${toc_config}" > ../docs/lib.md
echo Building docs for: utils
pydoc-markdown -m jc.utils "${toc_config}" > ../docs/utils.md
+echo Building docs for: streaming
+pydoc-markdown -m jc.streaming "${toc_config}" > ../docs/streaming.md
+
echo Building docs for: universal parser
pydoc-markdown -m jc.parsers.universal "${toc_config}" > ../docs/parsers/universal.md
diff --git a/docs/lib.md b/docs/lib.md
index 8d2b7c2b..6b79fe1d 100644
--- a/docs/lib.md
+++ b/docs/lib.md
@@ -105,7 +105,7 @@ subset of `parser_mod_list()`.
### parser\_info
```python
-def parser_info(parser_mod_name: str) -> Union[Dict, None]
+def parser_info(parser_mod_name: str) -> Dict
```
Returns a dictionary that includes the module metadata.
@@ -118,10 +118,10 @@ This function will accept **module_name**, **cli-name**, and
### all\_parser\_info
```python
-def all_parser_info() -> List[Optional[Dict]]
+def all_parser_info() -> List[Dict]
```
-Returns a list of dictionaris that includes metadata for all modules.
+Returns a list of dictionaries that includes metadata for all modules.
diff --git a/docs/parsers/csv_s.md b/docs/parsers/csv_s.md
index eee33fae..6362a7a1 100644
--- a/docs/parsers/csv_s.md
+++ b/docs/parsers/csv_s.md
@@ -73,6 +73,7 @@ Examples:
### parse
```python
+@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False)
```
@@ -93,9 +94,9 @@ Yields:
Returns:
- Iterator object
+ Iterator object (generator)
### Parser Information
Compatibility: linux, darwin, cygwin, win32, aix, freebsd
-Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/date.md b/docs/parsers/date.md
index 5e2f4d42..e4966ba9 100644
--- a/docs/parsers/date.md
+++ b/docs/parsers/date.md
@@ -105,4 +105,4 @@ Returns:
### Parser Information
Compatibility: linux, darwin, freebsd
-Version 2.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 2.3 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/dig.md b/docs/parsers/dig.md
index fa3d492d..d1459682 100644
--- a/docs/parsers/dig.md
+++ b/docs/parsers/dig.md
@@ -350,4 +350,4 @@ Returns:
### Parser Information
Compatibility: linux, aix, freebsd, darwin, win32, cygwin
-Version 2.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 2.3 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/dir.md b/docs/parsers/dir.md
index db4651c9..02e08ac5 100644
--- a/docs/parsers/dir.md
+++ b/docs/parsers/dir.md
@@ -148,4 +148,4 @@ Returns:
### Parser Information
Compatibility: win32
-Version 1.4 by Rasheed Elsaleh (rasheed@rebelliondefense.com)
+Version 1.5 by Rasheed Elsaleh (rasheed@rebelliondefense.com)
diff --git a/docs/parsers/iostat_s.md b/docs/parsers/iostat_s.md
index fd116a85..1fc68dac 100644
--- a/docs/parsers/iostat_s.md
+++ b/docs/parsers/iostat_s.md
@@ -110,6 +110,7 @@ Examples:
### parse
```python
+@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False)
```
@@ -130,9 +131,9 @@ Yields:
Returns:
- Iterator object
+ Iterator object (generator)
### Parser Information
Compatibility: linux
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/ls.md b/docs/parsers/ls.md
index f9acc3df..2ae42823 100644
--- a/docs/parsers/ls.md
+++ b/docs/parsers/ls.md
@@ -144,4 +144,4 @@ Returns:
### Parser Information
Compatibility: linux, darwin, cygwin, aix, freebsd
-Version 1.10 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.11 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/ls_s.md b/docs/parsers/ls_s.md
index 1016aafa..bd0e5979 100644
--- a/docs/parsers/ls_s.md
+++ b/docs/parsers/ls_s.md
@@ -87,6 +87,7 @@ Examples:
### parse
```python
+@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False)
```
@@ -107,9 +108,9 @@ Yields:
Returns:
- Iterator object
+ Iterator object (generator)
### Parser Information
Compatibility: linux, darwin, cygwin, aix, freebsd
-Version 0.6 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/ping_s.md b/docs/parsers/ping_s.md
index f31a2c68..b904e2b7 100644
--- a/docs/parsers/ping_s.md
+++ b/docs/parsers/ping_s.md
@@ -93,6 +93,7 @@ Examples:
### parse
```python
+@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False)
```
@@ -113,9 +114,9 @@ Yields:
Returns:
- Iterator object
+ Iterator object (generator)
### Parser Information
Compatibility: linux, darwin, freebsd
-Version 0.6 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/rpm_qi.md b/docs/parsers/rpm_qi.md
index d3c4f71e..3d316d03 100644
--- a/docs/parsers/rpm_qi.md
+++ b/docs/parsers/rpm_qi.md
@@ -189,4 +189,4 @@ Returns:
### Parser Information
Compatibility: linux
-Version 1.4 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/rsync.md b/docs/parsers/rsync.md
new file mode 100644
index 00000000..04f8c664
--- /dev/null
+++ b/docs/parsers/rsync.md
@@ -0,0 +1,165 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+
+
+# jc.parsers.rsync
+
+jc - JSON CLI output utility `rsync` command output parser
+
+Supports the `-i` or `--itemize-changes` options with all levels of
+verbosity. This parser will process the STDOUT output or a log file
+generated with the `--log-file` option.
+
+Usage (cli):
+
+ $ rsync -i -a source/ dest | jc --rsync
+
+ or
+
+ $ jc rsync -i -a source/ dest
+
+ or
+
+ $ cat rsync-backup.log | jc --rsync
+
+Usage (module):
+
+ import jc
+ result = jc.parse('rsync', rsync_command_output)
+
+ or
+
+ import jc.parsers.rsync
+ result = jc.parsers.rsync.parse(rsync_command_output)
+
+Schema:
+
+ [
+ {
+ "summary": {
+ "date": string,
+ "time": string,
+ "process": integer,
+ "sent": integer,
+ "received": integer,
+ "total_size": integer,
+ "matches": integer,
+ "hash_hits": integer,
+ "false_alarms": integer,
+ "data": integer,
+ "bytes_sec": float,
+ "speedup": float
+ },
+ "files": [
+ {
+ "filename": string,
+ "date": string,
+ "time": string,
+ "process": integer,
+ "metadata": string,
+ "update_type": string/null, [0]
+ "file_type": string/null, [1]
+ "checksum_or_value_different": bool/null,
+ "size_different": bool/null,
+ "modification_time_different": bool/null,
+ "permissions_different": bool/null,
+ "owner_different": bool/null,
+ "group_different": bool/null,
+ "acl_different": bool/null,
+ "extended_attribute_different": bool/null,
+ "epoch": integer, [2]
+ }
+ ]
+ }
+ ]
+
+ [0] 'file sent', 'file received', 'local change or creation',
+ 'hard link', 'not updated', 'message'
+ [1] 'file', 'directory', 'symlink', 'device', 'special file'
+ [2] naive timestamp if time and date fields exist and can be converted.
+
+Examples:
+
+ $ rsync -i -a source/ dest | jc --rsync -p
+ [
+ {
+ "summary": {
+ "sent": 1708,
+ "received": 8209,
+ "bytes_sec": 19834.0,
+ "total_size": 235,
+ "speedup": 0.02
+ },
+ "files": [
+ {
+ "filename": "./",
+ "metadata": ".d..t......",
+ "update_type": "not updated",
+ "file_type": "directory",
+ "checksum_or_value_different": false,
+ "size_different": false,
+ "modification_time_different": true,
+ "permissions_different": false,
+ "owner_different": false,
+ "group_different": false,
+ "acl_different": false,
+ "extended_attribute_different": false
+ },
+ ...
+ ]
+ }
+ ]
+
+ $ rsync | jc --rsync -p -r
+ [
+ {
+ "summary": {
+ "sent": "1,708",
+ "received": "8,209",
+ "bytes_sec": "19,834.00",
+ "total_size": "235",
+ "speedup": "0.02"
+ },
+ "files": [
+ {
+ "filename": "./",
+ "metadata": ".d..t......",
+ "update_type": "not updated",
+ "file_type": "directory",
+ "checksum_or_value_different": false,
+ "size_different": false,
+ "modification_time_different": true,
+ "permissions_different": false,
+ "owner_different": false,
+ "group_different": false,
+ "acl_different": false,
+ "extended_attribute_different": false
+ },
+ ...
+ ]
+ }
+ ]
+
+
+
+### parse
+
+```python
+def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict]
+```
+
+Main text parsing function
+
+Parameters:
+
+ data: (string) text data to parse
+ raw: (boolean) unprocessed output if True
+ quiet: (boolean) suppress warning messages if True
+
+Returns:
+
+ List of Dictionaries. Raw or processed structured data.
+
+### Parser Information
+Compatibility: linux, darwin, freebsd
+
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/rsync_s.md b/docs/parsers/rsync_s.md
new file mode 100644
index 00000000..27b2c9a7
--- /dev/null
+++ b/docs/parsers/rsync_s.md
@@ -0,0 +1,127 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+
+
+# jc.parsers.rsync\_s
+
+jc - JSON CLI output utility `rsync` command output streaming parser
+
+> This streaming parser outputs JSON Lines
+
+Supports the `-i` or `--itemize-changes` options with all levels of
+verbosity. This parser will process the STDOUT output or a log file
+generated with the `--log-file` option.
+
+Usage (cli):
+
+ $ rsync -i -a source/ dest | jc --rsync-s
+
+ or
+
+ $ cat rsync-backup.log | jc --rsync-s
+
+Usage (module):
+
+ import jc
+ # result is an iterable object (generator)
+ result = jc.parse('rsync_s', rsync_command_output.splitlines())
+ for item in result:
+ # do something
+
+ or
+
+ import jc.parsers.rsync_s
+ # result is an iterable object (generator)
+ result = jc.parsers.rsync_s.parse(rsync_command_output.splitlines())
+ for item in result:
+ # do something
+
+Schema:
+
+ {
+ "type": string, # 'file' or 'summary'
+ "date": string,
+ "time": string,
+ "process": integer,
+ "sent": integer,
+ "received": integer,
+ "total_size": integer,
+ "matches": integer,
+ "hash_hits": integer,
+ "false_alarms": integer,
+ "data": integer,
+ "bytes_sec": float,
+ "speedup": float,
+ "filename": string,
+ "date": string,
+ "time": string,
+ "process": integer,
+ "metadata": string,
+ "update_type": string/null, [0]
+ "file_type": string/null, [1]
+ "checksum_or_value_different": bool/null,
+ "size_different": bool/null,
+ "modification_time_different": bool/null,
+ "permissions_different": bool/null,
+ "owner_different": bool/null,
+ "group_different": bool/null,
+ "acl_different": bool/null,
+ "extended_attribute_different": bool/null,
+ "epoch": integer, [2]
+
+ # Below object only exists if using -qq or ignore_exceptions=True
+
+ "_jc_meta":
+ {
+ "success": boolean, # false if error parsing
+ "error": string, # exists if "success" is false
+ "line": string # exists if "success" is false
+ }
+ }
+
+ [0] 'file sent', 'file received', 'local change or creation',
+ 'hard link', 'not updated', 'message'
+ [1] 'file', 'directory', 'symlink', 'device', 'special file'
+ [2] naive timestamp if time and date fields exist and can be converted.
+
+Examples:
+
+ $ rsync -i -a source/ dest | jc --rsync-s
+ {"type":"file","filename":"./","metadata":".d..t......","update_...}
+ ...
+
+ $ cat rsync_backup.log | jc --rsync-s
+ {"type":"file","filename":"./","date":"2022/01/28","time":"03:53...}
+ ...
+
+
+
+### parse
+
+```python
+@add_jc_meta
+def parse(data: Iterable[str], raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False) -> Union[Iterable[Dict], tuple]
+```
+
+Main text parsing generator function. Returns an iterator object.
+
+Parameters:
+
+ data: (iterable) line-based text data to parse
+ (e.g. sys.stdin or str.splitlines())
+
+ raw: (boolean) unprocessed output if True
+ quiet: (boolean) suppress warning messages if True
+ ignore_exceptions: (boolean) ignore parsing exceptions if True
+
+Yields:
+
+ Dictionary. Raw or processed structured data.
+
+Returns:
+
+ Iterator object (generator)
+
+### Parser Information
+Compatibility: linux, darwin, freebsd
+
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/stat.md b/docs/parsers/stat.md
index 72566c41..b5db81dd 100644
--- a/docs/parsers/stat.md
+++ b/docs/parsers/stat.md
@@ -198,4 +198,4 @@ Returns:
### Parser Information
Compatibility: linux, darwin, freebsd
-Version 1.10 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.11 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/stat_s.md b/docs/parsers/stat_s.md
index 4f560f7c..4a59f84d 100644
--- a/docs/parsers/stat_s.md
+++ b/docs/parsers/stat_s.md
@@ -91,6 +91,7 @@ Examples:
### parse
```python
+@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False)
```
@@ -111,9 +112,9 @@ Yields:
Returns:
- Iterator object
+ Iterator object (generator)
### Parser Information
Compatibility: linux, darwin, freebsd
-Version 0.5 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/systeminfo.md b/docs/parsers/systeminfo.md
index fa77289c..bd85b45f 100644
--- a/docs/parsers/systeminfo.md
+++ b/docs/parsers/systeminfo.md
@@ -239,4 +239,4 @@ Returns:
### Parser Information
Compatibility: win32
-Version 1.1 by Jon Smith (jon@rebelliondefense.com)
+Version 1.2 by Jon Smith (jon@rebelliondefense.com)
diff --git a/docs/parsers/timedatectl.md b/docs/parsers/timedatectl.md
index 8c4b1539..0e40adbe 100644
--- a/docs/parsers/timedatectl.md
+++ b/docs/parsers/timedatectl.md
@@ -92,4 +92,4 @@ Returns:
### Parser Information
Compatibility: linux
-Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/universal.md b/docs/parsers/universal.md
index f360f875..00dca1d3 100644
--- a/docs/parsers/universal.md
+++ b/docs/parsers/universal.md
@@ -40,7 +40,7 @@ Returns:
### sparse\_table\_parse
```python
-def sparse_table_parse(data: List[str], delim: Optional[str] = '\u2063') -> List[Dict]
+def sparse_table_parse(data: List[str], delim: str = '\u2063') -> List[Dict]
```
Parse tables with missing column data or with spaces in column data.
diff --git a/docs/parsers/upower.md b/docs/parsers/upower.md
index ffcbb334..be0384ad 100644
--- a/docs/parsers/upower.md
+++ b/docs/parsers/upower.md
@@ -226,4 +226,4 @@ Returns:
### Parser Information
Compatibility: linux
-Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.4 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/vmstat.md b/docs/parsers/vmstat.md
index 2291fb79..c6e76a38 100644
--- a/docs/parsers/vmstat.md
+++ b/docs/parsers/vmstat.md
@@ -154,4 +154,4 @@ Returns:
### Parser Information
Compatibility: linux
-Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/vmstat_s.md b/docs/parsers/vmstat_s.md
index 6f85dce7..2445f1b6 100644
--- a/docs/parsers/vmstat_s.md
+++ b/docs/parsers/vmstat_s.md
@@ -110,6 +110,7 @@ Examples:
### parse
```python
+@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False)
```
@@ -130,9 +131,9 @@ Yields:
Returns:
- Iterator object
+ Iterator object (generator)
### Parser Information
Compatibility: linux
-Version 0.6 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/who.md b/docs/parsers/who.md
index b1d5996f..94953475 100644
--- a/docs/parsers/who.md
+++ b/docs/parsers/who.md
@@ -163,4 +163,4 @@ Returns:
### Parser Information
Compatibility: linux, darwin, cygwin, aix, freebsd
-Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com)
diff --git a/docs/parsers/xrandr.md b/docs/parsers/xrandr.md
new file mode 100644
index 00000000..dcc7fea2
--- /dev/null
+++ b/docs/parsers/xrandr.md
@@ -0,0 +1,166 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+
+
+# jc.parsers.xrandr
+
+jc - JSON CLI output utility `xrandr` command output parser
+
+Usage (cli):
+
+ $ xrandr | jc --xrandr
+
+ or
+
+ $ jc xrandr
+
+Usage (module):
+
+ import jc
+ result = jc.parse('xrandr', xrandr_command_output)
+
+ or
+
+ import jc.parsers.xrandr
+ result = jc.parsers.xrandr.parse(xrandr_command_output)
+
+Schema:
+
+ {
+ "screens": [
+ {
+ "screen_number": integer,
+ "minimum_width": integer,
+ "minimum_height": integer,
+ "current_width": integer,
+ "current_height": integer,
+ "maximum_width": integer,
+ "maximum_height": integer,
+ "associated_device": {
+ "associated_modes": [
+ {
+ "resolution_width": integer,
+ "resolution_height": integer,
+ "is_high_resolution": boolean,
+ "frequencies": [
+ {
+ "frequency": float,
+ "is_current": boolean,
+ "is_preferred": boolean
+ }
+ ],
+ "is_connected": boolean,
+ "is_primary": boolean,
+ "device_name": string,
+ "resolution_width": integer,
+ "resolution_height": integer,
+ "offset_width": integer,
+ "offset_height": integer,
+ "dimension_width": integer,
+ "dimension_height": integer
+ }
+ }
+ ],
+ "unassociated_devices": [
+ {
+ "associated_modes": [
+ {
+ "resolution_width": integer,
+ "resolution_height": integer,
+ "is_high_resolution": boolean,
+ "frequencies": [
+ {
+ "frequency": float,
+ "is_current": boolean,
+ "is_preferred": boolean
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+
+Examples:
+
+ $ xrandr | jc --xrandr -p
+ {
+ "screens": [
+ {
+ "screen_number": 0,
+ "minimum_width": 8,
+ "minimum_height": 8,
+ "current_width": 1920,
+ "current_height": 1080,
+ "maximum_width": 32767,
+ "maximum_height": 32767,
+ "associated_device": {
+ "associated_modes": [
+ {
+ "resolution_width": 1920,
+ "resolution_height": 1080,
+ "is_high_resolution": false,
+ "frequencies": [
+ {
+ "frequency": 60.03,
+ "is_current": true,
+ "is_preferred": true
+ },
+ {
+ "frequency": 59.93,
+ "is_current": false,
+ "is_preferred": false
+ }
+ ]
+ },
+ {
+ "resolution_width": 1680,
+ "resolution_height": 1050,
+ "is_high_resolution": false,
+ "frequencies": [
+ {
+ "frequency": 59.88,
+ "is_current": false,
+ "is_preferred": false
+ }
+ ]
+ }
+ ],
+ "is_connected": true,
+ "is_primary": true,
+ "device_name": "eDP1",
+ "resolution_width": 1920,
+ "resolution_height": 1080,
+ "offset_width": 0,
+ "offset_height": 0,
+ "dimension_width": 310,
+ "dimension_height": 170
+ }
+ }
+ ],
+ "unassociated_devices": []
+ }
+
+
+
+### parse
+
+```python
+def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict
+```
+
+Main text parsing function
+
+Parameters:
+
+ data: (string) text data to parse
+ raw: (boolean) unprocessed output if True
+ quiet: (boolean) suppress warning messages if True
+
+Returns:
+
+ Dictionary. Raw or processed structured data.
+
+### Parser Information
+Compatibility: linux, darwin, cygwin, aix, freebsd
+
+Version 1.0 by Kevin Lyter (lyter_git at sent.com)
diff --git a/docs/parsers/zipinfo.md b/docs/parsers/zipinfo.md
index 49a4712a..665a1a31 100644
--- a/docs/parsers/zipinfo.md
+++ b/docs/parsers/zipinfo.md
@@ -44,7 +44,7 @@ Schema:
{
"flags": string,
"zipversion": string,
- "zipunder": string
+ "zipunder": string,
"filesize": integer,
"type": string,
"method": string,
diff --git a/docs/readme.md b/docs/readme.md
index 073af8bc..7410dd4f 100644
--- a/docs/readme.md
+++ b/docs/readme.md
@@ -14,6 +14,8 @@ and file-types to dictionaries and lists of dictionaries.
>>> help('jc')
>>> help('jc.lib')
>>> help('jc.utils')
+ >>> help('jc.streaming')
+ >>> help('jc.parsers.universal')
>>> jc.get_help('parser_module_name')
## Online Documentation
diff --git a/docs/streaming.md b/docs/streaming.md
new file mode 100644
index 00000000..b8971010
--- /dev/null
+++ b/docs/streaming.md
@@ -0,0 +1,114 @@
+# Table of Contents
+
+* [jc.streaming](#jc.streaming)
+ * [streaming\_input\_type\_check](#jc.streaming.streaming_input_type_check)
+ * [streaming\_line\_input\_type\_check](#jc.streaming.streaming_line_input_type_check)
+ * [stream\_success](#jc.streaming.stream_success)
+ * [stream\_error](#jc.streaming.stream_error)
+ * [add\_jc\_meta](#jc.streaming.add_jc_meta)
+ * [raise\_or\_yield](#jc.streaming.raise_or_yield)
+
+
+
+# jc.streaming
+
+jc - JSON CLI output utility streaming utils
+
+
+
+### streaming\_input\_type\_check
+
+```python
+def streaming_input_type_check(data: Iterable) -> None
+```
+
+Ensure input data is an iterable, but not a string or bytes. Raises
+`TypeError` if not.
+
+
+
+### streaming\_line\_input\_type\_check
+
+```python
+def streaming_line_input_type_check(line: str) -> None
+```
+
+Ensure each line is a string. Raises `TypeError` if not.
+
+
+
+### stream\_success
+
+```python
+def stream_success(output_line: Dict, ignore_exceptions: bool) -> Dict
+```
+
+Add `_jc_meta` object to output line if `ignore_exceptions=True`
+
+
+
+### stream\_error
+
+```python
+def stream_error(e: BaseException, line: str) -> Dict
+```
+
+Return an error `_jc_meta` field.
+
+
+
+### add\_jc\_meta
+
+```python
+def add_jc_meta(func)
+```
+
+Decorator for streaming parsers to add stream_success and stream_error
+objects. This simplifies the yield lines in the streaming parsers.
+
+With the decorator on parse():
+
+ # successfully parsed line:
+ yield output_line if raw else _process(output_line)
+
+ # unsuccessfully parsed line:
+ except Exception as e:
+ yield raise_or_yield(ignore_exceptions, e, line)
+
+Without the decorator on parse():
+
+ # successfully parsed line:
+ if raw:
+ yield stream_success(output_line, ignore_exceptions)
+ else:
+ stream_success(_process(output_line), ignore_exceptions)
+
+ # unsuccessfully parsed line:
+ except Exception as e:
+ yield stream_error(raise_or_yield(ignore_exceptions, e, line))
+
+In all cases above:
+
+ output_line: (Dict) successfully parsed line yielded as a dict
+
+ e: (BaseException) exception object as the first value
+ of the tuple if the line was not successfully parsed.
+
+ line: (str) string of the original line that did not
+ successfully parse.
+
+ ignore_exceptions: (bool) continue processing lines and ignore
+ exceptions if True.
+
+
+
+### raise\_or\_yield
+
+```python
+def raise_or_yield(ignore_exceptions: bool, e: BaseException, line: str) -> tuple
+```
+
+Return the exception object and line string if ignore_exceptions is
+True. Otherwise, re-raise the exception from the exception object with
+an annotation.
+
diff --git a/docs/utils.md b/docs/utils.md
index 2b914cac..c8c24180 100644
--- a/docs/utils.md
+++ b/docs/utils.md
@@ -8,11 +8,7 @@
* [convert\_to\_int](#jc.utils.convert_to_int)
* [convert\_to\_float](#jc.utils.convert_to_float)
* [convert\_to\_bool](#jc.utils.convert_to_bool)
- * [stream\_success](#jc.utils.stream_success)
- * [stream\_error](#jc.utils.stream_error)
* [input\_type\_check](#jc.utils.input_type_check)
- * [streaming\_input\_type\_check](#jc.utils.streaming_input_type_check)
- * [streaming\_line\_input\_type\_check](#jc.utils.streaming_line_input_type_check)
* [timestamp](#jc.utils.timestamp)
* [\_\_init\_\_](#jc.utils.timestamp.__init__)
@@ -67,7 +63,7 @@ Returns:
### compatibility
```python
-def compatibility(mod_name: str, compatible: List, quiet: Optional[bool] = False) -> None
+def compatibility(mod_name: str, compatible: List, quiet: bool = False) -> None
```
Checks for the parser's compatibility with the running OS
@@ -112,7 +108,7 @@ Returns:
### convert\_to\_int
```python
-def convert_to_int(value: Union[str, float]) -> Union[int, None]
+def convert_to_int(value: Union[str, float]) -> Optional[int]
```
Converts string and float input to int. Strips all non-numeric
@@ -131,7 +127,7 @@ Returns:
### convert\_to\_float
```python
-def convert_to_float(value: Union[str, int]) -> Union[float, None]
+def convert_to_float(value: Union[str, int]) -> Optional[float]
```
Converts string and int input to float. Strips all non-numeric
@@ -165,27 +161,6 @@ Returns:
True/False False unless a 'truthy' number or string is found
('y', 'yes', 'true', '1', 1, -1, etc.)
-
-
-### stream\_success
-
-```python
-def stream_success(output_line: Dict, ignore_exceptions: bool) -> Dict
-```
-
-Add `_jc_meta` object to output line if `ignore_exceptions=True`
-
-
-
-### stream\_error
-
-```python
-def stream_error(e: BaseException, ignore_exceptions: bool, line: str) -> Dict
-```
-
-Reraise the stream exception with annotation or print an error
-`_jc_meta` field if `ignore_exceptions=True`.
-
### input\_type\_check
@@ -196,27 +171,6 @@ def input_type_check(data: str) -> None
Ensure input data is a string. Raises `TypeError` if not.
-
-
-### streaming\_input\_type\_check
-
-```python
-def streaming_input_type_check(data: Iterable) -> None
-```
-
-Ensure input data is an iterable, but not a string or bytes. Raises
-`TypeError` if not.
-
-
-
-### streaming\_line\_input\_type\_check
-
-```python
-def streaming_line_input_type_check(line: str) -> None
-```
-
-Ensure each line is a string. Raises `TypeError` if not.
-
### timestamp Objects
@@ -230,29 +184,34 @@ class timestamp()
### \_\_init\_\_
```python
-def __init__(datetime_string: str) -> None
+def __init__(datetime_string: str, format_hint: Union[List, Tuple, None] = None) -> None
```
-Input a date-time text string of several formats and convert to a
+Input a datetime text string of several formats and convert to a
naive or timezone-aware epoch timestamp in UTC.
Parameters:
- datetime_string: (str) a string representation of a
- date-time in several supported formats
+ datetime_string (str): a string representation of a
+ datetime in several supported formats
-Attributes:
+ format_hint (list | tuple): an optional list of format ID
+ integers to instruct the timestamp object to try those
+ formats first in the order given. Other formats will be
+ tried after the format hint list is exhausted. This can
+ speed up timestamp conversion so several different formats
+ don't have to be tried in brute-force fashion.
- string (str) the input datetime string
+Returns a timestamp object with the following attributes:
- format (int) the format rule that was used to
- decode the datetime string. None if
- conversion fails
+ string (str): the input datetime string
- naive (int) timestamp based on locally configured
- timezone. None if conversion fails
+ format (int | None): the format rule that was used to decode
+ the datetime string. None if conversion fails.
- utc (int) aware timestamp only if UTC timezone
- detected in datetime string. None if
- conversion fails
+ naive (int | None): timestamp based on locally configured
+ timezone. None if conversion fails.
+
+ utc (int | None): aware timestamp only if UTC timezone
+ detected in datetime string. None if conversion fails.
diff --git a/jc/__init__.py b/jc/__init__.py
index d5a02709..a59ba5b2 100644
--- a/jc/__init__.py
+++ b/jc/__init__.py
@@ -10,6 +10,8 @@ and file-types to dictionaries and lists of dictionaries.
>>> help('jc')
>>> help('jc.lib')
>>> help('jc.utils')
+ >>> help('jc.streaming')
+ >>> help('jc.parsers.universal')
>>> jc.get_help('parser_module_name')
## Online Documentation
diff --git a/jc/cli.py b/jc/cli.py
index 20425df1..f1fd5660 100644
--- a/jc/cli.py
+++ b/jc/cli.py
@@ -89,13 +89,15 @@ def set_env_colors(env_colors=None):
"""
Return a dictionary to be used in Pygments custom style class.
- Grab custom colors from JC_COLORS environment variable. JC_COLORS env variable takes 4 comma
- separated string values and should be in the format of:
+ Grab custom colors from JC_COLORS environment variable. JC_COLORS env
+ variable takes 4 comma separated string values and should be in the
+ format of:
JC_COLORS=,,,
- Where colors are: black, red, green, yellow, blue, magenta, cyan, gray, brightblack, brightred,
- brightgreen, brightyellow, brightblue, brightmagenta, brightcyan, white, default
+ Where colors are: black, red, green, yellow, blue, magenta, cyan, gray,
+ brightblack, brightred, brightgreen, brightyellow,
+ brightblue, brightmagenta, brightcyan, white, default
Default colors:
@@ -132,8 +134,10 @@ def set_env_colors(env_colors=None):
def piped_output(force_color):
- """Return False if stdout is a TTY. True if output is being piped to another program
- and foce_color is True. This allows forcing of ANSI color codes even when using pipes.
+ """
+ Return False if stdout is a TTY. True if output is being piped to
+ another program and foce_color is True. This allows forcing of ANSI
+ color codes even when using pipes.
"""
return not sys.stdout.isatty() and not force_color
@@ -224,8 +228,8 @@ def helptext():
def help_doc(options):
"""
- Returns the parser documentation if a parser is found in the arguments, otherwise
- the general help text is returned.
+ Returns the parser documentation if a parser is found in the arguments,
+ otherwise the general help text is returned.
"""
for arg in options:
parser_name = parser_shortname(arg)
@@ -253,7 +257,10 @@ def versiontext():
def json_out(data, pretty=False, env_colors=None, mono=False, piped_out=False):
- """Return a JSON formatted string. String may include color codes or be pretty printed."""
+ """
+ Return a JSON formatted string. String may include color codes or be
+ pretty printed.
+ """
separators = (',', ':')
indent = None
@@ -277,10 +284,10 @@ def magic_parser(args):
Parse command arguments for magic syntax: jc -p ls -al
Return a tuple:
- valid_command (bool) is this a valid command? (exists in magic dict)
- run_command (list) list of the user's command to run. None if no command.
- jc_parser (str) parser to use for this user's command.
- jc_options (list) list of jc options
+ valid_command (bool) is this a valid cmd? (exists in magic dict)
+ run_command (list) list of the user's cmd to run. None if no cmd.
+ jc_parser (str) parser to use for this user's cmd.
+ jc_options (list) list of jc options
"""
# bail immediately if there are no args or a parser is defined
if len(args) <= 1 or args[1].startswith('--'):
@@ -335,12 +342,15 @@ def magic_parser(args):
def run_user_command(command):
- """Use subprocess to run the user's command. Returns the STDOUT, STDERR, and the Exit Code as a tuple."""
+ """
+ Use subprocess to run the user's command. Returns the STDOUT, STDERR,
+ and the Exit Code as a tuple.
+ """
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
- close_fds=False, # Allows inheriting file descriptors. Useful for process substitution
- universal_newlines=True)
+ close_fds=False, # Allows inheriting file descriptors;
+ universal_newlines=True) # useful for process substitution
stdout, stderr = proc.communicate()
return (
@@ -441,14 +451,18 @@ def main():
raise
error_msg = os.strerror(e.errno)
- utils.error_message([f'"{run_command_str}" command could not be run: {error_msg}. For details use the -d or -dd option.'])
+ utils.error_message([
+ f'"{run_command_str}" command could not be run: {error_msg}. For details use the -d or -dd option.'
+ ])
sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT))
except Exception:
if debug:
raise
- utils.error_message([f'"{run_command_str}" command could not be run. For details use the -d or -dd option.'])
+ utils.error_message([
+ f'"{run_command_str}" command could not be run. For details use the -d or -dd option.'
+ ])
sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT))
elif run_command is not None:
@@ -489,7 +503,10 @@ def main():
# streaming
if getattr(parser.info, 'streaming', None):
- result = parser.parse(sys.stdin, raw=raw, quiet=quiet, ignore_exceptions=ignore_exceptions)
+ result = parser.parse(sys.stdin,
+ raw=raw,
+ quiet=quiet,
+ ignore_exceptions=ignore_exceptions)
for line in result:
print(json_out(line,
pretty=pretty,
@@ -503,7 +520,9 @@ def main():
# regular
else:
data = magic_stdout or sys.stdin.read()
- result = parser.parse(data, raw=raw, quiet=quiet)
+ result = parser.parse(data,
+ raw=raw,
+ quiet=quiet)
print(json_out(result,
pretty=pretty,
env_colors=jc_colors,
@@ -517,10 +536,11 @@ def main():
if debug:
raise
- utils.error_message([f'Parser issue with {parser_name}:',
- f'{e.__class__.__name__}: {e}',
- 'If this is the correct parser, try setting the locale to C (LANG=C).',
- 'For details use the -d or -dd option. Use "jc -h" for help.'])
+ utils.error_message([
+ f'Parser issue with {parser_name}:', f'{e.__class__.__name__}: {e}',
+ 'If this is the correct parser, try setting the locale to C (LANG=C).',
+ 'For details use the -d or -dd option. Use "jc -h" for help.'
+ ])
sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT))
except json.JSONDecodeError:
diff --git a/jc/lib.py b/jc/lib.py
index ab6b6b38..bbf3a43b 100644
--- a/jc/lib.py
+++ b/jc/lib.py
@@ -6,10 +6,10 @@ import sys
import os
import re
import importlib
-from typing import Dict, List, Iterable, Union, Iterator, Optional
+from typing import Dict, List, Iterable, Union, Iterator
from jc import appdirs
-__version__ = '1.18.2'
+__version__ = '1.18.3'
parsers = [
'acpi',
@@ -69,6 +69,8 @@ parsers = [
'ps',
'route',
'rpm-qi',
+ 'rsync',
+ 'rsync-s',
'sfdisk',
'shadow',
'ss',
@@ -95,6 +97,7 @@ parsers = [
'wc',
'who',
'xml',
+ 'xrandr',
'yaml',
'zipinfo'
]
@@ -224,7 +227,7 @@ def plugin_parser_mod_list() -> List[str]:
"""
return [_cliname_to_modname(p) for p in local_parsers]
-def parser_info(parser_mod_name: str) -> Union[Dict, None]:
+def parser_info(parser_mod_name: str) -> Dict:
"""
Returns a dictionary that includes the module metadata.
@@ -235,9 +238,9 @@ def parser_info(parser_mod_name: str) -> Union[Dict, None]:
parser_mod_name = _cliname_to_modname(parser_mod_name)
parser_mod = _get_parser(parser_mod_name)
+ info_dict: Dict = {}
if hasattr(parser_mod, 'info'):
- info_dict: Dict = {}
info_dict['name'] = parser_mod_name
info_dict['argument'] = _parser_argument(parser_mod_name)
parser_entry = vars(parser_mod.info)
@@ -249,15 +252,13 @@ def parser_info(parser_mod_name: str) -> Union[Dict, None]:
if _modname_to_cliname(parser_mod_name) in local_parsers:
info_dict['plugin'] = True
- return info_dict
+ return info_dict
- return None
-
-def all_parser_info() -> List[Optional[Dict]]:
+def all_parser_info() -> List[Dict]:
"""
- Returns a list of dictionaris that includes metadata for all modules.
+ Returns a list of dictionaries that includes metadata for all modules.
"""
- return [parser_info(_cliname_to_modname(p)) for p in parsers]
+ return [parser_info(p) for p in parsers]
def get_help(parser_mod_name: str) -> None:
"""
diff --git a/jc/parsers/csv_s.py b/jc/parsers/csv_s.py
index c692dde8..e4c573c4 100644
--- a/jc/parsers/csv_s.py
+++ b/jc/parsers/csv_s.py
@@ -66,13 +66,13 @@ Examples:
import itertools
import csv
import jc.utils
-from jc.utils import stream_success, stream_error
+from jc.streaming import streaming_input_type_check, add_jc_meta, raise_or_yield
from jc.exceptions import ParseError
class info():
"""Provides parser metadata (version, author, etc.)"""
- version = '1.2'
+ version = '1.3'
description = 'CSV file streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@@ -101,6 +101,7 @@ def _process(proc_data):
return proc_data
+@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False):
"""
Main text parsing generator function. Returns an iterator object.
@@ -120,10 +121,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
Returns:
- Iterator object
+ Iterator object (generator)
"""
jc.utils.compatibility(__name__, info.compatible, quiet)
- jc.utils.streaming_input_type_check(data)
+ streaming_input_type_check(data)
# convert data to an iterable in case a sequence like a list is used as input.
# this allows the exhaustion of the input so we don't double-process later.
@@ -155,6 +156,6 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
for row in reader:
try:
- yield stream_success(row, ignore_exceptions) if raw else stream_success(_process(row), ignore_exceptions)
+ yield row if raw else _process(row)
except Exception as e:
- yield stream_error(e, ignore_exceptions, row)
+ yield raise_or_yield(ignore_exceptions, e, str(row))
diff --git a/jc/parsers/date.py b/jc/parsers/date.py
index d09eed5d..df710f69 100644
--- a/jc/parsers/date.py
+++ b/jc/parsers/date.py
@@ -83,7 +83,7 @@ import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
- version = '2.2'
+ version = '2.3'
description = '`date` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@@ -168,7 +168,7 @@ def parse(data, raw=False, quiet=False):
dt = None
dt_utc = None
- timestamp = jc.utils.timestamp(data)
+ timestamp = jc.utils.timestamp(data, format_hint=(1000, 6000, 7000))
if timestamp.naive:
dt = datetime.fromtimestamp(timestamp.naive)
if timestamp.utc:
diff --git a/jc/parsers/dig.py b/jc/parsers/dig.py
index b7d32d50..0af225b1 100644
--- a/jc/parsers/dig.py
+++ b/jc/parsers/dig.py
@@ -327,7 +327,7 @@ import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
- version = '2.2'
+ version = '2.3'
description = '`dig` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@@ -384,7 +384,7 @@ def _process(proc_data):
auth['ttl'] = jc.utils.convert_to_int(auth['ttl'])
if 'when' in entry:
- ts = jc.utils.timestamp(entry['when'])
+ ts = jc.utils.timestamp(entry['when'], format_hint=(1000, 7000))
entry['when_epoch'] = ts.naive
entry['when_epoch_utc'] = ts.utc
diff --git a/jc/parsers/dir.py b/jc/parsers/dir.py
index 5c65750d..f9f168b1 100644
--- a/jc/parsers/dir.py
+++ b/jc/parsers/dir.py
@@ -126,7 +126,7 @@ import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
- version = '1.4'
+ version = '1.5'
description = '`dir` command parser'
author = 'Rasheed Elsaleh'
author_email = 'rasheed@rebelliondefense.com'
@@ -152,7 +152,7 @@ def _process(proc_data):
# add timestamps
if 'date' in entry and 'time' in entry:
dt = entry['date'] + ' ' + entry['time']
- timestamp = jc.utils.timestamp(dt)
+ timestamp = jc.utils.timestamp(dt, format_hint=(1600,))
entry['epoch'] = timestamp.naive
# add ints
diff --git a/jc/parsers/foo.py b/jc/parsers/foo.py
index bd077f3f..afd56deb 100644
--- a/jc/parsers/foo.py
+++ b/jc/parsers/foo.py
@@ -38,8 +38,8 @@ Examples:
$ foo | jc --foo -p -r
[]
"""
-import jc.utils
from typing import List, Dict
+import jc.utils
class info():
diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py
index 44e0ee31..eadfc202 100644
--- a/jc/parsers/foo_s.py
+++ b/jc/parsers/foo_s.py
@@ -49,9 +49,11 @@ Examples:
{example output}
...
"""
-from typing import Dict, Iterable
+from typing import Dict, Iterable, Union
import jc.utils
-from jc.utils import stream_success, stream_error
+from jc.streaming import (
+ add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield
+)
from jc.exceptions import ParseError
@@ -63,7 +65,7 @@ class info():
author_email = 'johndoe@gmail.com'
# compatible options: linux, darwin, cygwin, win32, aix, freebsd
- compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
+ compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
streaming = True
@@ -91,12 +93,13 @@ def _process(proc_data: Dict) -> Dict:
return proc_data
+@add_jc_meta
def parse(
data: Iterable[str],
raw: bool = False,
quiet: bool = False,
ignore_exceptions: bool = False
-) -> Iterable[Dict]:
+) -> Union[Iterable[Dict], tuple]:
"""
Main text parsing generator function. Returns an iterator object.
@@ -115,24 +118,24 @@ def parse(
Returns:
- Iterator object
+ Iterator object (generator)
"""
jc.utils.compatibility(__name__, info.compatible, quiet)
- jc.utils.streaming_input_type_check(data)
+ streaming_input_type_check(data)
for line in data:
- output_line: Dict = {}
try:
- jc.utils.streaming_line_input_type_check(line)
+ streaming_line_input_type_check(line)
+ output_line: Dict = {}
# parse the content here
# check out helper functions in jc.utils
# and jc.parsers.universal
if output_line:
- yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
+ yield output_line if raw else _process(output_line)
else:
raise ParseError('Not foo data')
except Exception as e:
- yield stream_error(e, ignore_exceptions, line)
+ yield raise_or_yield(ignore_exceptions, e, line)
diff --git a/jc/parsers/iostat_s.py b/jc/parsers/iostat_s.py
index d6ac5b4b..d98f58bc 100644
--- a/jc/parsers/iostat_s.py
+++ b/jc/parsers/iostat_s.py
@@ -101,14 +101,16 @@ Examples:
...
"""
import jc.utils
-from jc.utils import stream_success, stream_error
+from jc.streaming import (
+ add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield
+)
from jc.exceptions import ParseError
import jc.parsers.universal
class info():
"""Provides parser metadata (version, author, etc.)"""
- version = '1.0'
+ version = '1.1'
description = '`iostat` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@@ -159,6 +161,8 @@ def _create_obj_list(section_list, section_name):
item['type'] = section_name
return output_list
+
+@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False):
"""
Main text parsing generator function. Returns an iterator object.
@@ -178,10 +182,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
Returns:
- Iterator object
+ Iterator object (generator)
"""
jc.utils.compatibility(__name__, info.compatible, quiet)
- jc.utils.streaming_input_type_check(data)
+ streaming_input_type_check(data)
section = '' # either 'cpu' or 'device'
headers = ''
@@ -189,9 +193,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
device_list = []
for line in data:
- output_line = {}
try:
- jc.utils.streaming_line_input_type_check(line)
+ streaming_line_input_type_check(line)
+ output_line = {}
# ignore blank lines and header line
if line == '\n' or line == '' or line.startswith('Linux'):
@@ -223,9 +227,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
device_list = []
if output_line:
- yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
+ yield output_line if raw else _process(output_line)
else:
raise ParseError('Not iostat data')
except Exception as e:
- yield stream_error(e, ignore_exceptions, line)
+ yield raise_or_yield(ignore_exceptions, e, line)
diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py
index e497c194..fba2c2a4 100644
--- a/jc/parsers/ls.py
+++ b/jc/parsers/ls.py
@@ -122,7 +122,7 @@ import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
- version = '1.10'
+ version = '1.11'
description = '`ls` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@@ -154,7 +154,7 @@ def _process(proc_data):
if 'date' in entry:
# to speed up processing only try to convert the date if it's not the default format
if not re.match(r'[a-zA-Z]{3}\s{1,2}\d{1,2}\s{1,2}[0-9:]{4,5}', entry['date']):
- ts = jc.utils.timestamp(entry['date'])
+ ts = jc.utils.timestamp(entry['date'], format_hint=(7200,))
entry['epoch'] = ts.naive
entry['epoch_utc'] = ts.utc
diff --git a/jc/parsers/ls_s.py b/jc/parsers/ls_s.py
index 7de75ce9..0ca3261c 100644
--- a/jc/parsers/ls_s.py
+++ b/jc/parsers/ls_s.py
@@ -79,13 +79,15 @@ Examples:
"""
import re
import jc.utils
-from jc.utils import stream_success, stream_error
+from jc.streaming import (
+ add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield
+)
from jc.exceptions import ParseError
class info():
"""Provides parser metadata (version, author, etc.)"""
- version = '0.6'
+ version = '1.0'
description = '`ls` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@@ -116,13 +118,14 @@ def _process(proc_data):
if 'date' in proc_data:
# to speed up processing only try to convert the date if it's not the default format
if not re.match(r'[a-zA-Z]{3}\s{1,2}\d{1,2}\s{1,2}[0-9:]{4,5}', proc_data['date']):
- ts = jc.utils.timestamp(proc_data['date'])
+ ts = jc.utils.timestamp(proc_data['date'], format_hint=(7200,))
proc_data['epoch'] = ts.naive
proc_data['epoch_utc'] = ts.utc
return proc_data
+@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False):
"""
Main text parsing generator function. Returns an iterator object.
@@ -142,16 +145,16 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
Returns:
- Iterator object
+ Iterator object (generator)
"""
jc.utils.compatibility(__name__, info.compatible, quiet)
- jc.utils.streaming_input_type_check(data)
+ streaming_input_type_check(data)
parent = ''
for line in data:
try:
- jc.utils.streaming_line_input_type_check(line)
+ streaming_line_input_type_check(line)
# skip line if it starts with 'total 1234'
if re.match(r'total [0-9]+', line):
@@ -163,7 +166,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
# Look for parent line if glob or -R is used
if not re.match(r'[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', line) \
- and line.strip().endswith(':'):
+ and line.strip().endswith(':'):
parent = line.strip()[:-1]
continue
@@ -196,7 +199,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
output_line['size'] = parsed_line[4]
output_line['date'] = ' '.join(parsed_line[5:8])
- yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
+ yield output_line if raw else _process(output_line)
except Exception as e:
- yield stream_error(e, ignore_exceptions, line)
+ yield raise_or_yield(ignore_exceptions, e, line)
diff --git a/jc/parsers/ping_s.py b/jc/parsers/ping_s.py
index 8536c3f5..ab80d11b 100644
--- a/jc/parsers/ping_s.py
+++ b/jc/parsers/ping_s.py
@@ -86,13 +86,15 @@ Examples:
import string
import ipaddress
import jc.utils
+from jc.streaming import (
+ add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield
+)
from jc.exceptions import ParseError
-from jc.utils import stream_success, stream_error
class info():
"""Provides parser metadata (version, author, etc.)"""
- version = '0.6'
+ version = '1.0'
description = '`ping` and `ping6` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@@ -469,6 +471,7 @@ def _linux_parse(line, s):
return output_line
+@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False):
"""
Main text parsing generator function. Returns an iterator object.
@@ -488,17 +491,17 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
Returns:
- Iterator object
+ Iterator object (generator)
"""
+ jc.utils.compatibility(__name__, info.compatible, quiet)
+ streaming_input_type_check(data)
+
s = _state()
- jc.utils.compatibility(__name__, info.compatible, quiet)
- jc.utils.streaming_input_type_check(data)
-
for line in data:
- output_line = {}
try:
- jc.utils.streaming_line_input_type_check(line)
+ streaming_line_input_type_check(line)
+ output_line = {}
# skip blank lines
if line.strip() == '':
@@ -542,9 +545,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
# yield the output line if it has data
if output_line:
- yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
+ yield output_line if raw else _process(output_line)
else:
continue
except Exception as e:
- yield stream_error(e, ignore_exceptions, line)
+ yield raise_or_yield(ignore_exceptions, e, line)
diff --git a/jc/parsers/rpm_qi.py b/jc/parsers/rpm_qi.py
index a0e6e689..2a6a81dd 100644
--- a/jc/parsers/rpm_qi.py
+++ b/jc/parsers/rpm_qi.py
@@ -166,7 +166,7 @@ import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
- version = '1.4'
+ version = '1.5'
description = '`rpm -qi` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@@ -197,12 +197,12 @@ def _process(proc_data):
entry[key] = jc.utils.convert_to_int(entry[key])
if 'build_date' in entry:
- timestamp = jc.utils.timestamp(entry['build_date'])
+ timestamp = jc.utils.timestamp(entry['build_date'], format_hint=(3000,))
entry['build_epoch'] = timestamp.naive
entry['build_epoch_utc'] = timestamp.utc
if 'install_date' in entry:
- timestamp = jc.utils.timestamp(entry['install_date'])
+ timestamp = jc.utils.timestamp(entry['install_date'], format_hint=(3000,))
entry['install_date_epoch'] = timestamp.naive
entry['install_date_epoch_utc'] = timestamp.utc
diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py
new file mode 100644
index 00000000..8b56761a
--- /dev/null
+++ b/jc/parsers/rsync.py
@@ -0,0 +1,493 @@
+"""jc - JSON CLI output utility `rsync` command output parser
+
+Supports the `-i` or `--itemize-changes` options with all levels of
+verbosity. This parser will process the STDOUT output or a log file
+generated with the `--log-file` option.
+
+Usage (cli):
+
+ $ rsync -i -a source/ dest | jc --rsync
+
+ or
+
+ $ jc rsync -i -a source/ dest
+
+ or
+
+ $ cat rsync-backup.log | jc --rsync
+
+Usage (module):
+
+ import jc
+ result = jc.parse('rsync', rsync_command_output)
+
+ or
+
+ import jc.parsers.rsync
+ result = jc.parsers.rsync.parse(rsync_command_output)
+
+Schema:
+
+ [
+ {
+ "summary": {
+ "date": string,
+ "time": string,
+ "process": integer,
+ "sent": integer,
+ "received": integer,
+ "total_size": integer,
+ "matches": integer,
+ "hash_hits": integer,
+ "false_alarms": integer,
+ "data": integer,
+ "bytes_sec": float,
+ "speedup": float
+ },
+ "files": [
+ {
+ "filename": string,
+ "date": string,
+ "time": string,
+ "process": integer,
+ "metadata": string,
+ "update_type": string/null, [0]
+ "file_type": string/null, [1]
+ "checksum_or_value_different": bool/null,
+ "size_different": bool/null,
+ "modification_time_different": bool/null,
+ "permissions_different": bool/null,
+ "owner_different": bool/null,
+ "group_different": bool/null,
+ "acl_different": bool/null,
+ "extended_attribute_different": bool/null,
+ "epoch": integer, [2]
+ }
+ ]
+ }
+ ]
+
+ [0] 'file sent', 'file received', 'local change or creation',
+ 'hard link', 'not updated', 'message'
+ [1] 'file', 'directory', 'symlink', 'device', 'special file'
+ [2] naive timestamp if time and date fields exist and can be converted.
+
+Examples:
+
+ $ rsync -i -a source/ dest | jc --rsync -p
+ [
+ {
+ "summary": {
+ "sent": 1708,
+ "received": 8209,
+ "bytes_sec": 19834.0,
+ "total_size": 235,
+ "speedup": 0.02
+ },
+ "files": [
+ {
+ "filename": "./",
+ "metadata": ".d..t......",
+ "update_type": "not updated",
+ "file_type": "directory",
+ "checksum_or_value_different": false,
+ "size_different": false,
+ "modification_time_different": true,
+ "permissions_different": false,
+ "owner_different": false,
+ "group_different": false,
+ "acl_different": false,
+ "extended_attribute_different": false
+ },
+ ...
+ ]
+ }
+ ]
+
+ $ rsync | jc --rsync -p -r
+ [
+ {
+ "summary": {
+ "sent": "1,708",
+ "received": "8,209",
+ "bytes_sec": "19,834.00",
+ "total_size": "235",
+ "speedup": "0.02"
+ },
+ "files": [
+ {
+ "filename": "./",
+ "metadata": ".d..t......",
+ "update_type": "not updated",
+ "file_type": "directory",
+ "checksum_or_value_different": false,
+ "size_different": false,
+ "modification_time_different": true,
+ "permissions_different": false,
+ "owner_different": false,
+ "group_different": false,
+ "acl_different": false,
+ "extended_attribute_different": false
+ },
+ ...
+ ]
+ }
+ ]
+"""
+import re
+from copy import deepcopy
+from typing import List, Dict
+import jc.utils
+
+
+class info():
+ """Provides parser metadata (version, author, etc.)"""
+ version = '1.0'
+ description = '`rsync` command parser'
+ author = 'Kelly Brazil'
+ author_email = 'kellyjonbrazil@gmail.com'
+ compatible = ['linux', 'darwin', 'freebsd']
+ magic_commands = ['rsync -i', 'rsync --itemize-changes']
+
+
+__version__ = info.version
+
+
+def _process(proc_data: List[Dict]) -> List[Dict]:
+ """
+ Final processing to conform to the schema.
+
+ Parameters:
+
+ proc_data: (List of Dictionaries) raw structured data to process
+
+ Returns:
+
+ List of Dictionaries. Structured to conform to the schema.
+ """
+ int_list = [
+ 'process', 'sent', 'received', 'total_size', 'matches', 'hash_hits',
+ 'false_alarms', 'data'
+ ]
+ float_list = ['bytes_sec', 'speedup']
+
+ for item in proc_data:
+ for key in item['summary']:
+ if key in int_list:
+ item['summary'][key] = jc.utils.convert_to_int(item['summary'][key])
+ if key in float_list:
+ item['summary'][key] = jc.utils.convert_to_float(item['summary'][key])
+
+ for entry in item['files']:
+ for key in entry:
+ if key in int_list:
+ entry[key] = jc.utils.convert_to_int(entry[key])
+
+ # add timestamp
+ if 'date' in entry and 'time' in entry:
+ date = entry['date'].replace('/', '-')
+ date_time = f'{date} {entry["time"]}'
+ ts = jc.utils.timestamp(date_time, format_hint=(7250,))
+ entry['epoch'] = ts.naive
+
+ return proc_data
+
+
+def parse(
+ data: str,
+ raw: bool = False,
+ quiet: bool = False
+) -> List[Dict]:
+ """
+ Main text parsing function
+
+ Parameters:
+
+ data: (string) text data to parse
+ raw: (boolean) unprocessed output if True
+ quiet: (boolean) suppress warning messages if True
+
+ Returns:
+
+ List of Dictionaries. Raw or processed structured data.
+ """
+ jc.utils.compatibility(__name__, info.compatible, quiet)
+ jc.utils.input_type_check(data)
+
+ raw_output: List = []
+
+ rsync_run_new: Dict = {
+ 'summary': {},
+ 'files': []
+ }
+
+ rsync_run = deepcopy(rsync_run_new)
+
+ last_process = ''
+
+ update_type = {
+ '<': 'file sent',
+ '>': 'file received',
+ 'c': 'local change or creation',
+ 'h': 'hard link',
+ '.': 'not updated',
+ '*': 'message',
+ '+': None
+ }
+
+ file_type = {
+ 'f': 'file',
+ 'd': 'directory',
+ 'L': 'symlink',
+ 'D': 'device',
+ 'S': 'special file',
+ '+': None
+ }
+
+ checksum_or_value_different = {
+ 'c': True,
+ '.': False,
+ '+': None,
+ ' ': None,
+ '?': None
+ }
+
+ size_different = {
+ 's': True,
+ '.': False,
+ '+': None,
+ ' ': None,
+ '?': None
+ }
+
+ modification_time_different = {
+ 't': True,
+ '.': False,
+ '+': None,
+ ' ': None,
+ '?': None
+ }
+
+ permissions_different = {
+ 'p': True,
+ '.': False,
+ '+': None,
+ ' ': None,
+ '?': None
+ }
+
+ owner_different = {
+ 'o': True,
+ '.': False,
+ '+': None,
+ ' ': None,
+ '?': None
+ }
+
+ group_different = {
+ 'g': True,
+ '.': False,
+ '+': None,
+ ' ': None,
+ '?': None
+ }
+
+ acl_different = {
+ 'a': True,
+ '.': False,
+ '+': None,
+ ' ': None,
+ '?': None
+ }
+
+ extended_attribute_different = {
+ 'x': True,
+ '.': False,
+ '+': None,
+ ' ': None,
+ '?': None
+ }
+
+ file_line_re = re.compile(r'(?P[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][u.+ ?][a.+ ?][x.+ ?]) (?P.+)')
+ file_line_mac_re = re.compile(r'(?P[<>ch.*][fdlDS][c.+ ?][s.+ ?][t.+ ?][p.+ ?][o.+ ?][g.+ ?][x.+ ?]) (?P.+)')
+ stat1_line_re = re.compile(r'(sent)\s+(?P[0-9,]+)\s+(bytes)\s+(received)\s+(?P[0-9,]+)\s+(bytes)\s+(?P[0-9,.]+)\s+(bytes/sec)')
+ stat2_line_re = re.compile(r'(total size is)\s+(?P[0-9,]+)\s+(speedup is)\s+(?P[0-9,.]+)')
+
+ file_line_log_re = re.compile(r'(?P\d\d\d\d/\d\d/\d\d)\s+(?P