diff --git a/docgen.sh b/docgen.sh index 13cd6fec..38495775 100755 --- a/docgen.sh +++ b/docgen.sh @@ -85,6 +85,9 @@ pydoc-markdown -m jc.lib "${toc_config}" > ../docs/lib.md echo Building docs for: utils pydoc-markdown -m jc.utils "${toc_config}" > ../docs/utils.md +echo Building docs for: streaming +pydoc-markdown -m jc.streaming "${toc_config}" > ../docs/streaming.md + echo Building docs for: universal parser pydoc-markdown -m jc.parsers.universal "${toc_config}" > ../docs/parsers/universal.md diff --git a/docs/parsers/rsync_s.md b/docs/parsers/rsync_s.md index 16a88451..7f0266a8 100644 --- a/docs/parsers/rsync_s.md +++ b/docs/parsers/rsync_s.md @@ -122,4 +122,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, freebsd -Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/readme.md b/docs/readme.md index 073af8bc..7410dd4f 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -14,6 +14,8 @@ and file-types to dictionaries and lists of dictionaries. >>> help('jc') >>> help('jc.lib') >>> help('jc.utils') + >>> help('jc.streaming') + >>> help('jc.parsers.universal') >>> jc.get_help('parser_module_name') ## Online Documentation diff --git a/docs/streaming.md b/docs/streaming.md new file mode 100644 index 00000000..b8971010 --- /dev/null +++ b/docs/streaming.md @@ -0,0 +1,114 @@ +# Table of Contents + +* [jc.streaming](#jc.streaming) + * [streaming\_input\_type\_check](#jc.streaming.streaming_input_type_check) + * [streaming\_line\_input\_type\_check](#jc.streaming.streaming_line_input_type_check) + * [stream\_success](#jc.streaming.stream_success) + * [stream\_error](#jc.streaming.stream_error) + * [add\_jc\_meta](#jc.streaming.add_jc_meta) + * [raise\_or\_yield](#jc.streaming.raise_or_yield) + + + +# jc.streaming + +jc - JSON CLI output utility streaming utils + + + +### streaming\_input\_type\_check + +```python +def streaming_input_type_check(data: Iterable) -> None +``` + +Ensure input data is an iterable, but not a string or bytes. Raises +`TypeError` if not. + + + +### streaming\_line\_input\_type\_check + +```python +def streaming_line_input_type_check(line: str) -> None +``` + +Ensure each line is a string. Raises `TypeError` if not. + + + +### stream\_success + +```python +def stream_success(output_line: Dict, ignore_exceptions: bool) -> Dict +``` + +Add `_jc_meta` object to output line if `ignore_exceptions=True` + + + +### stream\_error + +```python +def stream_error(e: BaseException, line: str) -> Dict +``` + +Return an error `_jc_meta` field. + + + +### add\_jc\_meta + +```python +def add_jc_meta(func) +``` + +Decorator for streaming parsers to add stream_success and stream_error +objects. This simplifies the yield lines in the streaming parsers. + +With the decorator on parse(): + + # successfully parsed line: + yield output_line if raw else _process(output_line) + + # unsuccessfully parsed line: + except Exception as e: + yield raise_or_yield(ignore_exceptions, e, line) + +Without the decorator on parse(): + + # successfully parsed line: + if raw: + yield stream_success(output_line, ignore_exceptions) + else: + stream_success(_process(output_line), ignore_exceptions) + + # unsuccessfully parsed line: + except Exception as e: + yield stream_error(raise_or_yield(ignore_exceptions, e, line)) + +In all cases above: + + output_line: (Dict) successfully parsed line yielded as a dict + + e: (BaseException) exception object as the first value + of the tuple if the line was not successfully parsed. + + line: (str) string of the original line that did not + successfully parse. + + ignore_exceptions: (bool) continue processing lines and ignore + exceptions if True. + + + +### raise\_or\_yield + +```python +def raise_or_yield(ignore_exceptions: bool, e: BaseException, line: str) -> tuple +``` + +Return the exception object and line string if ignore_exceptions is +True. Otherwise, re-raise the exception from the exception object with +an annotation. + diff --git a/docs/utils.md b/docs/utils.md index 55f899b7..12d6b4cb 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -8,12 +8,7 @@ * [convert\_to\_int](#jc.utils.convert_to_int) * [convert\_to\_float](#jc.utils.convert_to_float) * [convert\_to\_bool](#jc.utils.convert_to_bool) - * [stream\_success](#jc.utils.stream_success) - * [stream\_error](#jc.utils.stream_error) - * [add\_jc\_meta](#jc.utils.add_jc_meta) * [input\_type\_check](#jc.utils.input_type_check) - * [streaming\_input\_type\_check](#jc.utils.streaming_input_type_check) - * [streaming\_line\_input\_type\_check](#jc.utils.streaming_line_input_type_check) * [timestamp](#jc.utils.timestamp) * [\_\_init\_\_](#jc.utils.timestamp.__init__) @@ -166,73 +161,6 @@ Returns: True/False False unless a 'truthy' number or string is found ('y', 'yes', 'true', '1', 1, -1, etc.) - - -### stream\_success - -```python -def stream_success(output_line: Dict, ignore_exceptions: bool) -> Dict -``` - -Add `_jc_meta` object to output line if `ignore_exceptions=True` - - - -### stream\_error - -```python -def stream_error(e: BaseException, line: str) -> Dict -``` - -Return an error `_jc_meta` field. - - - -### add\_jc\_meta - -```python -def add_jc_meta(func) -``` - -Decorator for streaming parsers to add stream_success and stream_error -objects. This simplifies the yield lines in the streaming parsers. - -With the decorator on parse(): - - # successfully parsed line: - yield output_line if raw else _process(output_line) - - # unsuccessfully parsed line: - except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield e, line - -Without the decorator on parse(): - - # successfully parsed line: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) - - # unsuccessfully parsed line: - except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield stream_error(e, line) - -In all cases above: - - output_line: (Dict): successfully parsed line yielded as a dict - - e: (BaseException): exception object as the first value - of the tuple if the line was not successfully parsed. - - line: (str): string of the original line that did not - successfully parse. - ### input\_type\_check @@ -243,27 +171,6 @@ def input_type_check(data: str) -> None Ensure input data is a string. Raises `TypeError` if not. - - -### streaming\_input\_type\_check - -```python -def streaming_input_type_check(data: Iterable) -> None -``` - -Ensure input data is an iterable, but not a string or bytes. Raises -`TypeError` if not. - - - -### streaming\_line\_input\_type\_check - -```python -def streaming_line_input_type_check(line: str) -> None -``` - -Ensure each line is a string. Raises `TypeError` if not. - ### timestamp Objects diff --git a/jc/__init__.py b/jc/__init__.py index d5a02709..a59ba5b2 100644 --- a/jc/__init__.py +++ b/jc/__init__.py @@ -10,6 +10,8 @@ and file-types to dictionaries and lists of dictionaries. >>> help('jc') >>> help('jc.lib') >>> help('jc.utils') + >>> help('jc.streaming') + >>> help('jc.parsers.universal') >>> jc.get_help('parser_module_name') ## Online Documentation diff --git a/jc/parsers/csv_s.py b/jc/parsers/csv_s.py index be1267ac..215688b2 100644 --- a/jc/parsers/csv_s.py +++ b/jc/parsers/csv_s.py @@ -66,7 +66,7 @@ Examples: import itertools import csv import jc.utils -from jc.utils import ignore_exceptions_msg, add_jc_meta +from jc.streaming import streaming_input_type_check, add_jc_meta, raise_or_yield from jc.exceptions import ParseError @@ -124,7 +124,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Iterator object """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) # convert data to an iterable in case a sequence like a list is used as input. # this allows the exhaustion of the input so we don't double-process later. @@ -158,8 +158,4 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): try: yield row if raw else _process(row) except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield e, str(row) + yield raise_or_yield(ignore_exceptions, e, str(row)) diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index 80692a70..f5e33023 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -51,7 +51,9 @@ Examples: """ from typing import Dict, Iterable, Union import jc.utils -from jc.utils import ignore_exceptions_msg, add_jc_meta +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) from jc.exceptions import ParseError @@ -119,12 +121,12 @@ def parse( Iterator object """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) for line in data: try: + streaming_line_input_type_check(line) output_line: Dict = {} - jc.utils.streaming_line_input_type_check(line) # parse the content here # check out helper functions in jc.utils @@ -136,8 +138,4 @@ def parse( raise ParseError('Not foo data') except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield e, line + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/jc/parsers/iostat_s.py b/jc/parsers/iostat_s.py index e0097d01..57a6b75c 100644 --- a/jc/parsers/iostat_s.py +++ b/jc/parsers/iostat_s.py @@ -101,7 +101,9 @@ Examples: ... """ import jc.utils -from jc.utils import ignore_exceptions_msg, add_jc_meta +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) from jc.exceptions import ParseError import jc.parsers.universal @@ -183,7 +185,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Iterator object """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) section = '' # either 'cpu' or 'device' headers = '' @@ -192,10 +194,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): for line in data: try: - jc.utils.streaming_line_input_type_check(line) + streaming_line_input_type_check(line) output_line = {} - # ignore blank lines and header line if line == '\n' or line == '' or line.startswith('Linux'): continue @@ -231,8 +232,4 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raise ParseError('Not iostat data') except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield e, line + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/jc/parsers/ls_s.py b/jc/parsers/ls_s.py index a8b17a9c..55bb6154 100644 --- a/jc/parsers/ls_s.py +++ b/jc/parsers/ls_s.py @@ -79,7 +79,9 @@ Examples: """ import re import jc.utils -from jc.utils import ignore_exceptions_msg, add_jc_meta +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) from jc.exceptions import ParseError @@ -146,13 +148,13 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Iterator object """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) parent = '' for line in data: try: - jc.utils.streaming_line_input_type_check(line) + streaming_line_input_type_check(line) # skip line if it starts with 'total 1234' if re.match(r'total [0-9]+', line): @@ -200,8 +202,4 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): yield output_line if raw else _process(output_line) except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield e, line + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/jc/parsers/ping_s.py b/jc/parsers/ping_s.py index 3478a5b2..c2bc1d36 100644 --- a/jc/parsers/ping_s.py +++ b/jc/parsers/ping_s.py @@ -86,10 +86,10 @@ Examples: import string import ipaddress import jc.utils -from jc.exceptions import ParseError from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield ) +from jc.exceptions import ParseError class info(): @@ -500,8 +500,8 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): for line in data: try: - output_line = {} streaming_line_input_type_check(line) + output_line = {} # skip blank lines if line.strip() == '': diff --git a/jc/parsers/rsync_s.py b/jc/parsers/rsync_s.py index 560b34cf..3583ccd3 100644 --- a/jc/parsers/rsync_s.py +++ b/jc/parsers/rsync_s.py @@ -89,13 +89,13 @@ Examples: import re from typing import Dict, Iterable, Union import jc.utils -from jc.utils import ignore_exceptions_msg, add_jc_meta -from jc.exceptions import ParseError - +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.1' + version = '1.0' description = '`rsync` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -168,7 +168,7 @@ def parse( Iterator object """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) summary: Dict = {} process: str = '' @@ -272,7 +272,7 @@ def parse( for line in data: try: - jc.utils.streaming_line_input_type_check(line) + streaming_line_input_type_check(line) output_line: Dict = {} # ignore blank lines @@ -452,19 +452,12 @@ def parse( continue except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield e, line + yield raise_or_yield(ignore_exceptions, e, line) + # gather final item try: if summary: yield summary if raw else _process(summary) except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield e, line + yield raise_or_yield(ignore_exceptions, e, '') diff --git a/jc/parsers/stat_s.py b/jc/parsers/stat_s.py index 86cddfd5..2632f24c 100644 --- a/jc/parsers/stat_s.py +++ b/jc/parsers/stat_s.py @@ -83,7 +83,9 @@ Examples: """ import shlex import jc.utils -from jc.utils import ignore_exceptions_msg, add_jc_meta +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) from jc.exceptions import ParseError @@ -154,14 +156,14 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Iterator object """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) output_line = {} os_type = '' for line in data: try: - jc.utils.streaming_line_input_type_check(line) + streaming_line_input_type_check(line) line = line.rstrip() # ignore blank lines @@ -287,20 +289,12 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): output_line = {} except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield e, line + yield raise_or_yield(ignore_exceptions, e, line) + # gather final item try: - # gather final item if output_line: yield output_line if raw else _process(output_line) except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield e, line + yield raise_or_yield(ignore_exceptions, e, '') diff --git a/jc/parsers/vmstat_s.py b/jc/parsers/vmstat_s.py index be9a7e29..a2e7871f 100644 --- a/jc/parsers/vmstat_s.py +++ b/jc/parsers/vmstat_s.py @@ -101,7 +101,9 @@ Examples: ... """ import jc.utils -from jc.utils import ignore_exceptions_msg, add_jc_meta +from jc.streaming import ( + add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield +) from jc.exceptions import ParseError @@ -173,7 +175,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Iterator object """ jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.streaming_input_type_check(data) + streaming_input_type_check(data) procs = None buff_cache = None @@ -182,9 +184,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): tz = None for line in data: - output_line = {} try: - jc.utils.streaming_line_input_type_check(line) + streaming_line_input_type_check(line) + output_line = {} # skip blank lines if line.strip() == '': @@ -272,8 +274,4 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raise ParseError('Not vmstat data') except Exception as e: - if not ignore_exceptions: - e.args = (str(e) + ignore_exceptions_msg,) - raise e - - yield e, line + yield raise_or_yield(ignore_exceptions, e, line) diff --git a/jc/streaming.py b/jc/streaming.py index 9ec8ffc7..0e6dbc80 100644 --- a/jc/streaming.py +++ b/jc/streaming.py @@ -4,6 +4,21 @@ from functools import wraps from typing import Dict, Iterable +def streaming_input_type_check(data: Iterable) -> None: + """ + Ensure input data is an iterable, but not a string or bytes. Raises + `TypeError` if not. + """ + if not hasattr(data, '__iter__') or isinstance(data, (str, bytes)): + raise TypeError("Input data must be a non-string iterable object.") + + +def streaming_line_input_type_check(line: str) -> None: + """Ensure each line is a string. Raises `TypeError` if not.""" + if not isinstance(line, str): + raise TypeError("Input line must be a 'str' object.") + + def stream_success(output_line: Dict, ignore_exceptions: bool) -> Dict: """Add `_jc_meta` object to output line if `ignore_exceptions=True`""" if ignore_exceptions: @@ -43,8 +58,10 @@ def add_jc_meta(func): Without the decorator on parse(): # successfully parsed line: - yield stream_success(output_line, ignore_exceptions) if raw \\ - else stream_success(_process(output_line), ignore_exceptions) + if raw: + yield stream_success(output_line, ignore_exceptions) + else: + stream_success(_process(output_line), ignore_exceptions) # unsuccessfully parsed line: except Exception as e: @@ -82,26 +99,16 @@ def add_jc_meta(func): return wrapper -def streaming_input_type_check(data: Iterable) -> None: - """ - Ensure input data is an iterable, but not a string or bytes. Raises - `TypeError` if not. - """ - if not hasattr(data, '__iter__') or isinstance(data, (str, bytes)): - raise TypeError("Input data must be a non-string iterable object.") - - -def streaming_line_input_type_check(line: str) -> None: - """Ensure each line is a string. Raises `TypeError` if not.""" - if not isinstance(line, str): - raise TypeError("Input line must be a 'str' object.") - - def raise_or_yield( ignore_exceptions: bool, e: BaseException, line: str ) -> tuple: + """ + Return the exception object and line string if ignore_exceptions is + True. Otherwise, re-raise the exception from the exception object with + an annotation. + """ ignore_exceptions_msg = '... Use the ignore_exceptions option (-qq) to ignore streaming parser errors.' if not ignore_exceptions: diff --git a/man/jc.1 b/man/jc.1 index 7a5778db..089dc5b7 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-02-04 1.18.3 "JSON CLI output utility" +.TH jc 1 2022-02-07 1.18.3 "JSON CLI output utility" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS