diff --git a/CHANGELOG b/CHANGELOG index a5e4f957..535bbaff 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,7 +3,7 @@ jc changelog 20220202 v1.18.3 - Add rsync command and log file parser tested on linux and macOS - Add rsync command and log file streaming parser tested on linux and macOS -- Move exception handling from streaming parsers to cli module to simplify code +- Refactor ignore_exceptions functionality in streaming parsers 20220127 v1.18.2 - Fix for plugin parsers with underscores in the name diff --git a/docs/parsers/csv_s.md b/docs/parsers/csv_s.md index 9dbdac7d..3881f225 100644 --- a/docs/parsers/csv_s.md +++ b/docs/parsers/csv_s.md @@ -86,10 +86,7 @@ Parameters: raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: diff --git a/docs/parsers/iostat_s.md b/docs/parsers/iostat_s.md index 717f817c..899ad86a 100644 --- a/docs/parsers/iostat_s.md +++ b/docs/parsers/iostat_s.md @@ -123,10 +123,7 @@ Parameters: raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: diff --git a/docs/parsers/ls_s.md b/docs/parsers/ls_s.md index 3ef32e58..c79da9e4 100644 --- a/docs/parsers/ls_s.md +++ b/docs/parsers/ls_s.md @@ -100,10 +100,7 @@ Parameters: raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: diff --git a/docs/parsers/ping_s.md b/docs/parsers/ping_s.md index 143b48c1..04c279d6 100644 --- a/docs/parsers/ping_s.md +++ b/docs/parsers/ping_s.md @@ -106,10 +106,7 @@ Parameters: raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: diff --git a/docs/parsers/rsync_s.md b/docs/parsers/rsync_s.md index 2e233d7c..16a88451 100644 --- a/docs/parsers/rsync_s.md +++ b/docs/parsers/rsync_s.md @@ -109,10 +109,7 @@ Parameters: raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: diff --git a/docs/parsers/stat_s.md b/docs/parsers/stat_s.md index 5e25f2eb..4539e30d 100644 --- a/docs/parsers/stat_s.md +++ b/docs/parsers/stat_s.md @@ -104,10 +104,7 @@ Parameters: raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: diff --git a/docs/parsers/vmstat_s.md b/docs/parsers/vmstat_s.md index 47f9bac5..e13d460f 100644 --- a/docs/parsers/vmstat_s.md +++ b/docs/parsers/vmstat_s.md @@ -123,10 +123,7 @@ Parameters: raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: diff --git a/docs/utils.md b/docs/utils.md index 66307dc9..55f899b7 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -181,11 +181,10 @@ Add `_jc_meta` object to output line if `ignore_exceptions=True` ### stream\_error ```python -def stream_error(e: BaseException, ignore_exceptions: bool, line: str) -> Dict +def stream_error(e: BaseException, line: str) -> Dict ``` -Reraise the stream exception with annotation or print an error -`_jc_meta` field if `ignore_exceptions=True`. +Return an error `_jc_meta` field. @@ -205,6 +204,10 @@ With the decorator on parse(): # unsuccessfully parsed line: except Exception as e: + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + yield e, line Without the decorator on parse(): @@ -214,7 +217,11 @@ Without the decorator on parse(): # unsuccessfully parsed line: except Exception as e: - yield stream_error(e, ignore_exceptions, line) + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + + yield stream_error(e, line) In all cases above: diff --git a/jc/parsers/csv_s.py b/jc/parsers/csv_s.py index 37116255..be1267ac 100644 --- a/jc/parsers/csv_s.py +++ b/jc/parsers/csv_s.py @@ -66,7 +66,7 @@ Examples: import itertools import csv import jc.utils -from jc.utils import add_jc_meta +from jc.utils import ignore_exceptions_msg, add_jc_meta from jc.exceptions import ParseError @@ -113,10 +113,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: @@ -161,4 +158,8 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): try: yield row if raw else _process(row) except Exception as e: - yield e, row + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + + yield e, str(row) diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index 297e5dd9..44338639 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -51,7 +51,7 @@ Examples: """ from typing import Dict, Iterable, Union import jc.utils -from jc.utils import add_jc_meta +from jc.utils import ignore_exceptions_msg, add_jc_meta from jc.exceptions import ParseError @@ -108,10 +108,7 @@ def parse( raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: @@ -140,4 +137,8 @@ def parse( raise ParseError('Not foo data') except Exception as e: + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + yield e, line diff --git a/jc/parsers/iostat_s.py b/jc/parsers/iostat_s.py index eba6ca95..d5d6f00e 100644 --- a/jc/parsers/iostat_s.py +++ b/jc/parsers/iostat_s.py @@ -101,7 +101,7 @@ Examples: ... """ import jc.utils -from jc.utils import add_jc_meta +from jc.utils import ignore_exceptions_msg, add_jc_meta from jc.exceptions import ParseError import jc.parsers.universal @@ -172,10 +172,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: @@ -192,11 +189,13 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): headers = '' cpu_list = [] device_list = [] + line = '' - for line in data: - output_line = {} - try: + try: + for line in data: jc.utils.streaming_line_input_type_check(line) + output_line = {} + # ignore blank lines and header line if line == '\n' or line == '' or line.startswith('Linux'): @@ -232,5 +231,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): else: raise ParseError('Not iostat data') - except Exception as e: - yield e, line + except Exception as e: + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + + yield e, line diff --git a/jc/parsers/ls_s.py b/jc/parsers/ls_s.py index 51dc6fe9..a28c68a0 100644 --- a/jc/parsers/ls_s.py +++ b/jc/parsers/ls_s.py @@ -79,7 +79,7 @@ Examples: """ import re import jc.utils -from jc.utils import add_jc_meta +from jc.utils import ignore_exceptions_msg, add_jc_meta from jc.exceptions import ParseError @@ -135,10 +135,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: @@ -152,9 +149,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): jc.utils.streaming_input_type_check(data) parent = '' + line = '' - for line in data: - try: + try: + for line in data: jc.utils.streaming_line_input_type_check(line) # skip line if it starts with 'total 1234' @@ -202,5 +200,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): yield output_line if raw else _process(output_line) - except Exception as e: - yield e, line + except Exception as e: + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + + yield e, line diff --git a/jc/parsers/ping_s.py b/jc/parsers/ping_s.py index 12ae79b3..4779ee8f 100644 --- a/jc/parsers/ping_s.py +++ b/jc/parsers/ping_s.py @@ -87,7 +87,7 @@ import string import ipaddress import jc.utils from jc.exceptions import ParseError -from jc.utils import add_jc_meta +from jc.utils import ignore_exceptions_msg, add_jc_meta class info(): @@ -481,10 +481,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: @@ -495,13 +492,14 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Iterator object """ s = _state() + line = '' jc.utils.compatibility(__name__, info.compatible, quiet) jc.utils.streaming_input_type_check(data) - for line in data: - output_line = {} - try: + try: + for line in data: + output_line = {} jc.utils.streaming_line_input_type_check(line) # skip blank lines @@ -550,5 +548,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): else: continue - except Exception as e: - yield e, line + except Exception as e: + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + + yield e, line diff --git a/jc/parsers/rsync_s.py b/jc/parsers/rsync_s.py index 3582bbe2..625fcb08 100644 --- a/jc/parsers/rsync_s.py +++ b/jc/parsers/rsync_s.py @@ -89,7 +89,7 @@ Examples: import re from typing import Dict, Iterable, Union import jc.utils -from jc.utils import add_jc_meta +from jc.utils import ignore_exceptions_msg, add_jc_meta from jc.exceptions import ParseError @@ -157,10 +157,7 @@ def parse( raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: @@ -459,4 +456,8 @@ def parse( yield summary if raw else _process(summary) except Exception as e: + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + yield e, line diff --git a/jc/parsers/stat_s.py b/jc/parsers/stat_s.py index f57f80de..aac3b36f 100644 --- a/jc/parsers/stat_s.py +++ b/jc/parsers/stat_s.py @@ -83,7 +83,7 @@ Examples: """ import shlex import jc.utils -from jc.utils import add_jc_meta +from jc.utils import ignore_exceptions_msg, add_jc_meta from jc.exceptions import ParseError @@ -143,10 +143,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: @@ -160,10 +157,11 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): jc.utils.streaming_input_type_check(data) output_line = {} + line = '' os_type = '' - for line in data: - try: + try: + for line in data: jc.utils.streaming_line_input_type_check(line) line = line.rstrip() @@ -289,13 +287,13 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): yield output_line if raw else _process(output_line) output_line = {} - except Exception as e: - yield e, line - output_line = {} - - # gather final item - if output_line: - try: + # gather final item + if output_line: yield output_line if raw else _process(output_line) - except Exception as e: - yield e, line + + except Exception as e: + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + + yield e, line diff --git a/jc/parsers/vmstat_s.py b/jc/parsers/vmstat_s.py index 5d85cf11..be9a7e29 100644 --- a/jc/parsers/vmstat_s.py +++ b/jc/parsers/vmstat_s.py @@ -101,7 +101,7 @@ Examples: ... """ import jc.utils -from jc.utils import add_jc_meta +from jc.utils import ignore_exceptions_msg, add_jc_meta from jc.exceptions import ParseError @@ -162,10 +162,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True. - This can be used directly or - (preferably) by being passed to the - @add_jc_meta decorator. + ignore_exceptions: (boolean) ignore parsing exceptions if True Yields: @@ -275,4 +272,8 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raise ParseError('Not vmstat data') except Exception as e: + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + yield e, line diff --git a/jc/utils.py b/jc/utils.py index 6bdc6a50..fe05a5a2 100644 --- a/jc/utils.py +++ b/jc/utils.py @@ -230,15 +230,13 @@ def stream_success(output_line: Dict, ignore_exceptions: bool) -> Dict: return output_line -def stream_error(e: BaseException, ignore_exceptions: bool, line: str) -> Dict: - """ - Reraise the stream exception with annotation or print an error - `_jc_meta` field if `ignore_exceptions=True`. - """ - if not ignore_exceptions: - e.args = (str(e) + '... Use the ignore_exceptions option (-qq) to ignore streaming parser errors.',) - raise e +ignore_exceptions_msg = '... Use the ignore_exceptions option (-qq) to ignore streaming parser errors.' + +def stream_error(e: BaseException, line: str) -> Dict: + """ + Return an error `_jc_meta` field. + """ return { '_jc_meta': { @@ -261,6 +259,10 @@ def add_jc_meta(func): # unsuccessfully parsed line: except Exception as e: + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + yield e, line Without the decorator on parse(): @@ -270,7 +272,11 @@ def add_jc_meta(func): # unsuccessfully parsed line: except Exception as e: - yield stream_error(e, ignore_exceptions, line) + if not ignore_exceptions: + e.args = (str(e) + ignore_exceptions_msg,) + raise e + + yield stream_error(e, line) In all cases above: @@ -296,7 +302,7 @@ def add_jc_meta(func): else: exception_obj = value[0] line = value[1] - yield stream_error(exception_obj, ignore_exceptions, line) + yield stream_error(exception_obj, line) return wrapper diff --git a/man/jc.1 b/man/jc.1 index e3f87ee9..7a5778db 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-02-02 1.18.3 "JSON CLI output utility" +.TH jc 1 2022-02-04 1.18.3 "JSON CLI output utility" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS