refactor ignore_exceptions

CHANGELOG

@@ -3,7 +3,7 @@ jc changelog
 20220202 v1.18.3
 - Add rsync command and log file parser tested on linux and macOS
 - Add rsync command and log file streaming parser tested on linux and macOS
-- Move exception handling from streaming parsers to cli module to simplify code
+- Refactor ignore_exceptions functionality in streaming parsers
 
 20220127 v1.18.2
 - Fix for plugin parsers with underscores in the name

docs/parsers/csv_s.md

@@ -86,10 +86,7 @@ Parameters:
 
     raw:               (boolean)   unprocessed output if True
     quiet:             (boolean)   suppress warning messages if True
-    ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+    ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                   This can be used directly or
-                                   (preferably) by being passed to the
-                                   @add_jc_meta decorator.
 
 Yields:
 

docs/parsers/iostat_s.md

@@ -123,10 +123,7 @@ Parameters:
 
     raw:               (boolean)   unprocessed output if True
     quiet:             (boolean)   suppress warning messages if True
-    ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+    ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                   This can be used directly or
-                                   (preferably) by being passed to the
-                                   @add_jc_meta decorator.
 
 Yields:
 

docs/parsers/ls_s.md

@@ -100,10 +100,7 @@ Parameters:
 
     raw:               (boolean)   unprocessed output if True
     quiet:             (boolean)   suppress warning messages if True
-    ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+    ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                   This can be used directly or
-                                   (preferably) by being passed to the
-                                   @add_jc_meta decorator.
 
 Yields:
 

docs/parsers/ping_s.md

@@ -106,10 +106,7 @@ Parameters:
 
     raw:               (boolean)   unprocessed output if True
     quiet:             (boolean)   suppress warning messages if True
-    ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+    ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                   This can be used directly or
-                                   (preferably) by being passed to the
-                                   @add_jc_meta decorator.
 
 Yields:
 

docs/parsers/rsync_s.md

@@ -109,10 +109,7 @@ Parameters:
 
     raw:               (boolean)   unprocessed output if True
     quiet:             (boolean)   suppress warning messages if True
-    ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+    ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                   This can be used directly or
-                                   (preferably) by being passed to the
-                                   @add_jc_meta decorator.
 
 Yields:
 

docs/parsers/stat_s.md

@@ -104,10 +104,7 @@ Parameters:
 
     raw:               (boolean)   unprocessed output if True
     quiet:             (boolean)   suppress warning messages if True
-    ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+    ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                   This can be used directly or
-                                   (preferably) by being passed to the
-                                   @add_jc_meta decorator.
 
 Yields:
 

docs/parsers/vmstat_s.md

@@ -123,10 +123,7 @@ Parameters:
 
     raw:               (boolean)   unprocessed output if True
     quiet:             (boolean)   suppress warning messages if True
-    ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+    ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                   This can be used directly or
-                                   (preferably) by being passed to the
-                                   @add_jc_meta decorator.
 
 Yields:
 

docs/utils.md

@@ -181,11 +181,10 @@ Add `_jc_meta` object to output line if `ignore_exceptions=True`
 ### stream\_error
 
 ```python
-def stream_error(e: BaseException, ignore_exceptions: bool, line: str) -> Dict
+def stream_error(e: BaseException, line: str) -> Dict
 ```
 
-Reraise the stream exception with annotation or print an error
+Return an error `_jc_meta` field.
-`_jc_meta` field if `ignore_exceptions=True`.
 
 <a id="jc.utils.add_jc_meta"></a>
 
@@ -205,6 +204,10 @@ With the decorator on parse():
 
     # unsuccessfully parsed line:
     except Exception as e:
+        if not ignore_exceptions:
+            e.args = (str(e) + ignore_exceptions_msg,)
+            raise e
+
         yield e, line
 
 Without the decorator on parse():
@@ -214,7 +217,11 @@ Without the decorator on parse():
 
     # unsuccessfully parsed line:
     except Exception as e:
-        yield stream_error(e, ignore_exceptions, line)
+        if not ignore_exceptions:
+            e.args = (str(e) + ignore_exceptions_msg,)
+            raise e
+
+        yield stream_error(e, line)
 
 In all cases above:
 

jc/parsers/csv_s.py

@@ -66,7 +66,7 @@ Examples:
 import itertools
 import csv
 import jc.utils
-from jc.utils import add_jc_meta
+from jc.utils import ignore_exceptions_msg, add_jc_meta
 from jc.exceptions import ParseError
 
 
@@ -113,10 +113,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
 
         raw:               (boolean)   unprocessed output if True
         quiet:             (boolean)   suppress warning messages if True
-        ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+        ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                       This can be used directly or
-                                       (preferably) by being passed to the
-                                       @add_jc_meta decorator.
 
     Yields:
 
@@ -161,4 +158,8 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
         try:
             yield row if raw else _process(row)
         except Exception as e:
-            yield e, row
+            if not ignore_exceptions:
+                e.args = (str(e) + ignore_exceptions_msg,)
+                raise e
+
+            yield e, str(row)

jc/parsers/foo_s.py

@@ -51,7 +51,7 @@ Examples:
 """
 from typing import Dict, Iterable, Union
 import jc.utils
-from jc.utils import add_jc_meta
+from jc.utils import ignore_exceptions_msg, add_jc_meta
 from jc.exceptions import ParseError
 
 
@@ -108,10 +108,7 @@ def parse(
 
         raw:               (boolean)   unprocessed output if True
         quiet:             (boolean)   suppress warning messages if True
-        ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+        ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                       This can be used directly or
-                                       (preferably) by being passed to the
-                                       @add_jc_meta decorator.
 
     Yields:
 
@@ -140,4 +137,8 @@ def parse(
                 raise ParseError('Not foo data')
 
     except Exception as e:
+        if not ignore_exceptions:
+            e.args = (str(e) + ignore_exceptions_msg,)
+            raise e
+
         yield e, line

jc/parsers/iostat_s.py

@@ -101,7 +101,7 @@ Examples:
     ...
 """
 import jc.utils
-from jc.utils import add_jc_meta
+from jc.utils import ignore_exceptions_msg, add_jc_meta
 from jc.exceptions import ParseError
 import jc.parsers.universal
 
@@ -172,10 +172,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
 
         raw:               (boolean)   unprocessed output if True
         quiet:             (boolean)   suppress warning messages if True
-        ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+        ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                       This can be used directly or
-                                       (preferably) by being passed to the
-                                       @add_jc_meta decorator.
 
     Yields:
 
@@ -192,11 +189,13 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
     headers = ''
     cpu_list = []
     device_list = []
+    line = ''
 
-    for line in data:
+    try:
-        output_line = {}
+        for line in data:
-        try:
             jc.utils.streaming_line_input_type_check(line)
+            output_line = {}
+
 
             # ignore blank lines and header line
             if line == '\n' or line == '' or line.startswith('Linux'):
@@ -232,5 +231,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
             else:
                 raise ParseError('Not iostat data')
 
-        except Exception as e:
+    except Exception as e:
-            yield e, line
+        if not ignore_exceptions:
+            e.args = (str(e) + ignore_exceptions_msg,)
+            raise e
+
+        yield e, line

jc/parsers/ls_s.py

@@ -79,7 +79,7 @@ Examples:
 """
 import re
 import jc.utils
-from jc.utils import add_jc_meta
+from jc.utils import ignore_exceptions_msg, add_jc_meta
 from jc.exceptions import ParseError
 
 
@@ -135,10 +135,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
 
         raw:               (boolean)   unprocessed output if True
         quiet:             (boolean)   suppress warning messages if True
-        ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+        ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                       This can be used directly or
-                                       (preferably) by being passed to the
-                                       @add_jc_meta decorator.
 
     Yields:
 
@@ -152,9 +149,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
     jc.utils.streaming_input_type_check(data)
 
     parent = ''
+    line = ''
 
-    for line in data:
+    try:
-        try:
+        for line in data:
             jc.utils.streaming_line_input_type_check(line)
 
             # skip line if it starts with 'total 1234'
@@ -202,5 +200,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
 
             yield output_line if raw else _process(output_line)
 
-        except Exception as e:
+    except Exception as e:
-            yield e, line
+        if not ignore_exceptions:
+            e.args = (str(e) + ignore_exceptions_msg,)
+            raise e
+
+        yield e, line

jc/parsers/ping_s.py

@@ -87,7 +87,7 @@ import string
 import ipaddress
 import jc.utils
 from jc.exceptions import ParseError
-from jc.utils import add_jc_meta
+from jc.utils import ignore_exceptions_msg, add_jc_meta
 
 
 class info():
@@ -481,10 +481,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
 
         raw:               (boolean)   unprocessed output if True
         quiet:             (boolean)   suppress warning messages if True
-        ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+        ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                       This can be used directly or
-                                       (preferably) by being passed to the
-                                       @add_jc_meta decorator.
 
     Yields:
 
@@ -495,13 +492,14 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
         Iterator object
     """
     s = _state()
+    line = ''
 
     jc.utils.compatibility(__name__, info.compatible, quiet)
     jc.utils.streaming_input_type_check(data)
 
-    for line in data:
+    try:
-        output_line = {}
+        for line in data:
-        try:
+            output_line = {}
             jc.utils.streaming_line_input_type_check(line)
 
             # skip blank lines
@@ -550,5 +548,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
             else:
                 continue
 
-        except Exception as e:
+    except Exception as e:
-            yield e, line
+        if not ignore_exceptions:
+            e.args = (str(e) + ignore_exceptions_msg,)
+            raise e
+
+        yield e, line

jc/parsers/rsync_s.py

@@ -89,7 +89,7 @@ Examples:
 import re
 from typing import Dict, Iterable, Union
 import jc.utils
-from jc.utils import add_jc_meta
+from jc.utils import ignore_exceptions_msg, add_jc_meta
 from jc.exceptions import ParseError
 
 
@@ -157,10 +157,7 @@ def parse(
 
         raw:               (boolean)   unprocessed output if True
         quiet:             (boolean)   suppress warning messages if True
-        ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+        ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                       This can be used directly or
-                                       (preferably) by being passed to the
-                                       @add_jc_meta decorator.
 
     Yields:
 
@@ -459,4 +456,8 @@ def parse(
             yield summary if raw else _process(summary)
 
     except Exception as e:
+        if not ignore_exceptions:
+            e.args = (str(e) + ignore_exceptions_msg,)
+            raise e
+
         yield e, line

jc/parsers/stat_s.py

@@ -83,7 +83,7 @@ Examples:
 """
 import shlex
 import jc.utils
-from jc.utils import add_jc_meta
+from jc.utils import ignore_exceptions_msg, add_jc_meta
 from jc.exceptions import ParseError
 
 
@@ -143,10 +143,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
 
         raw:               (boolean)   unprocessed output if True
         quiet:             (boolean)   suppress warning messages if True
-        ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+        ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                       This can be used directly or
-                                       (preferably) by being passed to the
-                                       @add_jc_meta decorator.
 
     Yields:
 
@@ -160,10 +157,11 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
     jc.utils.streaming_input_type_check(data)
 
     output_line = {}
+    line = ''
     os_type = ''
 
-    for line in data:
+    try:
-        try:
+        for line in data:
             jc.utils.streaming_line_input_type_check(line)
             line = line.rstrip()
 
@@ -289,13 +287,13 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
                     yield output_line if raw else _process(output_line)
                     output_line = {}
 
-        except Exception as e:
+        # gather final item
-            yield e, line
+        if output_line:
-            output_line = {}
-
-    # gather final item
-    if output_line:
-        try:
             yield output_line if raw else _process(output_line)
-        except Exception as e:
+
-            yield e, line
+    except Exception as e:
+        if not ignore_exceptions:
+            e.args = (str(e) + ignore_exceptions_msg,)
+            raise e
+
+        yield e, line

jc/parsers/vmstat_s.py

@@ -101,7 +101,7 @@ Examples:
     ...
 """
 import jc.utils
-from jc.utils import add_jc_meta
+from jc.utils import ignore_exceptions_msg, add_jc_meta
 from jc.exceptions import ParseError
 
 
@@ -162,10 +162,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
 
         raw:               (boolean)   unprocessed output if True
         quiet:             (boolean)   suppress warning messages if True
-        ignore_exceptions: (boolean)   ignore parsing exceptions if True.
+        ignore_exceptions: (boolean)   ignore parsing exceptions if True
-                                       This can be used directly or
-                                       (preferably) by being passed to the
-                                       @add_jc_meta decorator.
 
     Yields:
 
@@ -275,4 +272,8 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
                 raise ParseError('Not vmstat data')
 
         except Exception as e:
+            if not ignore_exceptions:
+                e.args = (str(e) + ignore_exceptions_msg,)
+                raise e
+
             yield e, line

jc/utils.py

@@ -230,15 +230,13 @@ def stream_success(output_line: Dict, ignore_exceptions: bool) -> Dict:
     return output_line
 
 
-def stream_error(e: BaseException, ignore_exceptions: bool, line: str) -> Dict:
+ignore_exceptions_msg = '... Use the ignore_exceptions option (-qq) to ignore streaming parser errors.'
-    """
-    Reraise the stream exception with annotation or print an error
-    `_jc_meta` field if `ignore_exceptions=True`.
-    """
-    if not ignore_exceptions:
-        e.args = (str(e) + '... Use the ignore_exceptions option (-qq) to ignore streaming parser errors.',)
-        raise e
 
+
+def stream_error(e: BaseException, line: str) -> Dict:
+    """
+    Return an error `_jc_meta` field.
+    """
     return {
         '_jc_meta':
             {
@@ -261,6 +259,10 @@ def add_jc_meta(func):
 
         # unsuccessfully parsed line:
         except Exception as e:
+            if not ignore_exceptions:
+                e.args = (str(e) + ignore_exceptions_msg,)
+                raise e
+
             yield e, line
 
     Without the decorator on parse():
@@ -270,7 +272,11 @@ def add_jc_meta(func):
 
         # unsuccessfully parsed line:
         except Exception as e:
-            yield stream_error(e, ignore_exceptions, line)
+            if not ignore_exceptions:
+                e.args = (str(e) + ignore_exceptions_msg,)
+                raise e
+
+            yield stream_error(e, line)
 
     In all cases above:
 
@@ -296,7 +302,7 @@ def add_jc_meta(func):
             else:
                 exception_obj = value[0]
                 line = value[1]
-                yield stream_error(exception_obj, ignore_exceptions, line)
+                yield stream_error(exception_obj, line)
 
     return wrapper
 

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2022-02-02 1.18.3 "JSON CLI output utility"
+.TH jc 1 2022-02-04 1.18.3 "JSON CLI output utility"
 .SH NAME
 jc \- JSONifies the output of many CLI tools and file-types
 .SH SYNOPSIS