diff --git a/jc/parsers/csv_s.py b/jc/parsers/csv_s.py index c692dde8..37116255 100644 --- a/jc/parsers/csv_s.py +++ b/jc/parsers/csv_s.py @@ -66,13 +66,13 @@ Examples: import itertools import csv import jc.utils -from jc.utils import stream_success, stream_error +from jc.utils import add_jc_meta from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.2' + version = '1.3' description = 'CSV file streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -101,6 +101,7 @@ def _process(proc_data): return proc_data +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False): """ Main text parsing generator function. Returns an iterator object. @@ -112,7 +113,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True + ignore_exceptions: (boolean) ignore parsing exceptions if True. + This can be used directly or + (preferably) by being passed to the + @add_jc_meta decorator. Yields: @@ -155,6 +159,6 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): for row in reader: try: - yield stream_success(row, ignore_exceptions) if raw else stream_success(_process(row), ignore_exceptions) + yield row if raw else _process(row) except Exception as e: - yield stream_error(e, ignore_exceptions, row) + yield e, row diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index de5194d9..297e5dd9 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -49,9 +49,9 @@ Examples: {example output} ... """ -from typing import Dict, Iterable +from typing import Dict, Iterable, Union import jc.utils -from jc.utils import stream_success, stream_error +from jc.utils import add_jc_meta from jc.exceptions import ParseError @@ -91,12 +91,13 @@ def _process(proc_data: Dict) -> Dict: return proc_data +@add_jc_meta def parse( data: Iterable[str], raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Iterable[Dict]: +) -> Union[Iterable[Dict], tuple]: """ Main text parsing generator function. Returns an iterator object. @@ -107,7 +108,10 @@ def parse( raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True + ignore_exceptions: (boolean) ignore parsing exceptions if True. + This can be used directly or + (preferably) by being passed to the + @add_jc_meta decorator. Yields: @@ -131,9 +135,9 @@ def parse( # and jc.parsers.universal if output_line: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) else: raise ParseError('Not foo data') except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield e, line diff --git a/jc/parsers/iostat_s.py b/jc/parsers/iostat_s.py index d6ac5b4b..eba6ca95 100644 --- a/jc/parsers/iostat_s.py +++ b/jc/parsers/iostat_s.py @@ -101,14 +101,14 @@ Examples: ... """ import jc.utils -from jc.utils import stream_success, stream_error +from jc.utils import add_jc_meta from jc.exceptions import ParseError import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`iostat` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -159,6 +159,8 @@ def _create_obj_list(section_list, section_name): item['type'] = section_name return output_list + +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False): """ Main text parsing generator function. Returns an iterator object. @@ -170,7 +172,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True + ignore_exceptions: (boolean) ignore parsing exceptions if True. + This can be used directly or + (preferably) by being passed to the + @add_jc_meta decorator. Yields: @@ -223,9 +228,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): device_list = [] if output_line: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) else: raise ParseError('Not iostat data') except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield e, line diff --git a/jc/parsers/ls_s.py b/jc/parsers/ls_s.py index 7de75ce9..51dc6fe9 100644 --- a/jc/parsers/ls_s.py +++ b/jc/parsers/ls_s.py @@ -79,13 +79,13 @@ Examples: """ import re import jc.utils -from jc.utils import stream_success, stream_error +from jc.utils import add_jc_meta from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '0.6' + version = '1.0' description = '`ls` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -123,6 +123,7 @@ def _process(proc_data): return proc_data +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False): """ Main text parsing generator function. Returns an iterator object. @@ -134,7 +135,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True + ignore_exceptions: (boolean) ignore parsing exceptions if True. + This can be used directly or + (preferably) by being passed to the + @add_jc_meta decorator. Yields: @@ -196,7 +200,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): output_line['size'] = parsed_line[4] output_line['date'] = ' '.join(parsed_line[5:8]) - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield e, line diff --git a/jc/parsers/ping_s.py b/jc/parsers/ping_s.py index 8536c3f5..12ae79b3 100644 --- a/jc/parsers/ping_s.py +++ b/jc/parsers/ping_s.py @@ -87,12 +87,12 @@ import string import ipaddress import jc.utils from jc.exceptions import ParseError -from jc.utils import stream_success, stream_error +from jc.utils import add_jc_meta class info(): """Provides parser metadata (version, author, etc.)""" - version = '0.6' + version = '1.0' description = '`ping` and `ping6` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -469,6 +469,7 @@ def _linux_parse(line, s): return output_line +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False): """ Main text parsing generator function. Returns an iterator object. @@ -480,7 +481,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True + ignore_exceptions: (boolean) ignore parsing exceptions if True. + This can be used directly or + (preferably) by being passed to the + @add_jc_meta decorator. Yields: @@ -542,9 +546,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): # yield the output line if it has data if output_line: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) else: continue except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield e, line diff --git a/jc/parsers/rsync_s.py b/jc/parsers/rsync_s.py index 11ef6395..3582bbe2 100644 --- a/jc/parsers/rsync_s.py +++ b/jc/parsers/rsync_s.py @@ -87,15 +87,15 @@ Examples: ... """ import re -from typing import Dict, Iterable +from typing import Dict, Iterable, Union import jc.utils -from jc.utils import stream_success, stream_error +from jc.utils import add_jc_meta from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`rsync` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -140,12 +140,13 @@ def _process(proc_data: Dict) -> Dict: return proc_data +@add_jc_meta def parse( data: Iterable[str], raw: bool = False, quiet: bool = False, ignore_exceptions: bool = False -) -> Iterable[Dict]: +) -> Union[Iterable[Dict], tuple]: """ Main text parsing generator function. Returns an iterator object. @@ -156,7 +157,10 @@ def parse( raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True + ignore_exceptions: (boolean) ignore parsing exceptions if True. + This can be used directly or + (preferably) by being passed to the + @add_jc_meta decorator. Yields: @@ -300,7 +304,7 @@ def parse( 'extended_attribute_different': extended_attribute_different[meta[10]] } - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) continue file_line_mac = file_line_mac_re.match(line) @@ -322,14 +326,14 @@ def parse( 'group_different': group_different[meta[7]] } - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) continue file_line_log = file_line_log_re.match(line) if file_line_log: if process != last_process: if summary: - yield stream_success(summary, ignore_exceptions) if raw else stream_success(_process(summary), ignore_exceptions) + yield output_line if raw else _process(output_line) last_process = process summary = {} @@ -358,14 +362,14 @@ def parse( 'extended_attribute_different': extended_attribute_different[meta[10]] } - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) continue file_line_log_mac = file_line_log_mac_re.match(line) if file_line_log_mac: if process != last_process: if summary: - yield stream_success(summary, ignore_exceptions) if raw else stream_success(_process(summary), ignore_exceptions) + yield output_line if raw else _process(output_line) last_process = process summary = {} @@ -392,7 +396,7 @@ def parse( 'group_different': group_different[meta[7]] } - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) continue stat1_line = stat1_line_re.match(line) @@ -452,7 +456,7 @@ def parse( continue if summary: - yield stream_success(summary, ignore_exceptions) if raw else stream_success(_process(summary), ignore_exceptions) + yield summary if raw else _process(summary) except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield e, line diff --git a/jc/parsers/stat_s.py b/jc/parsers/stat_s.py index ce7f6237..f57f80de 100644 --- a/jc/parsers/stat_s.py +++ b/jc/parsers/stat_s.py @@ -83,13 +83,13 @@ Examples: """ import shlex import jc.utils -from jc.utils import stream_success, stream_error +from jc.utils import add_jc_meta from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '0.5' + version = '1.0' description = '`stat` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -130,6 +130,8 @@ def _process(proc_data): return proc_data + +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False): """ Main text parsing generator function. Returns an iterator object. @@ -141,7 +143,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True + ignore_exceptions: (boolean) ignore parsing exceptions if True. + This can be used directly or + (preferably) by being passed to the + @add_jc_meta decorator. Yields: @@ -175,7 +180,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): # line #1 if line.startswith(' File: '): if output_line: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) output_line = {} line_list = line.split(maxsplit=1) @@ -281,16 +286,16 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): } if output_line: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) output_line = {} except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield e, line output_line = {} # gather final item if output_line: try: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield e, line diff --git a/jc/parsers/vmstat_s.py b/jc/parsers/vmstat_s.py index 0e0a319c..5d85cf11 100644 --- a/jc/parsers/vmstat_s.py +++ b/jc/parsers/vmstat_s.py @@ -101,13 +101,13 @@ Examples: ... """ import jc.utils -from jc.utils import stream_success, stream_error +from jc.utils import add_jc_meta from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '0.6' + version = '1.0' description = '`vmstat` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -150,6 +150,7 @@ def _process(proc_data): return proc_data +@add_jc_meta def parse(data, raw=False, quiet=False, ignore_exceptions=False): """ Main text parsing generator function. Returns an iterator object. @@ -161,7 +162,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True - ignore_exceptions: (boolean) ignore parsing exceptions if True + ignore_exceptions: (boolean) ignore parsing exceptions if True. + This can be used directly or + (preferably) by being passed to the + @add_jc_meta decorator. Yields: @@ -266,9 +270,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): } if output_line: - yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + yield output_line if raw else _process(output_line) else: raise ParseError('Not vmstat data') except Exception as e: - yield stream_error(e, ignore_exceptions, line) + yield e, line diff --git a/jc/utils.py b/jc/utils.py index 76eb66f6..6bdc6a50 100644 --- a/jc/utils.py +++ b/jc/utils.py @@ -3,6 +3,7 @@ import sys import re import locale import shutil +from functools import wraps from datetime import datetime, timezone from textwrap import TextWrapper from typing import Dict, Iterable, List, Union, Optional @@ -248,6 +249,58 @@ def stream_error(e: BaseException, ignore_exceptions: bool, line: str) -> Dict: } +def add_jc_meta(func): + """ + Decorator for streaming parsers to add stream_success and stream_error + objects. This simplifies the yield lines in the streaming parsers. + + With the decorator on parse(): + + # successfully parsed line: + yield output_line if raw else _process(output_line) + + # unsuccessfully parsed line: + except Exception as e: + yield e, line + + Without the decorator on parse(): + + # successfully parsed line: + yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) + + # unsuccessfully parsed line: + except Exception as e: + yield stream_error(e, ignore_exceptions, line) + + In all cases above: + + output_line: (Dict): successfully parsed line yielded as a dict + + e: (BaseException): exception object as the first value + of the tuple if the line was not successfully parsed. + + line: (str): string of the original line that did not + successfully parse. + """ + @wraps(func) + def wrapper(*args, **kwargs): + ignore_exceptions = kwargs.get('ignore_exceptions', False) + gen = func(*args, **kwargs) + for value in gen: + # if the yielded value is a dict, then we know it was a + # successfully parsed line + if isinstance(value, dict): + yield stream_success(value, ignore_exceptions) + + # otherwise it will be a tuple and we know it was an error + else: + exception_obj = value[0] + line = value[1] + yield stream_error(exception_obj, ignore_exceptions, line) + + return wrapper + + def input_type_check(data: str) -> None: """Ensure input data is a string. Raises `TypeError` if not.""" if not isinstance(data, str):