python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-19 00:17:51 +02:00
Code Issues Releases Activity

add add_jc_meta decorator

Browse Source
This commit is contained in:
Kelly Brazil
2022-02-03 15:44:18 -08:00
parent 2986771f07
commit 49929c714c
9 changed files with 139 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
jc/parsers/csv_s.py
View File

@ -66,13 +66,13 @@ Examples:
import itertools
import csv
import jc.utils
from jc.utils import stream_success, stream_error
from jc.utils import add_jc_meta
from jc.exceptions import ParseError
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.2'
version = '1.3'
description = 'CSV file streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -101,6 +101,7 @@ def _process(proc_data):
return proc_data
@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False):
"""
Main text parsing generator function. Returns an iterator object.
@ -112,7 +113,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
ignore_exceptions: (boolean) ignore parsing exceptions if True
ignore_exceptions: (boolean) ignore parsing exceptions if True.
This can be used directly or
(preferably) by being passed to the
@add_jc_meta decorator.
Yields:
@ -155,6 +159,6 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
for row in reader:
try:
yield stream_success(row, ignore_exceptions) if raw else stream_success(_process(row), ignore_exceptions)
yield row if raw else _process(row)
except Exception as e:
yield stream_error(e, ignore_exceptions, row)
yield e, row

16
jc/parsers/foo_s.py
View File

@ -49,9 +49,9 @@ Examples:
{example output}
...
"""
from typing import Dict, Iterable
from typing import Dict, Iterable, Union
import jc.utils
from jc.utils import stream_success, stream_error
from jc.utils import add_jc_meta
from jc.exceptions import ParseError
@ -91,12 +91,13 @@ def _process(proc_data: Dict) -> Dict:
return proc_data
@add_jc_meta
def parse(
data: Iterable[str],
raw: bool = False,
quiet: bool = False,
ignore_exceptions: bool = False
) -> Iterable[Dict]:
) -> Union[Iterable[Dict], tuple]:
"""
Main text parsing generator function. Returns an iterator object.
@ -107,7 +108,10 @@ def parse(
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
ignore_exceptions: (boolean) ignore parsing exceptions if True
ignore_exceptions: (boolean) ignore parsing exceptions if True.
This can be used directly or
(preferably) by being passed to the
@add_jc_meta decorator.
Yields:
@ -131,9 +135,9 @@ def parse(
# and jc.parsers.universal
if output_line:
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
else:
raise ParseError('Not foo data')
except Exception as e:
yield stream_error(e, ignore_exceptions, line)
yield e, line

15
jc/parsers/iostat_s.py
View File

@ -101,14 +101,14 @@ Examples:
...
"""
import jc.utils
from jc.utils import stream_success, stream_error
from jc.utils import add_jc_meta
from jc.exceptions import ParseError
import jc.parsers.universal
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.0'
version = '1.1'
description = '`iostat` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -159,6 +159,8 @@ def _create_obj_list(section_list, section_name):
item['type'] = section_name
return output_list
@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False):
"""
Main text parsing generator function. Returns an iterator object.
@ -170,7 +172,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
ignore_exceptions: (boolean) ignore parsing exceptions if True
ignore_exceptions: (boolean) ignore parsing exceptions if True.
This can be used directly or
(preferably) by being passed to the
@add_jc_meta decorator.
Yields:
@ -223,9 +228,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
device_list = []
if output_line:
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
else:
raise ParseError('Not iostat data')
except Exception as e:
yield stream_error(e, ignore_exceptions, line)
yield e, line

14
jc/parsers/ls_s.py
View File

@ -79,13 +79,13 @@ Examples:
"""
import re
import jc.utils
from jc.utils import stream_success, stream_error
from jc.utils import add_jc_meta
from jc.exceptions import ParseError
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '0.6'
version = '1.0'
description = '`ls` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -123,6 +123,7 @@ def _process(proc_data):
return proc_data
@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False):
"""
Main text parsing generator function. Returns an iterator object.
@ -134,7 +135,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
ignore_exceptions: (boolean) ignore parsing exceptions if True
ignore_exceptions: (boolean) ignore parsing exceptions if True.
This can be used directly or
(preferably) by being passed to the
@add_jc_meta decorator.
Yields:
@ -196,7 +200,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
output_line['size'] = parsed_line[4]
output_line['date'] = ' '.join(parsed_line[5:8])
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
except Exception as e:
yield stream_error(e, ignore_exceptions, line)
yield e, line

14
jc/parsers/ping_s.py
View File

@ -87,12 +87,12 @@ import string
import ipaddress
import jc.utils
from jc.exceptions import ParseError
from jc.utils import stream_success, stream_error
from jc.utils import add_jc_meta
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '0.6'
version = '1.0'
description = '`ping` and `ping6` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -469,6 +469,7 @@ def _linux_parse(line, s):
return output_line
@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False):
"""
Main text parsing generator function. Returns an iterator object.
@ -480,7 +481,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
ignore_exceptions: (boolean) ignore parsing exceptions if True
ignore_exceptions: (boolean) ignore parsing exceptions if True.
This can be used directly or
(preferably) by being passed to the
@add_jc_meta decorator.
Yields:
@ -542,9 +546,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
# yield the output line if it has data
if output_line:
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
else:
continue
except Exception as e:
yield stream_error(e, ignore_exceptions, line)
yield e, line

30
jc/parsers/rsync_s.py
View File

@ -87,15 +87,15 @@ Examples:
...
"""
import re
from typing import Dict, Iterable
from typing import Dict, Iterable, Union
import jc.utils
from jc.utils import stream_success, stream_error
from jc.utils import add_jc_meta
from jc.exceptions import ParseError
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.0'
version = '1.1'
description = '`rsync` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -140,12 +140,13 @@ def _process(proc_data: Dict) -> Dict:
return proc_data
@add_jc_meta
def parse(
data: Iterable[str],
raw: bool = False,
quiet: bool = False,
ignore_exceptions: bool = False
) -> Iterable[Dict]:
) -> Union[Iterable[Dict], tuple]:
"""
Main text parsing generator function. Returns an iterator object.
@ -156,7 +157,10 @@ def parse(
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
ignore_exceptions: (boolean) ignore parsing exceptions if True
ignore_exceptions: (boolean) ignore parsing exceptions if True.
This can be used directly or
(preferably) by being passed to the
@add_jc_meta decorator.
Yields:
@ -300,7 +304,7 @@ def parse(
'extended_attribute_different': extended_attribute_different[meta[10]]
}
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
continue
file_line_mac = file_line_mac_re.match(line)
@ -322,14 +326,14 @@ def parse(
'group_different': group_different[meta[7]]
}
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
continue
file_line_log = file_line_log_re.match(line)
if file_line_log:
if process != last_process:
if summary:
yield stream_success(summary, ignore_exceptions) if raw else stream_success(_process(summary), ignore_exceptions)
yield output_line if raw else _process(output_line)
last_process = process
summary = {}
@ -358,14 +362,14 @@ def parse(
'extended_attribute_different': extended_attribute_different[meta[10]]
}
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
continue
file_line_log_mac = file_line_log_mac_re.match(line)
if file_line_log_mac:
if process != last_process:
if summary:
yield stream_success(summary, ignore_exceptions) if raw else stream_success(_process(summary), ignore_exceptions)
yield output_line if raw else _process(output_line)
last_process = process
summary = {}
@ -392,7 +396,7 @@ def parse(
'group_different': group_different[meta[7]]
}
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
continue
stat1_line = stat1_line_re.match(line)
@ -452,7 +456,7 @@ def parse(
continue
if summary:
yield stream_success(summary, ignore_exceptions) if raw else stream_success(_process(summary), ignore_exceptions)
yield summary if raw else _process(summary)
except Exception as e:
yield stream_error(e, ignore_exceptions, line)
yield e, line

21
jc/parsers/stat_s.py
View File

@ -83,13 +83,13 @@ Examples:
"""
import shlex
import jc.utils
from jc.utils import stream_success, stream_error
from jc.utils import add_jc_meta
from jc.exceptions import ParseError
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '0.5'
version = '1.0'
description = '`stat` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -130,6 +130,8 @@ def _process(proc_data):
return proc_data
@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False):
"""
Main text parsing generator function. Returns an iterator object.
@ -141,7 +143,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
ignore_exceptions: (boolean) ignore parsing exceptions if True
ignore_exceptions: (boolean) ignore parsing exceptions if True.
This can be used directly or
(preferably) by being passed to the
@add_jc_meta decorator.
Yields:
@ -175,7 +180,7 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
# line #1
if line.startswith(' File: '):
if output_line:
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
output_line = {}
line_list = line.split(maxsplit=1)
@ -281,16 +286,16 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
}
if output_line:
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
output_line = {}
except Exception as e:
yield stream_error(e, ignore_exceptions, line)
yield e, line
output_line = {}
# gather final item
if output_line:
try:
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
except Exception as e:
yield stream_error(e, ignore_exceptions, line)
yield e, line

14
jc/parsers/vmstat_s.py
View File

@ -101,13 +101,13 @@ Examples:
...
"""
import jc.utils
from jc.utils import stream_success, stream_error
from jc.utils import add_jc_meta
from jc.exceptions import ParseError
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '0.6'
version = '1.0'
description = '`vmstat` command streaming parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -150,6 +150,7 @@ def _process(proc_data):
return proc_data
@add_jc_meta
def parse(data, raw=False, quiet=False, ignore_exceptions=False):
"""
Main text parsing generator function. Returns an iterator object.
@ -161,7 +162,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
ignore_exceptions: (boolean) ignore parsing exceptions if True
ignore_exceptions: (boolean) ignore parsing exceptions if True.
This can be used directly or
(preferably) by being passed to the
@add_jc_meta decorator.
Yields:
@ -266,9 +270,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False):
}
if output_line:
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
yield output_line if raw else _process(output_line)
else:
raise ParseError('Not vmstat data')
except Exception as e:
yield stream_error(e, ignore_exceptions, line)
yield e, line

53
jc/utils.py
View File

@ -3,6 +3,7 @@ import sys
import re
import locale
import shutil
from functools import wraps
from datetime import datetime, timezone
from textwrap import TextWrapper
from typing import Dict, Iterable, List, Union, Optional
@ -248,6 +249,58 @@ def stream_error(e: BaseException, ignore_exceptions: bool, line: str) -> Dict:
}
def add_jc_meta(func):
"""
Decorator for streaming parsers to add stream_success and stream_error
objects. This simplifies the yield lines in the streaming parsers.
With the decorator on parse():
# successfully parsed line:
yield output_line if raw else _process(output_line)
# unsuccessfully parsed line:
except Exception as e:
yield e, line
Without the decorator on parse():
# successfully parsed line:
yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
# unsuccessfully parsed line:
except Exception as e:
yield stream_error(e, ignore_exceptions, line)
In all cases above:
output_line: (Dict): successfully parsed line yielded as a dict
e: (BaseException): exception object as the first value
of the tuple if the line was not successfully parsed.
line: (str): string of the original line that did not
successfully parse.
"""
@wraps(func)
def wrapper(*args, **kwargs):
ignore_exceptions = kwargs.get('ignore_exceptions', False)
gen = func(*args, **kwargs)
for value in gen:
# if the yielded value is a dict, then we know it was a
# successfully parsed line
if isinstance(value, dict):
yield stream_success(value, ignore_exceptions)
# otherwise it will be a tuple and we know it was an error
else:
exception_obj = value[0]
line = value[1]
yield stream_error(exception_obj, ignore_exceptions, line)
return wrapper
def input_type_check(data: str) -> None:
"""Ensure input data is a string. Raises `TypeError` if not."""
if not isinstance(data, str):