From 8e9ff7fa9fd3342f1ba40f2c9158796b1ff18e4e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 6 Jan 2024 18:23:46 -0800 Subject: [PATCH] refactor to use get_parser() --- CHANGELOG | 3 +- docs/lib.md | 23 ++++++++--- docs/utils.md | 8 ++-- jc/cli.py | 6 +-- jc/lib.py | 95 ++++++++++++++++++++++++-------------------- man/jc.1 | 2 +- tests/test_jc_lib.py | 16 +++++--- 7 files changed, 90 insertions(+), 63 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 058a019e..736a51b2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,6 @@ jc changelog -20240103 v1.24.1 +20240106 v1.24.1 - Add `kv-dup` parser for Key/Value files with duplicate keys - TODO: Add `path-list` string parser to parse path list strings found in env variables - Add `--slurp` functionality to wrap output from multiple lines into a single array. @@ -14,6 +14,7 @@ jc changelog - Add snap package build scripts - Refactor parser aliases for `kv`, `pkg_index_deb`, `lsb_release`, and `os-release` - Add `line_slice` function to `utils.py` +- Add `get_parser` function to `lib.py` - Update copyright date 20231216 v1.24.0 diff --git a/docs/lib.md b/docs/lib.md index d6c89f7b..537ca2ee 100644 --- a/docs/lib.md +++ b/docs/lib.md @@ -23,10 +23,11 @@ jc - JSON Convert lib module ### get\_parser ```python -def get_parser(parser_mod_name) -> ModuleType +def get_parser(parser_mod_name: Union[str, ModuleType]) -> ModuleType ``` -Return the parser module object +Return the parser module object and check that the module is a valid +parser module. Parameters: @@ -81,15 +82,25 @@ To get a list of available parser module names, use `parser_mod_list()`. Alternatively, a parser module object can be supplied: >>> import jc - >>> import jc.parsers.date as jc_date + >>> jc_date = jc.get_parser('date') >>> date_obj = jc.parse(jc_date, 'Tue Jan 18 10:23:07 PST 2022') >>> print(f'The year is: {date_obj["year"]}') The year is: 2022 -You can also use the lower-level parser modules directly: +You can also use the parser modules directly via `get_parser()`: + + >>> import jc + >>> jc_date = jc.get_parser('date') + >>> date_obj = jc_date.parse('Tue Jan 18 10:23:07 PST 2022') + >>> print(f'The year is: {date_obj["year"]}') + The year is: 2022 + +Finally, you can access the low-level parser modules manually: >>> import jc.parsers.date - >>> jc.parsers.date.parse('Tue Jan 18 10:23:07 PST 2022') + >>> date_obj = jc.parsers.date.parse('Tue Jan 18 10:23:07 PST 2022') + >>> print(f'The year is: {date_obj["year"]}') + The year is: 2022 Though, accessing plugin parsers directly is a bit more cumbersome, so this higher-level API is recommended. Here is how you can access plugin @@ -112,7 +123,7 @@ Parameters: variants of the module name. A Module object can also be passed - directly or via _get_parser() + directly or via get_parser() data: (string or data to parse (string or bytes for bytes or standard parsers, iterable of diff --git a/docs/utils.md b/docs/utils.md index 78c7ac29..51ce2ee5 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -237,9 +237,11 @@ Ensure input data is a string. Raises `TypeError` if not. ### line\_slice ```python -def line_slice(data: Union[str, Iterable], - slice_start: Optional[int] = None, - slice_end: Optional[int] = None) -> Union[str, Iterable] +def line_slice( + data: Union[str, Iterable[str], TextIO, bytes, None], + slice_start: Optional[int] = None, + slice_end: Optional[int] = None +) -> Union[str, Iterable[str], TextIO, bytes, None] ``` Slice input data by lines - lazily, if possible. diff --git a/jc/cli.py b/jc/cli.py index d841a5e3..6b9383ae 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -13,7 +13,7 @@ import subprocess from typing import List, Dict, Iterable, Union, Optional, TextIO from types import ModuleType from .lib import ( - __version__, parser_info, all_parser_info, parsers, _get_parser, _parser_is_streaming, + __version__, parser_info, all_parser_info, parsers, get_parser, _parser_is_streaming, parser_mod_list, standard_parser_mod_list, plugin_parser_mod_list, streaming_parser_mod_list, slurpable_parser_mod_list, _parser_is_slurpable ) @@ -584,7 +584,7 @@ class JcCli(): def set_parser_module_and_parser_name(self) -> None: if self.magic_found_parser: - self.parser_module = _get_parser(self.magic_found_parser) + self.parser_module = get_parser(self.magic_found_parser) self.parser_name = self.parser_shortname(self.magic_found_parser) else: @@ -593,7 +593,7 @@ class JcCli(): self.parser_name = self.parser_shortname(arg) if self.parser_name in parsers: - self.parser_module = _get_parser(arg) + self.parser_module = get_parser(arg) found = True break diff --git a/jc/lib.py b/jc/lib.py index ff7c1b29..15e4e23a 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -7,6 +7,7 @@ from typing import List, Iterable, Optional, Union, Iterator from types import ModuleType from .jc_types import ParserInfoType, JSONDictType from jc import appdirs +from jc import utils __version__ = '1.24.1' @@ -242,6 +243,9 @@ def _is_valid_parser_plugin(name: str, local_parsers_dir: str) -> bool: if hasattr(plugin, 'info') and hasattr(plugin, 'parse'): del plugin return True + else: + utils.warning_message([f'Not installing invalid parser plugin "{parser_mod_name}" at {local_parsers_dir}']) + return False except Exception: return False return False @@ -270,9 +274,10 @@ def _parser_argument(parser_mod_name: str) -> str: parser = _modname_to_cliname(parser_mod_name) return f'--{parser}' -def get_parser(parser_mod_name) -> ModuleType: +def get_parser(parser_mod_name: Union[str, ModuleType]) -> ModuleType: """ - Return the parser module object + Return the parser module object and check that the module is a valid + parser module. Parameters: @@ -283,13 +288,18 @@ def get_parser(parser_mod_name) -> ModuleType: Returns: Parser: the parser module object - - """ if isinstance(parser_mod_name, ModuleType): jc_parser = parser_mod_name else: - jc_parser = _get_parser(parser_mod_name) + try: + jc_parser = _get_parser(parser_mod_name) + except ModuleNotFoundError: + raise ModuleNotFoundError(f'"{parser_mod_name}" is not found or is not a valid parser module.') + + if not hasattr(jc_parser, 'info') or not hasattr(jc_parser, 'parse'): + raise ModuleNotFoundError(f'"{jc_parser}" is not a valid parser module.') + return jc_parser def _get_parser(parser_mod_name: str) -> ModuleType: @@ -382,15 +392,25 @@ def parse( Alternatively, a parser module object can be supplied: >>> import jc - >>> import jc.parsers.date as jc_date + >>> jc_date = jc.get_parser('date') >>> date_obj = jc.parse(jc_date, 'Tue Jan 18 10:23:07 PST 2022') >>> print(f'The year is: {date_obj["year"]}') The year is: 2022 - You can also use the lower-level parser modules directly: + You can also use the parser modules directly via `get_parser()`: + + >>> import jc + >>> jc_date = jc.get_parser('date') + >>> date_obj = jc_date.parse('Tue Jan 18 10:23:07 PST 2022') + >>> print(f'The year is: {date_obj["year"]}') + The year is: 2022 + + Finally, you can access the low-level parser modules manually: >>> import jc.parsers.date - >>> jc.parsers.date.parse('Tue Jan 18 10:23:07 PST 2022') + >>> date_obj = jc.parsers.date.parse('Tue Jan 18 10:23:07 PST 2022') + >>> print(f'The year is: {date_obj["year"]}') + The year is: 2022 Though, accessing plugin parsers directly is a bit more cumbersome, so this higher-level API is recommended. Here is how you can access plugin @@ -413,7 +433,7 @@ def parse( variants of the module name. A Module object can also be passed - directly or via _get_parser() + directly or via get_parser() data: (string or data to parse (string or bytes for bytes or standard parsers, iterable of @@ -451,7 +471,7 @@ def parser_mod_list( """Returns a list of all available parser module names.""" plist: List[str] = [] for p in parsers: - parser = _get_parser(p) + parser = get_parser(p) if not show_hidden and _parser_is_hidden(parser): continue @@ -473,7 +493,7 @@ def plugin_parser_mod_list( """ plist: List[str] = [] for p in local_parsers: - parser = _get_parser(p) + parser = get_parser(p) if not show_hidden and _parser_is_hidden(parser): continue @@ -496,7 +516,7 @@ def standard_parser_mod_list( """ plist: List[str] = [] for p in parsers: - parser = _get_parser(p) + parser = get_parser(p) if not _parser_is_streaming(parser): @@ -520,7 +540,7 @@ def streaming_parser_mod_list( """ plist: List[str] = [] for p in parsers: - parser = _get_parser(p) + parser = get_parser(p) if _parser_is_streaming(parser): @@ -544,7 +564,7 @@ def slurpable_parser_mod_list( """ plist: List[str] = [] for p in parsers: - parser = _get_parser(p) + parser = get_parser(p) if _parser_is_slurpable(parser): @@ -575,33 +595,26 @@ def parser_info( documentation: (boolean) include parser docstring if True """ - if isinstance(parser_mod_name, ModuleType): - parser_mod = parser_mod_name - parser_mod_name = parser_mod.__name__.split('.')[-1] - else: - # ensure parser_mod_name is a true module name and not a cli name - parser_mod_name = _cliname_to_modname(parser_mod_name) - parser_mod = _get_parser(parser_mod_name) + parser_mod = get_parser(parser_mod_name) + parser_mod_name = parser_mod.__name__.split('.')[-1] info_dict: ParserInfoType = {} + info_dict['name'] = parser_mod_name + info_dict['argument'] = _parser_argument(parser_mod_name) + parser_entry = vars(parser_mod.info) - if hasattr(parser_mod, 'info'): - info_dict['name'] = parser_mod_name - info_dict['argument'] = _parser_argument(parser_mod_name) - parser_entry = vars(parser_mod.info) + for k, v in parser_entry.items(): + if not k.startswith('__'): + info_dict[k] = v # type: ignore - for k, v in parser_entry.items(): - if not k.startswith('__'): - info_dict[k] = v # type: ignore + if _modname_to_cliname(parser_mod_name) in local_parsers: + info_dict['plugin'] = True - if _modname_to_cliname(parser_mod_name) in local_parsers: - info_dict['plugin'] = True - - if documentation: - docs = parser_mod.__doc__ - if not docs: - docs = 'No documentation available.\n' - info_dict['documentation'] = docs + if documentation: + docs = parser_mod.__doc__ + if not docs: + docs = 'No documentation available.\n' + info_dict['documentation'] = docs return info_dict @@ -625,7 +638,7 @@ def all_parser_info( """ plist: List[str] = [] for p in parsers: - parser = _get_parser(p) + parser = get_parser(p) if not show_hidden and _parser_is_hidden(parser): continue @@ -633,7 +646,7 @@ def all_parser_info( if not show_deprecated and _parser_is_deprecated(parser): continue - plist.append(_cliname_to_modname(p)) + plist.append(p) p_info_list: List[ParserInfoType] = [parser_info(p, documentation=documentation) for p in plist] @@ -647,9 +660,5 @@ def get_help(parser_mod_name: Union[str, ModuleType]) -> None: **--argument-name** variants of the module name string as well as a parser module object. """ - if isinstance(parser_mod_name, ModuleType): - jc_parser = parser_mod_name - else: - jc_parser = _get_parser(parser_mod_name) - + jc_parser = get_parser(parser_mod_name) help(jc_parser) diff --git a/man/jc.1 b/man/jc.1 index 3e0a8e35..bcfa05e1 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2024-01-05 1.24.1 "JSON Convert" +.TH jc 1 2024-01-06 1.24.1 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings diff --git a/tests/test_jc_lib.py b/tests/test_jc_lib.py index 1ede4956..d968cdfd 100644 --- a/tests/test_jc_lib.py +++ b/tests/test_jc_lib.py @@ -78,7 +78,7 @@ class MyTests(unittest.TestCase): def test_lib_all_parser_info_show_deprecated(self): # save old state old_parsers = deepcopy(jc.lib.parsers) - old_get_parser = deepcopy(jc.lib._get_parser) + old_get_parser = deepcopy(jc.lib.get_parser) # mock data class mock_parser_info: @@ -92,21 +92,23 @@ class MyTests(unittest.TestCase): class mock_parser: info = mock_parser_info + def parse(): + pass jc.lib.parsers = ['deprecated'] - jc.lib._get_parser = lambda x: mock_parser # type: ignore + jc.lib.get_parser = lambda x: mock_parser # type: ignore result = jc.lib.all_parser_info(show_deprecated=True) # reset jc.lib.parsers = old_parsers - jc.lib._get_parser = old_get_parser + jc.lib.get_parser = old_get_parser self.assertEqual(len(result), 1) def test_lib_all_parser_info_show_hidden(self): # save old state old_parsers = deepcopy(jc.lib.parsers) - old_get_parser = deepcopy(jc.lib._get_parser) + old_get_parser = deepcopy(jc.lib.get_parser) # mock data class mock_parser_info: @@ -120,14 +122,16 @@ class MyTests(unittest.TestCase): class mock_parser: info = mock_parser_info + def parse(): + pass jc.lib.parsers = ['deprecated'] - jc.lib._get_parser = lambda x: mock_parser # type: ignore + jc.lib.get_parser = lambda x: mock_parser # type: ignore result = jc.lib.all_parser_info(show_hidden=True) # reset jc.lib.parsers = old_parsers - jc.lib._get_parser = old_get_parser + jc.lib.get_parser = old_get_parser self.assertEqual(len(result), 1)