diff --git a/CHANGELOG b/CHANGELOG index f43f4257..2d59e8e7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,13 +1,19 @@ jc changelog -20231222 v1.24.1 +20240103 v1.24.1 - Add `kv-dup` parser for Key/Value files with duplicate keys +- TODO: Add `path-list` string parser to parse path list strings found in env variables +- Add `--slurp` functionality to wrap output from multiple lines into a single array. + Note, this only works with single-line input parsers. (e.g. `date`, `ip-address`, `url`, etc.) + Streaming parsers are not supported. Use `jc -hhh` to find parsers compatible with the slurp option. - Enhance `proc-net-tcp` parser to add opposite endian support for architectures like the s390x - Enhance `url` parser to add `parent`, `filename`, `stem`, and `extension` fields - Fix `ini` and `ini-dup` parsers to consistently handle null values as empty strings - Add source link to online parser documentation - Refactor parser aliases for `kv`, `pkg_index_deb`, `lsb_release`, and `os-release` +- TODO: Add `line_slice` function to `utils.py` +- TODO: Update copyright date 20231216 v1.24.0 - Add `debconf-show` command parser diff --git a/jc/__init__.py b/jc/__init__.py index 5c26c5a7..4a093165 100644 --- a/jc/__init__.py +++ b/jc/__init__.py @@ -131,6 +131,7 @@ from .lib import ( plugin_parser_mod_list as plugin_parser_mod_list, standard_parser_mod_list as standard_parser_mod_list, streaming_parser_mod_list as streaming_parser_mod_list, + slurpable_parser_mod_list as slurpable_parser_mod_list, parser_info as parser_info, all_parser_info as all_parser_info, get_help as get_help diff --git a/jc/cli.py b/jc/cli.py index 1fc2fa95..a66b8b5b 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -15,7 +15,8 @@ from typing import List, Dict, Iterable, Union, Optional, TextIO from types import ModuleType from .lib import ( __version__, parser_info, all_parser_info, parsers, _get_parser, _parser_is_streaming, - parser_mod_list, standard_parser_mod_list, plugin_parser_mod_list, streaming_parser_mod_list + parser_mod_list, standard_parser_mod_list, plugin_parser_mod_list, streaming_parser_mod_list, + slurpable_parser_mod_list, _parser_is_slurpable ) from .jc_types import JSONDictType, CustomColorType, ParserInfoType from . import utils @@ -72,7 +73,7 @@ class JcCli(): 'data_in', 'data_out', 'options', 'args', 'parser_module', 'parser_name', 'indent', 'pad', 'custom_colors', 'show_hidden', 'show_categories', 'ascii_only', 'json_separators', 'json_indent', 'run_timestamp', 'about', 'debug', 'verbose_debug', 'force_color', 'mono', - 'help_me', 'pretty', 'quiet', 'ignore_exceptions', 'raw', 'meta_out', 'unbuffer', + 'help_me', 'pretty', 'quiet', 'ignore_exceptions', 'raw', 'slurp', 'meta_out', 'unbuffer', 'version_info', 'yaml_output', 'bash_comp', 'zsh_comp', 'magic_found_parser', 'magic_options', 'magic_run_command', 'magic_run_command_str', 'magic_stdout', 'magic_stderr', 'magic_returncode', 'slice_str', 'slice_start', 'slice_end' @@ -111,6 +112,7 @@ class JcCli(): self.quiet: bool = False self.ignore_exceptions: bool = False self.raw: bool = False + self.slurp: bool = False self.meta_out: bool = False self.unbuffer: bool = False self.version_info: bool = False @@ -219,6 +221,7 @@ class JcCli(): generic = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'generic' in x.get('tags', [])] standard = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'standard' in x.get('tags', [])] command = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'command' in x.get('tags', [])] + slurpable = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'slurpable' in x.get('tags', [])] file_str_bin = [ {'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'file' in x.get('tags', []) or @@ -230,6 +233,7 @@ class JcCli(): 'Generic Parsers:': generic, 'Standard Spec Parsers:': standard, 'File/String/Binary Parsers:': file_str_bin, + 'Slurpable Parsers:': slurpable, 'Streaming Parsers:': streaming, 'Command Parsers:': command } @@ -694,6 +698,41 @@ class JcCli(): elif self.data_in: self.data_in = list(self.data_in)[self.slice_start:self.slice_end] + def create_slurp_output(self) -> None: + """Slurp output into an array. Only works for single-line strings.""" + if self.parser_module and not _parser_is_slurpable(self.parser_module): + utils.error_message([ + f'Slurp option not available with the {self.parser_name} parser.' + ]) + self.exit_error() + + if self.parser_module and isinstance(self.data_in, str): + self.data_out = [] + for line in self.data_in.splitlines(): + parsed_line = self.parser_module.parse( + line, + raw=self.raw, + quiet=self.quiet + ) + self.data_out.append(parsed_line) + + if self.meta_out: + self.run_timestamp = datetime.now(timezone.utc) + self.add_metadata_to_output() + + def create_normal_output(self) -> None: + if self.parser_module: + self.data_out = self.parser_module.parse( + self.data_in, + raw=self.raw, + quiet=self.quiet + ) + + if self.meta_out: + self.run_timestamp = datetime.now(timezone.utc) + self.add_metadata_to_output() + + def streaming_parse_and_print(self) -> None: """only supports UTF-8 string data for now""" self.data_in = sys.stdin @@ -729,15 +768,11 @@ class JcCli(): self.slicer() if self.parser_module: - self.data_out = self.parser_module.parse( - self.data_in, - raw=self.raw, - quiet=self.quiet - ) + if self.slurp: + self.create_slurp_output() - if self.meta_out: - self.run_timestamp = datetime.now(timezone.utc) - self.add_metadata_to_output() + else: + self.create_normal_output() self.safe_print_out() @@ -786,6 +821,7 @@ class JcCli(): self.quiet = 'q' in self.options self.ignore_exceptions = self.options.count('q') > 1 self.raw = 'r' in self.options + self.slurp = 's' in self.options self.meta_out = 'M' in self.options self.unbuffer = 'u' in self.options self.version_info = 'v' in self.options diff --git a/jc/cli_data.py b/jc/cli_data.py index 48422209..2cc4a19b 100644 --- a/jc/cli_data.py +++ b/jc/cli_data.py @@ -11,6 +11,7 @@ long_options_map: Dict[str, List[str]] = { '--pretty': ['p', 'pretty print output'], '--quiet': ['q', 'suppress warnings (double to ignore streaming errors)'], '--raw': ['r', 'raw output'], + '--slurp': ['s', 'slurp multiple lines into an array'], '--unbuffer': ['u', 'unbuffer output'], '--version': ['v', 'version info'], '--yaml-out': ['y', 'YAML output'], diff --git a/jc/lib.py b/jc/lib.py index 31138212..3fbf8e45 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -278,6 +278,18 @@ def _get_parser(parser_mod_name: str) -> ModuleType: modpath: str = 'jcparsers.' if parser_cli_name in local_parsers else 'jc.parsers.' return importlib.import_module(f'{modpath}{parser_mod_name}') +def _parser_is_slurpable(parser: ModuleType) -> bool: + """ + Returns True if this parser can use the `--slurp` command option, else False + + parser is a parser module object. + """ + tag_list = getattr(parser.info, 'tags', []) + if 'slurpable' in tag_list: + return True + + return False + def _parser_is_streaming(parser: ModuleType) -> bool: """ Returns True if this is a streaming parser, else False @@ -503,6 +515,30 @@ def streaming_parser_mod_list( return plist +def slurpable_parser_mod_list( + show_hidden: bool = False, + show_deprecated: bool = False +) -> List[str]: + """ + Returns a list of slurpable parser module names. This function is a + subset of `parser_mod_list()`. + """ + plist: List[str] = [] + for p in parsers: + parser = _get_parser(p) + + if _parser_is_slurpable(parser): + + if not show_hidden and _parser_is_hidden(parser): + continue + + if not show_deprecated and _parser_is_deprecated(parser): + continue + + plist.append(_cliname_to_modname(p)) + + return plist + def parser_info( parser_mod_name: Union[str, ModuleType], documentation: bool = False diff --git a/jc/parsers/date.py b/jc/parsers/date.py index 594f3b80..abcc3e7a 100644 --- a/jc/parsers/date.py +++ b/jc/parsers/date.py @@ -78,13 +78,13 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '2.5' + version = '2.6' description = '`date` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['date'] - tags = ['command'] + tags = ['command', 'slurpable'] __version__ = info.version diff --git a/jc/parsers/datetime_iso.py b/jc/parsers/datetime_iso.py index e7ef9404..6fddff33 100644 --- a/jc/parsers/datetime_iso.py +++ b/jc/parsers/datetime_iso.py @@ -69,13 +69,13 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = 'ISO 8601 Datetime string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Using the pyiso8601 library from https://github.com/micktwomey/pyiso8601/releases/tag/1.0.2' compatible = ['linux', 'aix', 'freebsd', 'darwin', 'win32', 'cygwin'] - tags = ['standard', 'string'] + tags = ['standard', 'string', 'slurpable'] __version__ = info.version diff --git a/jc/parsers/email_address.py b/jc/parsers/email_address.py index f20c6726..9dd6833f 100644 --- a/jc/parsers/email_address.py +++ b/jc/parsers/email_address.py @@ -42,12 +42,12 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = 'Email Address string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] - tags = ['standard', 'string'] + tags = ['standard', 'string', 'slurpable'] __version__ = info.version diff --git a/jc/parsers/foo.py b/jc/parsers/foo.py index 8a4d1c97..8188e517 100644 --- a/jc/parsers/foo.py +++ b/jc/parsers/foo.py @@ -49,10 +49,14 @@ class info(): # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] - # tags options: generic, standard, file, string, binary, command + # tags options: generic, standard, file, string, binary, command, slurpable tags = ['command'] magic_commands = ['foo'] + # other attributes - only enable if needed + deprecated = False + hidden = False + __version__ = info.version diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index 93416e41..5d62c62f 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -55,14 +55,21 @@ class info(): description = '`foo` command streaming parser' author = 'John Doe' author_email = 'johndoe@gmail.com' + # details = 'enter any other details here' # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] - # tags options: generic, standard, file, string, binary, command + # tags options: generic, standard, file, string, binary, command, slurpable tags = ['command'] + + # required for streaming parsers streaming = True + # other attributes - only enable if needed + deprecated = False + hidden = False + __version__ = info.version diff --git a/jc/parsers/ip_address.py b/jc/parsers/ip_address.py index 103ea31e..991159c1 100644 --- a/jc/parsers/ip_address.py +++ b/jc/parsers/ip_address.py @@ -533,12 +533,12 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.3' + version = '1.4' description = 'IPv4 and IPv6 Address string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] - tags = ['standard', 'string'] + tags = ['standard', 'string', 'slurpable'] __version__ = info.version diff --git a/jc/parsers/jwt.py b/jc/parsers/jwt.py index df659343..c1658cfa 100644 --- a/jc/parsers/jwt.py +++ b/jc/parsers/jwt.py @@ -51,12 +51,12 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = 'JWT string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] - tags = ['standard', 'string'] + tags = ['standard', 'string', 'slurpable'] __version__ = info.version diff --git a/jc/parsers/semver.py b/jc/parsers/semver.py index e433de62..08e2f30e 100644 --- a/jc/parsers/semver.py +++ b/jc/parsers/semver.py @@ -54,12 +54,12 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = 'Semantic Version string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] - tags = ['standard', 'string'] + tags = ['standard', 'string', 'slurpable'] __version__ = info.version diff --git a/jc/parsers/timestamp.py b/jc/parsers/timestamp.py index 73c9d991..0c71d385 100644 --- a/jc/parsers/timestamp.py +++ b/jc/parsers/timestamp.py @@ -97,12 +97,12 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = 'Unix Epoch Timestamp string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'aix', 'freebsd', 'darwin', 'win32', 'cygwin'] - tags = ['standard', 'string'] + tags = ['standard', 'string', 'slurpable'] __version__ = info.version diff --git a/jc/parsers/url.py b/jc/parsers/url.py index 5ecea8ef..263d41f5 100644 --- a/jc/parsers/url.py +++ b/jc/parsers/url.py @@ -240,12 +240,12 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.1' + version = '1.2' description = 'URL string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] - tags = ['standard', 'string'] + tags = ['standard', 'string', 'slurpable'] __version__ = info.version diff --git a/jc/parsers/ver.py b/jc/parsers/ver.py index b4e5fd61..a34d764f 100644 --- a/jc/parsers/ver.py +++ b/jc/parsers/ver.py @@ -89,13 +89,13 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = 'Version string parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Based on distutils/version.py from CPython 3.9.5.' compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] - tags = ['generic', 'string'] + tags = ['generic', 'string', 'slurpable'] __version__ = info.version