1
0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-17 00:07:37 +02:00

fix for more graceful handling of missing optional libraries

This commit is contained in:
Kelly Brazil
2024-02-10 11:16:38 -08:00
parent e6bdbb42d3
commit d3e7f09ca3
16 changed files with 325 additions and 199 deletions

View File

@ -1,7 +1,10 @@
jc changelog
20240205 v1.25.1
- Fix remove extra "$" in slicing help
20240210 v1.25.1
- Fix for crash when optional libraries are not installed (e.g. xmltodict)
- Fix for `ini` parser crashing with some keys with no values
- Add tests for missing optional libraries
- Documentation updates
20240204 v1.25.0
- Add `--slurp` functionality to wrap output from multiple lines into a single array.

View File

@ -98,4 +98,4 @@ Compatibility: linux, darwin, cygwin, win32, aix, freebsd
Source: [`jc/parsers/ini.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/ini.py)
Version 2.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 2.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

View File

@ -76,4 +76,4 @@ Compatibility: linux, darwin, cygwin, win32, aix, freebsd
Source: [`jc/parsers/plist.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/plist.py)
Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

View File

@ -100,4 +100,4 @@ Compatibility: linux, darwin, cygwin, win32, aix, freebsd
Source: [`jc/parsers/xml.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/xml.py)
Version 1.9 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.10 by Kelly Brazil (kellyjonbrazil@gmail.com)

View File

@ -408,7 +408,7 @@ class JcCli():
ensure_ascii=self.ascii_only
)
if not self.mono:
if not self.mono and PYGMENTS_INSTALLED:
class JcStyle(Style):
styles: CustomColorType = self.custom_colors

View File

@ -251,7 +251,8 @@ def _is_valid_parser_plugin(name: str, local_parsers_dir: str) -> bool:
else:
utils.warning_message([f'Not installing invalid parser plugin "{parser_mod_name}" at {local_parsers_dir}'])
return False
except Exception:
except Exception as e:
utils.warning_message([f'Not installing parser plugin "{parser_mod_name}" at {local_parsers_dir} due to error: {e}'])
return False
return False
@ -324,7 +325,16 @@ def _get_parser(parser_mod_name: str) -> ModuleType:
parser_mod_name = _cliname_to_modname(parser_mod_name)
parser_cli_name = _modname_to_cliname(parser_mod_name)
modpath: str = 'jcparsers.' if parser_cli_name in local_parsers else 'jc.parsers.'
return importlib.import_module(f'{modpath}{parser_mod_name}')
mod = None
try:
mod = importlib.import_module(f'{modpath}{parser_mod_name}')
except Exception as e:
mod = importlib.import_module(f'{modpath}disabled_parser')
mod.__name__ = parser_mod_name
utils.warning_message([f'"{parser_mod_name}" parser disabled due to error: {e}'])
return mod
def _parser_is_slurpable(parser: ModuleType) -> bool:
"""

View File

@ -0,0 +1,23 @@
"""jc - JSON Convert broken parser - for testing purposes only"""
import non_existent_library
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.0'
description = 'broken parser'
author = 'N/A'
author_email = 'N/A'
compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
hidden = True
__version__ = info.version
def parse(
data: str,
raw: bool = False,
quiet: bool = False
) -> dict:
"""Main text parsing function"""
return {}

View File

@ -0,0 +1,22 @@
"""jc - JSON Convert disabled parser"""
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.0'
description = 'Disabled parser'
author = 'N/A'
author_email = 'N/A'
compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
hidden = True
__version__ = info.version
def parse(
data: str,
raw: bool = False,
quiet: bool = False
) -> dict:
"""Main text parsing function"""
return {}

View File

@ -44,6 +44,14 @@ Examples:
...
}
"""
import sys
# ugly hack because I accidentally shadowed the xml module from the
# standard library with the xml parser. :(
for index, item in enumerate(sys.path.copy()):
if 'jc/jc/parsers' in item or '':
del sys.path[index]
from typing import Dict, Union
import plistlib
import binascii
@ -53,7 +61,7 @@ import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.1'
version = '1.2'
description = 'PLIST file parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'

View File

@ -73,15 +73,10 @@ Examples:
import jc.utils
from jc.exceptions import LibraryNotInstalled
try:
import xmltodict
except Exception:
raise LibraryNotInstalled('The xmltodict library is not installed.')
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.9'
version = '1.10'
description = 'XML file parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -93,7 +88,7 @@ class info():
__version__ = info.version
def _process(proc_data, has_data=False):
def _process(proc_data, has_data=False, xml_mod=None):
"""
Final processing to conform to the schema.
@ -105,16 +100,19 @@ def _process(proc_data, has_data=False):
Dictionary representing an XML document.
"""
if not xml_mod:
raise LibraryNotInstalled('The xmltodict library is not installed.')
proc_output = []
if has_data:
# standard output with @ prefix for attributes
try:
proc_output = xmltodict.parse(proc_data,
proc_output = xml_mod.parse(proc_data,
dict_constructor=dict,
process_comments=True)
except (ValueError, TypeError):
proc_output = xmltodict.parse(proc_data, dict_constructor=dict)
proc_output = xml_mod.parse(proc_data, dict_constructor=dict)
return proc_output
@ -133,6 +131,12 @@ def parse(data, raw=False, quiet=False):
Dictionary. Raw or processed structured data.
"""
xmltodict = None
try:
import xmltodict
except Exception:
raise LibraryNotInstalled('The xmltodict library is not installed.')
jc.utils.compatibility(__name__, info.compatible, quiet)
jc.utils.input_type_check(data)
@ -156,4 +160,4 @@ def parse(data, raw=False, quiet=False):
return raw_output
return _process(data, has_data)
return _process(data, has_data, xml_mod=xmltodict)

View File

@ -1,4 +1,4 @@
.TH jc 1 2024-02-07 1.25.1 "JSON Convert"
.TH jc 1 2024-02-10 1.25.1 "JSON Convert"
.SH NAME
\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types,
and strings

7
runtests-missing-libs.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
# system should be in "America/Los_Angeles" timezone for all tests to pass
# ensure no local plugin parsers are installed for all tests to pass
pip uninstall pygments ruamel.yaml xmltodict --yes
python3 -m unittest -v
pip install pygments ruamel.yaml xmltodict

View File

@ -1,12 +1,20 @@
import os
import unittest
from datetime import datetime, timezone
try:
import pygments
from pygments.token import (Name, Number, String, Keyword)
PYGMENTS_INSTALLED=True
except ModuleNotFoundError:
except:
PYGMENTS_INSTALLED=False
try:
import ruamel.yaml
RUAMELYAML_INSTALLED = True
except:
RUAMELYAML_INSTALLED = False
from jc.cli import JcCli
import jc.parsers.url as url_parser
import jc.parsers.proc as proc_parser
@ -47,164 +55,165 @@ class MyTests(unittest.TestCase):
resulting_attributes = (cli.magic_found_parser, cli.magic_options, cli.magic_run_command)
self.assertEqual(expected, resulting_attributes)
@unittest.skipIf(not PYGMENTS_INSTALLED, 'pygments library not installed')
def test_cli_set_env_colors(self):
if PYGMENTS_INSTALLED:
if pygments.__version__.startswith('2.3.'):
env = {
'': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
},
' ': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
},
'default,default,default,default': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
},
'red,red,red,red': {
Name.Tag: 'bold #ansidarkred',
Keyword: '#ansidarkred',
Number: '#ansidarkred',
String: '#ansidarkred'
},
'red,red,yada,red': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
},
'red,red,red': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
},
'red,red,red,red,red,red': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
}
if pygments.__version__.startswith('2.3.'):
env = {
'': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
},
' ': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
},
'default,default,default,default': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
},
'red,red,red,red': {
Name.Tag: 'bold #ansidarkred',
Keyword: '#ansidarkred',
Number: '#ansidarkred',
String: '#ansidarkred'
},
'red,red,yada,red': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
},
'red,red,red': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
},
'red,red,red,red,red,red': {
Name.Tag: 'bold #ansidarkblue',
Keyword: '#ansidarkgray',
Number: '#ansipurple',
String: '#ansidarkgreen'
}
else:
env = {
'': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
},
' ': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
},
'default,default,default,default': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
},
'red,red,red,red': {
Name.Tag: 'bold ansired',
Keyword: 'ansired',
Number: 'ansired',
String: 'ansired'
},
'red,red,yada,red': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
},
'red,red,red': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
},
'red,red,red,red,red,red': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
}
}
else:
env = {
'': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
},
' ': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
},
'default,default,default,default': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
},
'red,red,red,red': {
Name.Tag: 'bold ansired',
Keyword: 'ansired',
Number: 'ansired',
String: 'ansired'
},
'red,red,yada,red': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
},
'red,red,red': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
},
'red,red,red,red,red,red': {
Name.Tag: 'bold ansiblue',
Keyword: 'ansibrightblack',
Number: 'ansimagenta',
String: 'ansigreen'
}
}
for jc_colors, expected_colors in env.items():
cli = JcCli()
os.environ["JC_COLORS"] = jc_colors
cli.set_custom_colors()
self.assertEqual(cli.custom_colors, expected_colors)
for jc_colors, expected_colors in env.items():
cli = JcCli()
os.environ["JC_COLORS"] = jc_colors
cli.set_custom_colors()
self.assertEqual(cli.custom_colors, expected_colors)
@unittest.skipIf(not PYGMENTS_INSTALLED, 'pygments library not installed')
def test_cli_json_out(self):
if PYGMENTS_INSTALLED:
test_input = [
None,
{},
[],
'',
{"key1": "value1", "key2": 2, "key3": None, "key4": 3.14, "key5": True},
]
if pygments.__version__.startswith('2.3.'):
expected_output = [
'\x1b[30;01mnull\x1b[39;00m',
'{}',
'[]',
'\x1b[32m""\x1b[39m',
'{\x1b[34;01m"key1"\x1b[39;00m:\x1b[32m"value1"\x1b[39m,\x1b[34;01m"key2"\x1b[39;00m:\x1b[35m2\x1b[39m,\x1b[34;01m"key3"\x1b[39;00m:\x1b[30;01mnull\x1b[39;00m,\x1b[34;01m"key4"\x1b[39;00m:\x1b[35m3.14\x1b[39m,\x1b[34;01m"key5"\x1b[39;00m:\x1b[30;01mtrue\x1b[39;00m}'
]
else:
expected_output = [
'\x1b[90mnull\x1b[39m',
'{}',
'[]',
'\x1b[32m""\x1b[39m',
'{\x1b[34;01m"key1"\x1b[39;00m:\x1b[32m"value1"\x1b[39m,\x1b[34;01m"key2"\x1b[39;00m:\x1b[35m2\x1b[39m,\x1b[34;01m"key3"\x1b[39;00m:\x1b[90mnull\x1b[39m,\x1b[34;01m"key4"\x1b[39;00m:\x1b[35m3.14\x1b[39m,\x1b[34;01m"key5"\x1b[39;00m:\x1b[90mtrue\x1b[39m}'
]
for test_dict, expected_json in zip(test_input, expected_output):
cli = JcCli()
os.environ["JC_COLORS"] = "default,default,default,default"
cli.set_custom_colors()
cli.data_out = test_dict
self.assertEqual(cli.json_out(), expected_json)
def test_cli_json_out_mono(self):
if PYGMENTS_INSTALLED:
test_input = [
None,
{},
[],
'',
{"key1": "value1", "key2": 2, "key3": None, "key4": 3.14, "key5": True},
]
test_input = [
None,
{},
[],
'',
{"key1": "value1", "key2": 2, "key3": None, "key4": 3.14, "key5": True},
]
if pygments.__version__.startswith('2.3.'):
expected_output = [
'null',
'\x1b[30;01mnull\x1b[39;00m',
'{}',
'[]',
'""',
'{"key1":"value1","key2":2,"key3":null,"key4":3.14,"key5":true}'
'\x1b[32m""\x1b[39m',
'{\x1b[34;01m"key1"\x1b[39;00m:\x1b[32m"value1"\x1b[39m,\x1b[34;01m"key2"\x1b[39;00m:\x1b[35m2\x1b[39m,\x1b[34;01m"key3"\x1b[39;00m:\x1b[30;01mnull\x1b[39;00m,\x1b[34;01m"key4"\x1b[39;00m:\x1b[35m3.14\x1b[39m,\x1b[34;01m"key5"\x1b[39;00m:\x1b[30;01mtrue\x1b[39;00m}'
]
else:
expected_output = [
'\x1b[90mnull\x1b[39m',
'{}',
'[]',
'\x1b[32m""\x1b[39m',
'{\x1b[34;01m"key1"\x1b[39;00m:\x1b[32m"value1"\x1b[39m,\x1b[34;01m"key2"\x1b[39;00m:\x1b[35m2\x1b[39m,\x1b[34;01m"key3"\x1b[39;00m:\x1b[90mnull\x1b[39m,\x1b[34;01m"key4"\x1b[39;00m:\x1b[35m3.14\x1b[39m,\x1b[34;01m"key5"\x1b[39;00m:\x1b[90mtrue\x1b[39m}'
]
for test_dict, expected_json in zip(test_input, expected_output):
cli = JcCli()
cli.set_custom_colors()
cli.mono = True
cli.data_out = test_dict
self.assertEqual(cli.json_out(), expected_json)
for test_dict, expected_json in zip(test_input, expected_output):
cli = JcCli()
os.environ["JC_COLORS"] = "default,default,default,default"
cli.set_custom_colors()
cli.data_out = test_dict
self.assertEqual(cli.json_out(), expected_json)
@unittest.skipIf(not PYGMENTS_INSTALLED, 'pygments library not installed')
def test_cli_json_out_mono(self):
test_input = [
None,
{},
[],
'',
{"key1": "value1", "key2": 2, "key3": None, "key4": 3.14, "key5": True},
]
expected_output = [
'null',
'{}',
'[]',
'""',
'{"key1":"value1","key2":2,"key3":null,"key4":3.14,"key5":true}'
]
for test_dict, expected_json in zip(test_input, expected_output):
cli = JcCli()
cli.set_custom_colors()
cli.mono = True
cli.data_out = test_dict
self.assertEqual(cli.json_out(), expected_json)
@unittest.skipIf(not PYGMENTS_INSTALLED, 'pygments library not installed')
def test_cli_json_out_pretty(self):
test_input = [
{"key1": "value1", "key2": 2, "key3": None, "key4": 3.14, "key5": True},
@ -229,40 +238,60 @@ class MyTests(unittest.TestCase):
cli.data_out = test_dict
self.assertEqual(cli.json_out(), expected_json)
@unittest.skipIf(PYGMENTS_INSTALLED, 'pygments library installed')
def test_cli_json_out_pretty_no_pygments(self):
test_input = [
{"key1": "value1", "key2": 2, "key3": None, "key4": 3.14, "key5": True},
{"key1": [{"subkey1": "subvalue1"}, {"subkey2": [1, 2, 3]}], "key2": True}
]
expected_output = [
'{\n "key1": "value1",\n "key2": 2,\n "key3": null,\n "key4": 3.14,\n "key5": true\n}',
'{\n "key1": [\n {\n "subkey1": "subvalue1"\n },\n {\n "subkey2": [\n 1,\n 2,\n 3\n ]\n }\n ],\n "key2": true\n}'
]
for test_dict, expected_json in zip(test_input, expected_output):
cli = JcCli()
cli.pretty = True
cli.set_custom_colors()
cli.data_out = test_dict
self.assertEqual(cli.json_out(), expected_json)
@unittest.skipIf(not PYGMENTS_INSTALLED, 'pygments library not installed')
def test_cli_yaml_out(self):
if PYGMENTS_INSTALLED:
test_input = [
None,
{},
[],
'',
{"key1": "value1", "key2": 2, "key3": None, "key4": 3.14, "key5": True},
test_input = [
None,
{},
[],
'',
{"key1": "value1", "key2": 2, "key3": None, "key4": 3.14, "key5": True},
]
if pygments.__version__.startswith('2.3.'):
expected_output = [
'---\n...',
'--- {}',
'--- []',
"--- \x1b[32m'\x1b[39m\x1b[32m'\x1b[39m",
'---\nkey1: value1\nkey2: 2\nkey3:\nkey4: 3.14\nkey5: true'
]
else:
expected_output = [
'---\n...',
'--- {}',
'--- []',
"--- \x1b[32m'\x1b[39m\x1b[32m'\x1b[39m",
'---\n\x1b[34;01mkey1\x1b[39;00m: value1\n\x1b[34;01mkey2\x1b[39;00m: 2\n\x1b[34;01mkey3\x1b[39;00m:\n\x1b[34;01mkey4\x1b[39;00m: 3.14\n\x1b[34;01mkey5\x1b[39;00m: true'
]
if pygments.__version__.startswith('2.3.'):
expected_output = [
'---\n...',
'--- {}',
'--- []',
"--- \x1b[32m'\x1b[39m\x1b[32m'\x1b[39m",
'---\nkey1: value1\nkey2: 2\nkey3:\nkey4: 3.14\nkey5: true'
]
else:
expected_output = [
'---\n...',
'--- {}',
'--- []',
"--- \x1b[32m'\x1b[39m\x1b[32m'\x1b[39m",
'---\n\x1b[34;01mkey1\x1b[39;00m: value1\n\x1b[34;01mkey2\x1b[39;00m: 2\n\x1b[34;01mkey3\x1b[39;00m:\n\x1b[34;01mkey4\x1b[39;00m: 3.14\n\x1b[34;01mkey5\x1b[39;00m: true'
]
for test_dict, expected_json in zip(test_input, expected_output):
cli = JcCli()
os.environ["JC_COLORS"] = "default,default,default,default"
cli.set_custom_colors()
cli.data_out = test_dict
self.assertEqual(cli.yaml_out(), expected_json)
for test_dict, expected_json in zip(test_input, expected_output):
cli = JcCli()
os.environ["JC_COLORS"] = "default,default,default,default"
cli.set_custom_colors()
cli.data_out = test_dict
self.assertEqual(cli.yaml_out(), expected_json)
@unittest.skipIf(not RUAMELYAML_INSTALLED, 'ruamel.yaml library not installed')
def test_cli_yaml_out_mono(self):
test_input = [
None,

View File

@ -11,6 +11,12 @@ class MyTests(unittest.TestCase):
p = jc.lib.get_parser('arp')
self.assertIsInstance(p, ModuleType)
def test_lib_get_parser_broken_parser(self):
"""get_parser substitutes the disabled_parser if a parser is broken"""
broken = jc.lib.get_parser('broken_parser')
disabled = jc.lib.get_parser('disabled_parser')
self.assertIs(broken, disabled)
def test_lib_get_parser_module(self):
p = jc.lib.get_parser(csv_parser)
self.assertIsInstance(p, ModuleType)

View File

@ -2,7 +2,6 @@ import os
import unittest
import json
import jc.parsers.xml
import xmltodict
# fix for whether tests are run directly or via runtests.sh
try:
@ -10,10 +9,18 @@ try:
except:
from _vendor.packaging import version # type: ignore
# check the version of installed xmltodict library
try:
import xmltodict
XMLTODICT_INSTALLED = True
XMLTODICT_0_13_0_OR_HIGHER = version.parse(xmltodict.__version__) >= version.parse('0.13.0') # type: ignore
except:
XMLTODICT_INSTALLED = False
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
XMLTODICT_0_13_0_OR_HIGHER = version.parse(xmltodict.__version__) >= version.parse('0.13.0')
@unittest.skipIf(not XMLTODICT_INSTALLED, 'xmltodict library not installed')
class MyTests(unittest.TestCase):
# input

View File

@ -5,7 +5,14 @@ import jc.parsers.yaml
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
try:
import ruamel.yaml
RUAMELYAML_INSTALLED = True
except:
RUAMELYAML_INSTALLED = False
@unittest.skipIf(not RUAMELYAML_INSTALLED, 'ruamel.yaml library not installed')
class MyTests(unittest.TestCase):
# input