fix for more graceful handling of missing optional libraries

CHANGELOG

@@ -1,7 +1,10 @@
 jc changelog
 
-20240205 v1.25.1
-- Fix remove extra "$" in slicing help
+20240210 v1.25.1
+- Fix for crash when optional libraries are not installed (e.g. xmltodict)
+- Fix for `ini` parser crashing with some keys with no values
+- Add tests for missing optional libraries
+- Documentation updates
 
 20240204 v1.25.0
 - Add `--slurp` functionality to wrap output from multiple lines into a single array.

docs/parsers/ini.md

@@ -98,4 +98,4 @@ Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
 Source: [`jc/parsers/ini.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/ini.py)
 
-Version 2.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 2.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/plist.md

@@ -76,4 +76,4 @@ Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
 Source: [`jc/parsers/plist.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/plist.py)
 
-Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/xml.md

@@ -100,4 +100,4 @@ Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
 Source: [`jc/parsers/xml.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/xml.py)
 
-Version 1.9 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.10 by Kelly Brazil (kellyjonbrazil@gmail.com)

jc/cli.py

@@ -408,7 +408,7 @@ class JcCli():
             ensure_ascii=self.ascii_only
         )
 
-        if not self.mono:
+        if not self.mono and PYGMENTS_INSTALLED:
             class JcStyle(Style):
                 styles: CustomColorType = self.custom_colors
 

jc/lib.py

@@ -251,7 +251,8 @@ def _is_valid_parser_plugin(name: str, local_parsers_dir: str) -> bool:
             else:
                 utils.warning_message([f'Not installing invalid parser plugin "{parser_mod_name}" at {local_parsers_dir}'])
                 return False
-        except Exception:
+        except Exception as e:
+            utils.warning_message([f'Not installing parser plugin "{parser_mod_name}" at {local_parsers_dir} due to error: {e}'])
             return False
     return False
 
@@ -324,7 +325,16 @@ def _get_parser(parser_mod_name: str) -> ModuleType:
     parser_mod_name = _cliname_to_modname(parser_mod_name)
     parser_cli_name = _modname_to_cliname(parser_mod_name)
     modpath: str = 'jcparsers.' if parser_cli_name in local_parsers else 'jc.parsers.'
-    return importlib.import_module(f'{modpath}{parser_mod_name}')
+    mod = None
+
+    try:
+        mod =  importlib.import_module(f'{modpath}{parser_mod_name}')
+    except Exception as e:
+        mod =  importlib.import_module(f'{modpath}disabled_parser')
+        mod.__name__ = parser_mod_name
+        utils.warning_message([f'"{parser_mod_name}" parser disabled due to error: {e}'])
+
+    return mod
 
 def _parser_is_slurpable(parser: ModuleType) -> bool:
     """

jc/parsers/broken_parser.py

@@ -0,0 +1,23 @@
+"""jc - JSON Convert broken parser - for testing purposes only"""
+import non_existent_library
+
+class info():
+    """Provides parser metadata (version, author, etc.)"""
+    version = '1.0'
+    description = 'broken parser'
+    author = 'N/A'
+    author_email = 'N/A'
+    compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    hidden = True
+
+
+__version__ = info.version
+
+
+def parse(
+    data: str,
+    raw: bool = False,
+    quiet: bool = False
+) -> dict:
+    """Main text parsing function"""
+    return {}

jc/parsers/disabled_parser.py

@@ -0,0 +1,22 @@
+"""jc - JSON Convert disabled parser"""
+
+class info():
+    """Provides parser metadata (version, author, etc.)"""
+    version = '1.0'
+    description = 'Disabled parser'
+    author = 'N/A'
+    author_email = 'N/A'
+    compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    hidden = True
+
+
+__version__ = info.version
+
+
+def parse(
+    data: str,
+    raw: bool = False,
+    quiet: bool = False
+) -> dict:
+    """Main text parsing function"""
+    return {}

jc/parsers/plist.py

@@ -44,6 +44,14 @@ Examples:
       ...
     }
 """
+import sys
+
+# ugly hack because I accidentally shadowed the xml module from the
+# standard library with the xml parser. :(
+for index, item in enumerate(sys.path.copy()):
+    if 'jc/jc/parsers' in item or '':
+        del sys.path[index]
+
 from typing import Dict, Union
 import plistlib
 import binascii
@@ -53,7 +61,7 @@ import jc.utils
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.1'
+    version = '1.2'
     description = 'PLIST file parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'

jc/parsers/xml.py

@@ -73,15 +73,10 @@ Examples:
 import jc.utils
 from jc.exceptions import LibraryNotInstalled
 
-try:
-    import xmltodict
-except Exception:
-    raise LibraryNotInstalled('The xmltodict library is not installed.')
-
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.9'
+    version = '1.10'
     description = 'XML file parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -93,7 +88,7 @@ class info():
 __version__ = info.version
 
 
-def _process(proc_data, has_data=False):
+def _process(proc_data, has_data=False, xml_mod=None):
     """
     Final processing to conform to the schema.
 
@@ -105,16 +100,19 @@ def _process(proc_data, has_data=False):
 
         Dictionary representing an XML document.
     """
+    if not xml_mod:
+        raise LibraryNotInstalled('The xmltodict library is not installed.')
+
     proc_output = []
 
     if has_data:
         # standard output with @ prefix for attributes
         try:
-            proc_output = xmltodict.parse(proc_data,
+            proc_output = xml_mod.parse(proc_data,
                                           dict_constructor=dict,
                                           process_comments=True)
         except (ValueError, TypeError):
-            proc_output = xmltodict.parse(proc_data, dict_constructor=dict)
+            proc_output = xml_mod.parse(proc_data, dict_constructor=dict)
 
     return proc_output
 
@@ -133,6 +131,12 @@ def parse(data, raw=False, quiet=False):
 
         Dictionary. Raw or processed structured data.
     """
+    xmltodict = None
+    try:
+        import xmltodict
+    except Exception:
+        raise LibraryNotInstalled('The xmltodict library is not installed.')
+
     jc.utils.compatibility(__name__, info.compatible, quiet)
     jc.utils.input_type_check(data)
 
@@ -156,4 +160,4 @@ def parse(data, raw=False, quiet=False):
 
         return raw_output
 
-    return _process(data, has_data)
+    return _process(data, has_data, xml_mod=xmltodict)

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2024-02-07 1.25.1 "JSON Convert"
+.TH jc 1 2024-02-10 1.25.1 "JSON Convert"
 .SH NAME
 \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types,
 and strings

runtests-missing-libs.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# system should be in "America/Los_Angeles" timezone for all tests to pass
+# ensure no local plugin parsers are installed for all tests to pass
+
+pip uninstall pygments ruamel.yaml xmltodict --yes
+python3 -m unittest -v
+pip install pygments ruamel.yaml xmltodict

tests/test_jc_cli.py

@@ -1,12 +1,20 @@
 import os
 import unittest
 from datetime import datetime, timezone
+
 try:
     import pygments
     from pygments.token import (Name, Number, String, Keyword)
     PYGMENTS_INSTALLED=True
-except ModuleNotFoundError:
+except:
     PYGMENTS_INSTALLED=False
+
+try:
+    import ruamel.yaml
+    RUAMELYAML_INSTALLED = True
+except:
+    RUAMELYAML_INSTALLED = False
+
 from jc.cli import JcCli
 import jc.parsers.url as url_parser
 import jc.parsers.proc as proc_parser
@@ -47,8 +55,8 @@ class MyTests(unittest.TestCase):
             resulting_attributes = (cli.magic_found_parser, cli.magic_options, cli.magic_run_command)
             self.assertEqual(expected, resulting_attributes)
 
+    @unittest.skipIf(not PYGMENTS_INSTALLED, 'pygments library not installed')
     def test_cli_set_env_colors(self):
-        if PYGMENTS_INSTALLED:
         if pygments.__version__.startswith('2.3.'):
             env = {
                 '': {
@@ -146,8 +154,8 @@ class MyTests(unittest.TestCase):
             cli.set_custom_colors()
             self.assertEqual(cli.custom_colors, expected_colors)
 
+    @unittest.skipIf(not PYGMENTS_INSTALLED, 'pygments library not installed')
     def test_cli_json_out(self):
-        if PYGMENTS_INSTALLED:
         test_input = [
             None,
             {},
@@ -180,8 +188,8 @@ class MyTests(unittest.TestCase):
             cli.data_out = test_dict
             self.assertEqual(cli.json_out(), expected_json)
 
+    @unittest.skipIf(not PYGMENTS_INSTALLED, 'pygments library not installed')
     def test_cli_json_out_mono(self):
-        if PYGMENTS_INSTALLED:
         test_input = [
             None,
             {},
@@ -205,6 +213,7 @@ class MyTests(unittest.TestCase):
             cli.data_out = test_dict
             self.assertEqual(cli.json_out(), expected_json)
 
+    @unittest.skipIf(not PYGMENTS_INSTALLED, 'pygments library not installed')
     def test_cli_json_out_pretty(self):
         test_input = [
             {"key1": "value1", "key2": 2, "key3": None, "key4": 3.14, "key5": True},
@@ -229,8 +238,27 @@ class MyTests(unittest.TestCase):
             cli.data_out = test_dict
             self.assertEqual(cli.json_out(), expected_json)
 
+    @unittest.skipIf(PYGMENTS_INSTALLED, 'pygments library installed')
+    def test_cli_json_out_pretty_no_pygments(self):
+        test_input = [
+            {"key1": "value1", "key2": 2, "key3": None, "key4": 3.14, "key5": True},
+            {"key1": [{"subkey1": "subvalue1"}, {"subkey2": [1, 2, 3]}], "key2": True}
+        ]
+
+        expected_output = [
+            '{\n  "key1": "value1",\n  "key2": 2,\n  "key3": null,\n  "key4": 3.14,\n  "key5": true\n}',
+            '{\n  "key1": [\n    {\n      "subkey1": "subvalue1"\n    },\n    {\n      "subkey2": [\n        1,\n        2,\n        3\n      ]\n    }\n  ],\n  "key2": true\n}'
+        ]
+
+        for test_dict, expected_json in zip(test_input, expected_output):
+            cli = JcCli()
+            cli.pretty = True
+            cli.set_custom_colors()
+            cli.data_out = test_dict
+            self.assertEqual(cli.json_out(), expected_json)
+
+    @unittest.skipIf(not PYGMENTS_INSTALLED, 'pygments library not installed')
     def test_cli_yaml_out(self):
-        if PYGMENTS_INSTALLED:
         test_input = [
             None,
             {},
@@ -263,6 +291,7 @@ class MyTests(unittest.TestCase):
             cli.data_out = test_dict
             self.assertEqual(cli.yaml_out(), expected_json)
 
+    @unittest.skipIf(not RUAMELYAML_INSTALLED, 'ruamel.yaml library not installed')
     def test_cli_yaml_out_mono(self):
         test_input = [
             None,

tests/test_jc_lib.py

@@ -11,6 +11,12 @@ class MyTests(unittest.TestCase):
         p = jc.lib.get_parser('arp')
         self.assertIsInstance(p, ModuleType)
 
+    def test_lib_get_parser_broken_parser(self):
+        """get_parser substitutes the disabled_parser if a parser is broken"""
+        broken = jc.lib.get_parser('broken_parser')
+        disabled = jc.lib.get_parser('disabled_parser')
+        self.assertIs(broken, disabled)
+
     def test_lib_get_parser_module(self):
         p = jc.lib.get_parser(csv_parser)
         self.assertIsInstance(p, ModuleType)

tests/test_xml.py

@@ -2,7 +2,6 @@ import os
 import unittest
 import json
 import jc.parsers.xml
-import xmltodict
 
 # fix for whether tests are run directly or via runtests.sh
 try:
@@ -10,10 +9,18 @@ try:
 except:
     from _vendor.packaging import version  # type: ignore
 
+# check the version of installed xmltodict library
+try:
+    import xmltodict
+    XMLTODICT_INSTALLED = True
+    XMLTODICT_0_13_0_OR_HIGHER = version.parse(xmltodict.__version__) >= version.parse('0.13.0')  # type: ignore
+except:
+    XMLTODICT_INSTALLED = False
+
+
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
-XMLTODICT_0_13_0_OR_HIGHER = version.parse(xmltodict.__version__) >= version.parse('0.13.0')
-
 
+@unittest.skipIf(not XMLTODICT_INSTALLED, 'xmltodict library not installed')
 class MyTests(unittest.TestCase):
 
     # input

tests/test_yaml.py

@@ -5,7 +5,14 @@ import jc.parsers.yaml
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 
+try:
+    import ruamel.yaml
+    RUAMELYAML_INSTALLED = True
+except:
+    RUAMELYAML_INSTALLED = False
 
+
+@unittest.skipIf(not RUAMELYAML_INSTALLED, 'ruamel.yaml library not installed')
 class MyTests(unittest.TestCase):
 
     # input