Merge pull request #238 from kellyjonbrazil/master

sync to dev

CHANGELOG

@@ -1,13 +1,19 @@
 jc changelog
 
+20220427 v1.18.8
+- Fix update-alternatives --query parser for cases where `slaves` are not present
+- Fix UnicodeEncodeError on some systems where LANG=C is set and unicode
+  characters are in the output
+- Update history parser: do not drop non-ASCII characters if the system
+  is configured for UTF-8 encoding
+- Enhance "magic syntax" to always use UTF-8 encoding
+
 20220425 v1.18.7
 - Add git log command parser
 - Add update-alternatives --query parser
 - Add update-alternatives --get-selections parser
 - Fix key/value and ini parsers to allow duplicate keys
 - Fix yaml file parser for files including timestamp objects
-- Fix UnicodeDecodeError on some systems where LANG=C is set and unicode
-  characters are in the output
 - Update xrandr parser: add a 'rotation' field
 - Fix failing tests by moving template files
 - Add python interpreter version and path to -v and -a output

EXAMPLES.md

@@ -1059,6 +1059,53 @@ cat /etc/fstab | jc --fstab -p
   }
 ]
 ```
+### git log
+```bash
+git log --stat | jc --git-log -p          or:  jc -p git log --stat
+```
+```json
+[
+  {
+    "commit": "728d882ed007b3c8b785018874a0eb06e1143b66",
+    "author": "Kelly Brazil",
+    "author_email": "kellyjonbrazil@gmail.com",
+    "date": "Wed Apr 20 09:50:19 2022 -0400",
+    "stats": {
+      "files_changed": 2,
+      "insertions": 90,
+      "deletions": 12,
+      "files": [
+        "docs/parsers/git_log.md",
+        "jc/parsers/git_log.py"
+      ]
+    },
+    "message": "add timestamp docs and examples",
+    "epoch": 1650462619,
+    "epoch_utc": null
+  },
+  {
+    "commit": "b53e42aca623181aa9bc72194e6eeef1e9a3a237",
+    "author": "Kelly Brazil",
+    "author_email": "kellyjonbrazil@gmail.com",
+    "date": "Wed Apr 20 09:44:42 2022 -0400",
+    "stats": {
+      "files_changed": 5,
+      "insertions": 29,
+      "deletions": 6,
+      "files": [
+        "docs/parsers/git_log.md",
+        "docs/utils.md",
+        "jc/parsers/git_log.py",
+        "jc/utils.py",
+        "man/jc.1"
+      ]
+    },
+    "message": "add calculated timestamp",
+    "epoch": 1650462282,
+    "epoch_utc": null
+  }
+]
+```
 ### /etc/group file
 ```bash
 cat /etc/group | jc --group -p

README.md

@@ -425,6 +425,9 @@ or by exporting to the environment before running commands:
 $ export LANG=C
 ```
 
+On some older systems UTF-8 output will be downgraded to ASCII with `\\u`
+escape sequences if the `C` locale does not support UTF-8 encoding.
+
 #### Timezones
 
 Some parsers have calculated epoch timestamp fields added to the output. Unless

docs/parsers/history.md

@@ -87,4 +87,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, aix, freebsd
 
-Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/update_alt_q.md

@@ -154,4 +154,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

jc/cli.py

@@ -37,7 +37,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     website = 'https://github.com/kellyjonbrazil/jc'
-    copyright = f'© 2019-2022 Kelly Brazil'
+    copyright = '© 2019-2022 Kelly Brazil'
     license = 'MIT License'
 
 
@@ -84,14 +84,24 @@ if PYGMENTS_INSTALLED:
         }
 
 
-def asciify(string):
+def safe_print_json(string, pretty=None, env_colors=None, mono=None,
-    """
+                    piped_out=None, flush=None):
-    Return a string downgraded from Unicode to ASCII with some simple
+    """Safely prints JSON output in both UTF-8 and ASCII systems"""
-    conversions.
+    try:
-    """
+        print(json_out(string,
-    string = string.replace('©', '(c)')
+                       pretty=pretty,
-    string = ascii(string)
+                       env_colors=env_colors,
-    return string.replace(r'\n', '\n')
+                       mono=mono,
+                       piped_out=piped_out),
+              flush=flush)
+    except UnicodeEncodeError:
+        print(json_out(string,
+                       pretty=pretty,
+                       env_colors=env_colors,
+                       mono=mono,
+                       piped_out=piped_out,
+                       ascii_only=True),
+              flush=flush)
 
 
 def set_env_colors(env_colors=None):
@@ -268,11 +278,12 @@ def versiontext():
     python path:  {sys.executable}
 
     {info.website}
-    {info.copyright}'''
+    {info.copyright}
+    '''
     return textwrap.dedent(versiontext_string)
 
 
-def json_out(data, pretty=False, env_colors=None, mono=False, piped_out=False):
+def json_out(data, pretty=False, env_colors=None, mono=False, piped_out=False, ascii_only=False):
     """
     Return a JSON formatted string. String may include color codes or be
     pretty printed.
@@ -284,28 +295,16 @@ def json_out(data, pretty=False, env_colors=None, mono=False, piped_out=False):
         separators = None
         indent = 2
 
+    j_string = json.dumps(data, indent=indent, separators=separators, ensure_ascii=ascii_only)
+
     if not mono and not piped_out:
         # set colors
         class JcStyle(Style):
             styles = set_env_colors(env_colors)
 
-        try:
+        return str(highlight(j_string, JsonLexer(), Terminal256Formatter(style=JcStyle))[0:-1])
-            return str(highlight(json.dumps(data,
-                                            indent=indent,
-                                            separators=separators,
-                                            ensure_ascii=False),
-                                 JsonLexer(), Terminal256Formatter(style=JcStyle))[0:-1])
-        except UnicodeEncodeError:
-            return str(highlight(json.dumps(data,
-                                            indent=indent,
-                                            separators=separators,
-                                            ensure_ascii=True),
-                                 JsonLexer(), Terminal256Formatter(style=JcStyle))[0:-1])
 
-    try:
+    return j_string
-        return json.dumps(data, indent=indent, separators=separators, ensure_ascii=False)
-    except UnicodeEncodeError:
-        return json.dumps(data, indent=indent, separators=separators, ensure_ascii=True)
 
 
 def magic_parser(args):
@@ -376,7 +375,8 @@ def run_user_command(command):
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             close_fds=False,            # Allows inheriting file descriptors;
-                            universal_newlines=True)    # useful for process substitution
+                            universal_newlines=True,    #     useful for process substitution
+                            encoding='UTF-8')
     stdout, stderr = proc.communicate()
 
     return (
@@ -443,25 +443,19 @@ def main():
         mono = True
 
     if about:
-        print(json_out(about_jc(),
+        safe_print_json(about_jc(),
                         pretty=pretty,
                         env_colors=jc_colors,
                         mono=mono,
-              piped_out=piped_output(force_color)))
+                        piped_out=piped_output(force_color))
         sys.exit(0)
 
     if help_me:
-        try:
+        utils._safe_print(help_doc(sys.argv))
-            print(help_doc(sys.argv))
-        except UnicodeEncodeError:
-            print(asciify(help_doc(sys.argv)))
         sys.exit(0)
 
     if version_info:
-        try:
+        utils._safe_print(versiontext())
-            print(versiontext())
-        except UnicodeEncodeError:
-            print(asciify(versiontext()))
         sys.exit(0)
 
     # if magic syntax used, try to run the command and error if it's not found, etc.
@@ -476,7 +470,7 @@ def main():
         try:
             magic_stdout, magic_stderr, magic_exit_code = run_user_command(run_command)
             if magic_stderr:
-                print(magic_stderr[:-1], file=sys.stderr)
+                utils._safe_print(magic_stderr[:-1], file=sys.stderr)
 
         except OSError as e:
             if debug:
@@ -540,11 +534,11 @@ def main():
                                   quiet=quiet,
                                   ignore_exceptions=ignore_exceptions)
             for line in result:
-                print(json_out(line,
+                safe_print_json(line,
                                 pretty=pretty,
                                 env_colors=jc_colors,
                                 mono=mono,
-                               piped_out=piped_output(force_color)),
+                                piped_out=piped_output(force_color),
                                 flush=unbuffer)
 
             sys.exit(combined_exit_code(magic_exit_code, 0))
@@ -555,11 +549,12 @@ def main():
             result = parser.parse(data,
                                   raw=raw,
                                   quiet=quiet)
-            print(json_out(result,
+
+            safe_print_json(result,
                             pretty=pretty,
                             env_colors=jc_colors,
                             mono=mono,
-                           piped_out=piped_output(force_color)),
+                            piped_out=piped_output(force_color),
                             flush=unbuffer)
 
         sys.exit(combined_exit_code(magic_exit_code, 0))

jc/lib.py

@@ -6,7 +6,7 @@ import importlib
 from typing import Dict, List, Iterable, Union, Iterator
 from jc import appdirs
 
-__version__ = '1.18.7'
+__version__ = '1.18.8'
 
 parsers = [
     'acpi',

jc/parsers/history.py

@@ -63,7 +63,7 @@ import jc.utils
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.6'
+    version = '1.7'
     description = '`history` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -117,17 +117,14 @@ def parse(data, raw=False, quiet=False):
     raw_output = {}
 
     if jc.utils.has_data(data):
+        linedata = data.splitlines()
 
-        # split lines and clear out any non-ascii chars
-        linedata = data.encode('ascii', errors='ignore').decode().splitlines()
-
-        # Skip any blank lines
         for entry in filter(None, linedata):
             try:
-                parsed_line = entry.split(maxsplit=1)
+                number, command = entry.split(maxsplit=1)
-                raw_output[parsed_line[0]] = parsed_line[1]
+                raw_output[number] = command
-            except IndexError:
+            except ValueError:
-                # need to catch indexerror in case there is weird input from prior commands
+                # need to catch ValueError in case there is weird input from prior commands
                 pass
 
     if raw:

jc/parsers/update_alt_q.py

@@ -132,7 +132,7 @@ import jc.utils
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.0'
+    version = '1.1'
     description = '`update-alternatives --query` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -241,11 +241,13 @@ def parse(
                 if not 'alternatives' in raw_output:
                     raw_output['alternatives'] = []
 
+                if alt_obj:
                     if slaves:
                         alt_obj['slaves'] = slaves
-                    raw_output['alternatives'].append(alt_obj)
                         slaves = []
 
+                    raw_output['alternatives'].append(alt_obj)
+
                 alt_obj = {"alternative": line_list[1]}
                 continue
 

jc/utils.py

@@ -9,6 +9,26 @@ from functools import lru_cache
 from typing import List, Iterable, Union, Optional
 
 
+def _asciify(string: str) -> str:
+    """
+    Return a string downgraded from Unicode to ASCII with some simple
+    conversions.
+    """
+    string = string.replace('©', '(c)')
+    # the ascii() function adds single quotes around the string
+    string = ascii(string)[1:-1]
+    string = string.replace(r'\n', '\n')
+    return string
+
+
+def _safe_print(string: str, sep=' ', end='\n', file=sys.stdout, flush=False) -> None:
+    """Output for both UTF-8 and ASCII encoding systems"""
+    try:
+        print(string, sep=sep, end=end, file=file, flush=flush)
+    except UnicodeEncodeError:
+        print(_asciify(string), sep=sep, end=end, file=file, flush=flush)
+
+
 def warning_message(message_lines: List[str]) -> None:
     """
     Prints warning message for non-fatal issues. The first line is
@@ -36,13 +56,13 @@ def warning_message(message_lines: List[str]) -> None:
     first_line = message_lines.pop(0)
     first_str = f'jc:  Warning - {first_line}'
     first_str = first_wrapper.fill(first_str)
-    print(first_str, file=sys.stderr)
+    _safe_print(first_str, file=sys.stderr)
 
     for line in message_lines:
         if line == '':
             continue
         message = next_wrapper.fill(line)
-        print(message, file=sys.stderr)
+        _safe_print(message, file=sys.stderr)
 
 
 def error_message(message_lines: List[str]) -> None:
@@ -68,13 +88,13 @@ def error_message(message_lines: List[str]) -> None:
     first_line = message_lines.pop(0)
     first_str = f'jc:  Error - {first_line}'
     first_str = first_wrapper.fill(first_str)
-    print(first_str, file=sys.stderr)
+    _safe_print(first_str, file=sys.stderr)
 
     for line in message_lines:
         if line == '':
             continue
         message = next_wrapper.fill(line)
-        print(message, file=sys.stderr)
+        _safe_print(message, file=sys.stderr)
 
 
 def compatibility(mod_name: str, compatible: List, quiet: bool = False) -> None:

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2022-04-25 1.18.7 "JSON Convert"
+.TH jc 1 2022-04-27 1.18.8 "JSON Convert"
 .SH NAME
 jc \- JSONifies the output of many CLI tools and file-types
 .SH SYNOPSIS
@@ -686,6 +686,8 @@ or by exporting to the environment before running commands:
 $ export LANG=C
 .RE
 
+On some older systems UTF-8 output will be downgraded to ASCII with `\\u` escape sequences if the \fBC\fP locale does not support UTF-8 encoding.
+
 \fBTimezones:\fP Some parsers have calculated epoch timestamp fields added to the output. Unless a timestamp field name has a \fB_utc\fP suffix it is considered naive. (i.e. based on the local timezone of the system the \fBjc\fP parser was run on).
 
 If a UTC timezone can be detected in the text of the command output, the timestamp will be timezone aware and have a \fB_utc\fP suffix on the key name. (e.g. \fBepoch_utc\fP) No other timezones are supported for aware timestamps.

setup.py

@@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
 
 setuptools.setup(
     name='jc',
-    version='1.18.7',
+    version='1.18.8',
     author='Kelly Brazil',
     author_email='kellyjonbrazil@gmail.com',
     description='Converts the output of popular command-line tools and file-types to JSON.',

templates/manpage_template

@@ -201,6 +201,8 @@ or by exporting to the environment before running commands:
 $ export LANG=C
 .RE
 
+On some older systems UTF-8 output will be downgraded to ASCII with `\\u` escape sequences if the \fBC\fP locale does not support UTF-8 encoding.
+
 \fBTimezones:\fP Some parsers have calculated epoch timestamp fields added to the output. Unless a timestamp field name has a \fB_utc\fP suffix it is considered naive. (i.e. based on the local timezone of the system the \fBjc\fP parser was run on).
 
 If a UTC timezone can be detected in the text of the command output, the timestamp will be timezone aware and have a \fB_utc\fP suffix on the key name. (e.g. \fBepoch_utc\fP) No other timezones are supported for aware timestamps.

templates/readme_template

@@ -328,6 +328,9 @@ or by exporting to the environment before running commands:
 $ export LANG=C
 ```
 
+On some older systems UTF-8 output will be downgraded to ASCII with `\\u`
+escape sequences if the `C` locale does not support UTF-8 encoding.
+
 #### Timezones
 
 Some parsers have calculated epoch timestamp fields added to the output. Unless

tests/fixtures/centos-7.7/history.out

@@ -998,3 +998,4 @@
  1062  ls
  1063  cd testfiles/
  1064  history > history.out 
+ 1065  export MYTEST=©2019-2022

tests/fixtures/generic/update-alternatives-query2.json

@@ -0,0 +1 @@
+{"name":"php-fpm.sock","link":"/run/php/php-fpm.sock","status":"auto","best":"/run/php/php8.1-fpm.sock","value":"/run/php/php8.1-fpm.sock","alternatives":[{"alternative":"/run/php/php7.4-fpm.sock","priority":74},{"alternative":"/run/php/php8.0-fpm.sock","priority":80},{"alternative":"/run/php/php8.1-fpm.sock","priority":81}]}

tests/fixtures/generic/update-alternatives-query2.out

@@ -0,0 +1,14 @@
+Name: php-fpm.sock
+Link: /run/php/php-fpm.sock
+Status: auto
+Best: /run/php/php8.1-fpm.sock
+Value: /run/php/php8.1-fpm.sock
+
+Alternative: /run/php/php7.4-fpm.sock
+Priority: 74
+
+Alternative: /run/php/php8.0-fpm.sock
+Priority: 80
+
+Alternative: /run/php/php8.1-fpm.sock
+Priority: 81

tests/test_update_alt_q.py

@@ -13,10 +13,17 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/update-alternatives-query.out'), 'r', encoding='utf-8') as f:
             self.update_alternatives_query = f.read()
 
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/update-alternatives-query2.out'), 'r', encoding='utf-8') as f:
+            self.update_alternatives_query2 = f.read()
+
         # output
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/update-alternatives-query.json'), 'r', encoding='utf-8') as f:
             self.update_alternatives_query_json = json.loads(f.read())
 
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/update-alternatives-query2.json'), 'r', encoding='utf-8') as f:
+            self.update_alternatives_query2_json = json.loads(f.read())
+
+
 
     def test_update_alt_q_nodata(self):
         """
@@ -30,6 +37,12 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(jc.parsers.update_alt_q.parse(self.update_alternatives_query, quiet=True), self.update_alternatives_query_json)
 
+    def test_update_alt_q_no_slaves(self):
+        """
+        Test 'update-alternatives --query' with no slaves in output
+        """
+        self.assertEqual(jc.parsers.update_alt_q.parse(self.update_alternatives_query2, quiet=True), self.update_alternatives_query2_json)
+
 
 if __name__ == '__main__':
     unittest.main()