python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-19 00:17:51 +02:00
Code Issues Releases Activity

Merge pull request #238 from kellyjonbrazil/master

Browse Source 
sync to dev
This commit is contained in:
Kelly Brazil
2022-04-28 16:21:22 +00:00
committed by GitHub
parent 49ca0cecf4 7608823ea7
commit 9c8730786b
19 changed files with 185 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
CHANGELOG
View File

@ -1,13 +1,19 @@
jc changelog
20220427 v1.18.8
- Fix update-alternatives --query parser for cases where `slaves` are not present
- Fix UnicodeEncodeError on some systems where LANG=C is set and unicode
characters are in the output
- Update history parser: do not drop non-ASCII characters if the system
is configured for UTF-8 encoding
- Enhance "magic syntax" to always use UTF-8 encoding
20220425 v1.18.7
- Add git log command parser
- Add update-alternatives --query parser
- Add update-alternatives --get-selections parser
- Fix key/value and ini parsers to allow duplicate keys
- Fix yaml file parser for files including timestamp objects
- Fix UnicodeDecodeError on some systems where LANG=C is set and unicode
characters are in the output
- Update xrandr parser: add a 'rotation' field
- Fix failing tests by moving template files
- Add python interpreter version and path to -v and -a output

47
EXAMPLES.md
View File

@ -1059,6 +1059,53 @@ cat /etc/fstab | jc --fstab -p
}
]
```
### git log
```bash
git log --stat | jc --git-log -p or: jc -p git log --stat
```
```json
[
{
"commit": "728d882ed007b3c8b785018874a0eb06e1143b66",
"author": "Kelly Brazil",
"author_email": "kellyjonbrazil@gmail.com",
"date": "Wed Apr 20 09:50:19 2022 -0400",
"stats": {
"files_changed": 2,
"insertions": 90,
"deletions": 12,
"files": [
"docs/parsers/git_log.md",
"jc/parsers/git_log.py"
]
},
"message": "add timestamp docs and examples",
"epoch": 1650462619,
"epoch_utc": null
},
{
"commit": "b53e42aca623181aa9bc72194e6eeef1e9a3a237",
"author": "Kelly Brazil",
"author_email": "kellyjonbrazil@gmail.com",
"date": "Wed Apr 20 09:44:42 2022 -0400",
"stats": {
"files_changed": 5,
"insertions": 29,
"deletions": 6,
"files": [
"docs/parsers/git_log.md",
"docs/utils.md",
"jc/parsers/git_log.py",
"jc/utils.py",
"man/jc.1"
]
},
"message": "add calculated timestamp",
"epoch": 1650462282,
"epoch_utc": null
}
]
```
### /etc/group file
```bash
cat /etc/group | jc --group -p

3
README.md
View File

@ -425,6 +425,9 @@ or by exporting to the environment before running commands:
$ export LANG=C
```
On some older systems UTF-8 output will be downgraded to ASCII with `\\u`
escape sequences if the `C` locale does not support UTF-8 encoding.
#### Timezones
Some parsers have calculated epoch timestamp fields added to the output. Unless

2
docs/parsers/history.md
View File

@ -87,4 +87,4 @@ Returns:
### Parser Information
Compatibility: linux, darwin, cygwin, aix, freebsd
Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com)

2
docs/parsers/update_alt_q.md
View File

@ -154,4 +154,4 @@ Returns:
### Parser Information
Compatibility: linux
Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

81
jc/cli.py
View File

@ -37,7 +37,7 @@ class info():
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
website = 'https://github.com/kellyjonbrazil/jc'
copyright = f'© 2019-2022 Kelly Brazil'
copyright = '© 2019-2022 Kelly Brazil'
license = 'MIT License'
@ -84,14 +84,24 @@ if PYGMENTS_INSTALLED:
}
def asciify(string):
"""
Return a string downgraded from Unicode to ASCII with some simple
conversions.
"""
string = string.replace('©', '(c)')
string = ascii(string)
return string.replace(r'\n', '\n')
def safe_print_json(string, pretty=None, env_colors=None, mono=None,
piped_out=None, flush=None):
"""Safely prints JSON output in both UTF-8 and ASCII systems"""
try:
print(json_out(string,
pretty=pretty,
env_colors=env_colors,
mono=mono,
piped_out=piped_out),
flush=flush)
except UnicodeEncodeError:
print(json_out(string,
pretty=pretty,
env_colors=env_colors,
mono=mono,
piped_out=piped_out,
ascii_only=True),
flush=flush)
def set_env_colors(env_colors=None):
@ -268,11 +278,12 @@ def versiontext():
python path: {sys.executable}
{info.website}
{info.copyright}'''
{info.copyright}
'''
return textwrap.dedent(versiontext_string)
def json_out(data, pretty=False, env_colors=None, mono=False, piped_out=False):
def json_out(data, pretty=False, env_colors=None, mono=False, piped_out=False, ascii_only=False):
"""
Return a JSON formatted string. String may include color codes or be
pretty printed.
@ -284,28 +295,16 @@ def json_out(data, pretty=False, env_colors=None, mono=False, piped_out=False):
separators = None
indent = 2
j_string = json.dumps(data, indent=indent, separators=separators, ensure_ascii=ascii_only)
if not mono and not piped_out:
# set colors
class JcStyle(Style):
styles = set_env_colors(env_colors)
try:
return str(highlight(json.dumps(data,
indent=indent,
separators=separators,
ensure_ascii=False),
JsonLexer(), Terminal256Formatter(style=JcStyle))[0:-1])
except UnicodeEncodeError:
return str(highlight(json.dumps(data,
indent=indent,
separators=separators,
ensure_ascii=True),
JsonLexer(), Terminal256Formatter(style=JcStyle))[0:-1])
return str(highlight(j_string, JsonLexer(), Terminal256Formatter(style=JcStyle))[0:-1])
try:
return json.dumps(data, indent=indent, separators=separators, ensure_ascii=False)
except UnicodeEncodeError:
return json.dumps(data, indent=indent, separators=separators, ensure_ascii=True)
return j_string
def magic_parser(args):
@ -376,7 +375,8 @@ def run_user_command(command):
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=False, # Allows inheriting file descriptors;
universal_newlines=True) # useful for process substitution
universal_newlines=True, # useful for process substitution
encoding='UTF-8')
stdout, stderr = proc.communicate()
return (
@ -443,25 +443,19 @@ def main():
mono = True
if about:
print(json_out(about_jc(),
safe_print_json(about_jc(),
pretty=pretty,
env_colors=jc_colors,
mono=mono,
piped_out=piped_output(force_color)))
piped_out=piped_output(force_color))
sys.exit(0)
if help_me:
try:
print(help_doc(sys.argv))
except UnicodeEncodeError:
print(asciify(help_doc(sys.argv)))
utils._safe_print(help_doc(sys.argv))
sys.exit(0)
if version_info:
try:
print(versiontext())
except UnicodeEncodeError:
print(asciify(versiontext()))
utils._safe_print(versiontext())
sys.exit(0)
# if magic syntax used, try to run the command and error if it's not found, etc.
@ -476,7 +470,7 @@ def main():
try:
magic_stdout, magic_stderr, magic_exit_code = run_user_command(run_command)
if magic_stderr:
print(magic_stderr[:-1], file=sys.stderr)
utils._safe_print(magic_stderr[:-1], file=sys.stderr)
except OSError as e:
if debug:
@ -540,11 +534,11 @@ def main():
quiet=quiet,
ignore_exceptions=ignore_exceptions)
for line in result:
print(json_out(line,
safe_print_json(line,
pretty=pretty,
env_colors=jc_colors,
mono=mono,
piped_out=piped_output(force_color)),
piped_out=piped_output(force_color),
flush=unbuffer)
sys.exit(combined_exit_code(magic_exit_code, 0))
@ -555,11 +549,12 @@ def main():
result = parser.parse(data,
raw=raw,
quiet=quiet)
print(json_out(result,
safe_print_json(result,
pretty=pretty,
env_colors=jc_colors,
mono=mono,
piped_out=piped_output(force_color)),
piped_out=piped_output(force_color),
flush=unbuffer)
sys.exit(combined_exit_code(magic_exit_code, 0))

2
jc/lib.py
View File

@ -6,7 +6,7 @@ import importlib
from typing import Dict, List, Iterable, Union, Iterator
from jc import appdirs
__version__ = '1.18.7'
__version__ = '1.18.8'
parsers = [
'acpi',

15
jc/parsers/history.py
View File

@ -63,7 +63,7 @@ import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.6'
version = '1.7'
description = '`history` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -117,17 +117,14 @@ def parse(data, raw=False, quiet=False):
raw_output = {}
if jc.utils.has_data(data):
linedata = data.splitlines()
# split lines and clear out any non-ascii chars
linedata = data.encode('ascii', errors='ignore').decode().splitlines()
# Skip any blank lines
for entry in filter(None, linedata):
try:
parsed_line = entry.split(maxsplit=1)
raw_output[parsed_line[0]] = parsed_line[1]
except IndexError:
# need to catch indexerror in case there is weird input from prior commands
number, command = entry.split(maxsplit=1)
raw_output[number] = command
except ValueError:
# need to catch ValueError in case there is weird input from prior commands
pass
if raw:

6
jc/parsers/update_alt_q.py
View File

@ -132,7 +132,7 @@ import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.0'
version = '1.1'
description = '`update-alternatives --query` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -241,11 +241,13 @@ def parse(
if not 'alternatives' in raw_output:
raw_output['alternatives'] = []
if alt_obj:
if slaves:
alt_obj['slaves'] = slaves
raw_output['alternatives'].append(alt_obj)
slaves = []
raw_output['alternatives'].append(alt_obj)
alt_obj = {"alternative": line_list[1]}
continue

28
jc/utils.py
View File

@ -9,6 +9,26 @@ from functools import lru_cache
from typing import List, Iterable, Union, Optional
def _asciify(string: str) -> str:
"""
Return a string downgraded from Unicode to ASCII with some simple
conversions.
"""
string = string.replace('©', '(c)')
# the ascii() function adds single quotes around the string
string = ascii(string)[1:-1]
string = string.replace(r'\n', '\n')
return string
def _safe_print(string: str, sep=' ', end='\n', file=sys.stdout, flush=False) -> None:
"""Output for both UTF-8 and ASCII encoding systems"""
try:
print(string, sep=sep, end=end, file=file, flush=flush)
except UnicodeEncodeError:
print(_asciify(string), sep=sep, end=end, file=file, flush=flush)
def warning_message(message_lines: List[str]) -> None:
"""
Prints warning message for non-fatal issues. The first line is
@ -36,13 +56,13 @@ def warning_message(message_lines: List[str]) -> None:
first_line = message_lines.pop(0)
first_str = f'jc: Warning - {first_line}'
first_str = first_wrapper.fill(first_str)
print(first_str, file=sys.stderr)
_safe_print(first_str, file=sys.stderr)
for line in message_lines:
if line == '':
continue
message = next_wrapper.fill(line)
print(message, file=sys.stderr)
_safe_print(message, file=sys.stderr)
def error_message(message_lines: List[str]) -> None:
@ -68,13 +88,13 @@ def error_message(message_lines: List[str]) -> None:
first_line = message_lines.pop(0)
first_str = f'jc: Error - {first_line}'
first_str = first_wrapper.fill(first_str)
print(first_str, file=sys.stderr)
_safe_print(first_str, file=sys.stderr)
for line in message_lines:
if line == '':
continue
message = next_wrapper.fill(line)
print(message, file=sys.stderr)
_safe_print(message, file=sys.stderr)
def compatibility(mod_name: str, compatible: List, quiet: bool = False) -> None:

4
man/jc.1
View File

@ -1,4 +1,4 @@
.TH jc 1 2022-04-25 1.18.7 "JSON Convert"
.TH jc 1 2022-04-27 1.18.8 "JSON Convert"
.SH NAME
jc \- JSONifies the output of many CLI tools and file-types
.SH SYNOPSIS
@ -686,6 +686,8 @@ or by exporting to the environment before running commands:
$ export LANG=C
.RE
On some older systems UTF-8 output will be downgraded to ASCII with `\\u` escape sequences if the \fBC\fP locale does not support UTF-8 encoding.
\fBTimezones:\fP Some parsers have calculated epoch timestamp fields added to the output. Unless a timestamp field name has a \fB_utc\fP suffix it is considered naive. (i.e. based on the local timezone of the system the \fBjc\fP parser was run on).
If a UTC timezone can be detected in the text of the command output, the timestamp will be timezone aware and have a \fB_utc\fP suffix on the key name. (e.g. \fBepoch_utc\fP) No other timezones are supported for aware timestamps.

2
setup.py
View File

@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
setuptools.setup(
name='jc',
version='1.18.7',
version='1.18.8',
author='Kelly Brazil',
author_email='kellyjonbrazil@gmail.com',
description='Converts the output of popular command-line tools and file-types to JSON.',

2
templates/manpage_template
View File

@ -201,6 +201,8 @@ or by exporting to the environment before running commands:
$ export LANG=C
.RE
On some older systems UTF-8 output will be downgraded to ASCII with `\\u` escape sequences if the \fBC\fP locale does not support UTF-8 encoding.
\fBTimezones:\fP Some parsers have calculated epoch timestamp fields added to the output. Unless a timestamp field name has a \fB_utc\fP suffix it is considered naive. (i.e. based on the local timezone of the system the \fBjc\fP parser was run on).
If a UTC timezone can be detected in the text of the command output, the timestamp will be timezone aware and have a \fB_utc\fP suffix on the key name. (e.g. \fBepoch_utc\fP) No other timezones are supported for aware timestamps.

3
templates/readme_template
View File

@ -328,6 +328,9 @@ or by exporting to the environment before running commands:
$ export LANG=C
```
On some older systems UTF-8 output will be downgraded to ASCII with `\\u`
escape sequences if the `C` locale does not support UTF-8 encoding.
#### Timezones
Some parsers have calculated epoch timestamp fields added to the output. Unless

2
tests/fixtures/centos-7.7/history.json vendored
View File

File diff suppressed because one or more lines are too long

1
tests/fixtures/centos-7.7/history.out vendored
View File

@ -998,3 +998,4 @@
1062 ls
1063 cd testfiles/
1064 history > history.out
1065 export MYTEST=©2019-2022

1
tests/fixtures/generic/update-alternatives-query2.json vendored Normal file
View File

@ -0,0 +1 @@
{"name":"php-fpm.sock","link":"/run/php/php-fpm.sock","status":"auto","best":"/run/php/php8.1-fpm.sock","value":"/run/php/php8.1-fpm.sock","alternatives":[{"alternative":"/run/php/php7.4-fpm.sock","priority":74},{"alternative":"/run/php/php8.0-fpm.sock","priority":80},{"alternative":"/run/php/php8.1-fpm.sock","priority":81}]}

14
tests/fixtures/generic/update-alternatives-query2.out vendored Normal file
View File

@ -0,0 +1,14 @@
Name: php-fpm.sock
Link: /run/php/php-fpm.sock
Status: auto
Best: /run/php/php8.1-fpm.sock
Value: /run/php/php8.1-fpm.sock
Alternative: /run/php/php7.4-fpm.sock
Priority: 74
Alternative: /run/php/php8.0-fpm.sock
Priority: 80
Alternative: /run/php/php8.1-fpm.sock
Priority: 81

13
tests/test_update_alt_q.py
View File

@ -13,10 +13,17 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/update-alternatives-query.out'), 'r', encoding='utf-8') as f:
self.update_alternatives_query = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/update-alternatives-query2.out'), 'r', encoding='utf-8') as f:
self.update_alternatives_query2 = f.read()
# output
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/update-alternatives-query.json'), 'r', encoding='utf-8') as f:
self.update_alternatives_query_json = json.loads(f.read())
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/update-alternatives-query2.json'), 'r', encoding='utf-8') as f:
self.update_alternatives_query2_json = json.loads(f.read())
def test_update_alt_q_nodata(self):
"""
@ -30,6 +37,12 @@ class MyTests(unittest.TestCase):
"""
self.assertEqual(jc.parsers.update_alt_q.parse(self.update_alternatives_query, quiet=True), self.update_alternatives_query_json)
def test_update_alt_q_no_slaves(self):
"""
Test 'update-alternatives --query' with no slaves in output
"""
self.assertEqual(jc.parsers.update_alt_q.parse(self.update_alternatives_query2, quiet=True), self.update_alternatives_query2_json)
if __name__ == '__main__':
unittest.main()