python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-17 00:07:37 +02:00
Code Issues Releases Activity

Merge pull request #256 from kellyjonbrazil/dev

Browse Source 
v1.20.1
This commit is contained in:
Kelly Brazil
2022-06-15 22:16:07 +00:00
committed by GitHub
parent 776ee66bae 7583f315ce
commit 7fd67fda13
33 changed files with 1202 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGELOG
View File

@ -1,5 +1,13 @@
jc changelog jc changelog
20220615 v1.20.1
- Add `postconf -M` parser tested on linux
- Update `asciitable` and `asciitable-m` parsers to preserve case in key
names when using the `-r` or `raw=True` options.
- Add long options (e.g. `--help`, `--about`, `--pretty`, etc.)
- Add shell completions for Bash and Zsh
- Fix `id` parser for cases where the user or group name is not present
20220531 v1.20.0 20220531 v1.20.0
- Add YAML output option with `-y` - Add YAML output option with `-y`
- Add `top -b` standard and streaming parsers tested on linux - Add `top -b` standard and streaming parsers tested on linux

129
EXAMPLES.md
View File

@ -2728,6 +2728,36 @@ pip show wrapt wheel | jc --pip-show -p # or: jc -p pip show wrapt whe
} }
] ]
``` ```
### postconf -M
```bash
postconf -M | jc --postconf -p # or jc -p postconf -M
```
```json
[
{
"service_name": "smtp",
"service_type": "inet",
"private": false,
"unprivileged": null,
"chroot": true,
"wake_up_time": null,
"process_limit": null,
"command": "smtpd",
"no_wake_up_before_first_use": null
},
{
"service_name": "pickup",
"service_type": "unix",
"private": false,
"unprivileged": null,
"chroot": true,
"wake_up_time": 60,
"process_limit": 1,
"command": "pickup",
"no_wake_up_before_first_use": false
}
]
```
### ps ### ps
```bash ```bash
ps -ef | jc --ps -p # or: jc -p ps -ef ps -ef | jc --ps -p # or: jc -p ps -ef
@ -3465,6 +3495,105 @@ timedatectl | jc --timedatectl -p # or: jc -p timedatectl
"epoch_utc": 1583888001 "epoch_utc": 1583888001
} }
``` ```
### tob -b
```bash
top -b -n 1 | jc --top -p # or jc -p tob -b -n 1
```
```json
[
{
"time": "11:20:43",
"uptime": 118,
"users": 2,
"load_1m": 0.0,
"load_5m": 0.01,
"load_15m": 0.05,
"tasks_total": 108,
"tasks_running": 2,
"tasks_sleeping": 106,
"tasks_stopped": 0,
"tasks_zombie": 0,
"cpu_user": 5.6,
"cpu_sys": 11.1,
"cpu_nice": 0.0,
"cpu_idle": 83.3,
"cpu_wait": 0.0,
"cpu_hardware": 0.0,
"cpu_software": 0.0,
"cpu_steal": 0.0,
"mem_total": 3.7,
"mem_free": 3.3,
"mem_used": 0.2,
"mem_buff_cache": 0.2,
"swap_total": 2.0,
"swap_free": 2.0,
"swap_used": 0.0,
"mem_available": 3.3,
"processes": [
{
"pid": 2225,
"user": "kbrazil",
"priority": 20,
"nice": 0,
"virtual_mem": 158.1,
"resident_mem": 2.2,
"shared_mem": 1.6,
"status": "running",
"percent_cpu": 12.5,
"percent_mem": 0.1,
"time_hundredths": "0:00.02",
"command": "top",
"parent_pid": 1884,
"uid": 1000,
"real_uid": 1000,
"real_user": "kbrazil",
"saved_uid": 1000,
"saved_user": "kbrazil",
"gid": 1000,
"group": "kbrazil",
"pgrp": 2225,
"tty": "pts/0",
"tty_process_gid": 2225,
"session_id": 1884,
"thread_count": 1,
"last_used_processor": 0,
"time": "0:00",
"swap": 0.0,
"code": 0.1,
"data": 1.0,
"major_page_fault_count": 0,
"minor_page_fault_count": 736,
"dirty_pages_count": 0,
"sleeping_in_function": null,
"flags": "..4.2...",
"cgroups": "1:name=systemd:/user.slice/user-1000.+",
"supplementary_gids": [
10,
1000
],
"supplementary_groups": [
"wheel",
"kbrazil"
],
"thread_gid": 2225,
"environment_variables": [
"XDG_SESSION_ID=2",
"HOSTNAME=localhost"
],
"major_page_fault_count_delta": 0,
"minor_page_fault_count_delta": 4,
"used": 2.2,
"ipc_namespace_inode": 4026531839,
"mount_namespace_inode": 4026531840,
"net_namespace_inode": 4026531956,
"pid_namespace_inode": 4026531836,
"user_namespace_inode": 4026531837,
"nts_namespace_inode": 4026531838
}
]
}
]
```
### tracepath ### tracepath
```bash ```bash
tracepath6 3ffe:2400:0:109::2 | jc --tracepath -p tracepath6 3ffe:2400:0:109::2 | jc --tracepath -p

32
README.md
View File

@ -210,6 +210,7 @@ option.
| ` --ping-s` | `ping` and `ping6` command streaming parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/ping_s) | | ` --ping-s` | `ping` and `ping6` command streaming parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/ping_s) |
| ` --pip-list` | `pip list` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_list) | | ` --pip-list` | `pip list` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_list) |
| ` --pip-show` | `pip show` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_show) | | ` --pip-show` | `pip show` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_show) |
| ` --postconf` | `postconf -M` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/postconf) |
| ` --ps` | `ps` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/ps) | | ` --ps` | `ps` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/ps) |
| ` --route` | `route` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/route) | | ` --route` | `route` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/route) |
| ` --rpm-qi` | `rpm -qi` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/rpm_qi) | | ` --rpm-qi` | `rpm -qi` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/rpm_qi) |
@ -250,22 +251,21 @@ option.
| ` --zipinfo` | `zipinfo` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/zipinfo) | | ` --zipinfo` | `zipinfo` command parser | [📃](https://kellyjonbrazil.github.io/jc/docs/parsers/zipinfo) |
### Options ### Options
- `-a` about `jc`. Prints information about `jc` and the parsers (in JSON or | Short | Long | Description |
YAML, of course!) |-------|-----------------|--------------------------------------------------------------------------------------------------------------|
- `-C` force color output even when using pipes (overrides `-m` and the | `-a` | `--about` | About `jc`. Prints information about `jc` and the parsers (in JSON or YAML, of course!) |
`NO_COLOR` env variable) | `-C` | `--force-color` | Force color output even when using pipes (overrides `-m` and the `NO_COLOR` env variable) |
- `-d` debug mode. Prints trace messages if parsing issues are encountered (use | `-d` | `--debug` | Debug mode. Prints trace messages if parsing issues are encountered (use`-dd` for verbose debugging) |
`-dd` for verbose debugging) | `-h` | `--help` | Help. Use `jc -h --parser_name` for parser documentation |
- `-h` help. Use `jc -h --parser_name` for parser documentation | `-m` | `--monochrome` | Monochrome output |
- `-m` monochrome JSON output | `-p` | `--pretty` | Pretty format the JSON output |
- `-p` pretty format the JSON output | `-q` | `--quiet` | Quiet mode. Suppresses parser warning messages (use `-qq` to ignore streaming parser errors) |
- `-q` quiet mode. Suppresses parser warning messages (use `-qq` to ignore | `-r` | `--raw` | Raw output. Provides more literal output, typically with string values and no additional semantic processing |
streaming parser errors) | `-u` | `--unbuffer` | Unbuffer output |
- `-r` raw output. Provides a more literal JSON output, typically with string | `-v` | `--version` | Version information |
values and no additional semantic processing | `-y` | `--yaml-out` | YAML output |
- `-u` unbuffer output | `-B` | `--bash-comp` | Generate Bash shell completion script |
- `-v` version information | `-Z` | `--zsh-comp` | Generate Zsh shell completion script |
- `-y` YAML output
### Exit Codes ### Exit Codes
Any fatal errors within `jc` will generate an exit code of `100`, otherwise the Any fatal errors within `jc` will generate an exit code of `100`, otherwise the

5
docs/parsers/asciitable.md
View File

@ -59,6 +59,9 @@ etc...
Headers (keys) are converted to snake-case. All values are returned as Headers (keys) are converted to snake-case. All values are returned as
strings, except empty strings, which are converted to None/null. strings, except empty strings, which are converted to None/null.
> Note: To preserve the case of the keys use the `-r` cli option or
> `raw=True` argument in `parse()`.
Usage (cli): Usage (cli):
$ cat table.txt | jc --asciitable $ cat table.txt | jc --asciitable
@ -141,4 +144,4 @@ Returns:
### Parser Information ### Parser Information
Compatibility: linux, darwin, cygwin, win32, aix, freebsd Compatibility: linux, darwin, cygwin, win32, aix, freebsd
Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com) Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

5
docs/parsers/asciitable_m.md
View File

@ -29,6 +29,9 @@ Headers (keys) are converted to snake-case and newlines between multi-line
headers are joined with an underscore. All values are returned as strings, headers are joined with an underscore. All values are returned as strings,
except empty strings, which are converted to None/null. except empty strings, which are converted to None/null.
> Note: To preserve the case of the keys use the `-r` cli option or
> `raw=True` argument in `parse()`.
> Note: table column separator characters (e.g. `|`) cannot be present > Note: table column separator characters (e.g. `|`) cannot be present
> inside the cell data. If detected, a warning message will be printed to > inside the cell data. If detected, a warning message will be printed to
> `STDERR` and the line will be skipped. The warning message can be > `STDERR` and the line will be skipped. The warning message can be
@ -126,4 +129,4 @@ Returns:
### Parser Information ### Parser Information
Compatibility: linux, darwin, cygwin, win32, aix, freebsd Compatibility: linux, darwin, cygwin, win32, aix, freebsd
Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com) Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

2
docs/parsers/id.md
View File

@ -128,4 +128,4 @@ Returns:
### Parser Information ### Parser Information
Compatibility: linux, darwin, aix, freebsd Compatibility: linux, darwin, aix, freebsd
Version 1.4 by Kelly Brazil (kellyjonbrazil@gmail.com) Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com)

9
docs/parsers/ini.md
View File

@ -11,9 +11,10 @@ Parses standard `INI` files and files containing simple key/value pairs.
- Comment prefix can be `#` or `;`. Comments must be on their own line. - Comment prefix can be `#` or `;`. Comments must be on their own line.
- If duplicate keys are found, only the last value will be used. - If duplicate keys are found, only the last value will be used.
> Note: Values starting and ending with quotation marks will have the marks > Note: Values starting and ending with double or single quotation marks
> removed. If you would like to keep the quotation marks, use the `-r` > will have the marks removed. If you would like to keep the quotation
> command-line argument or the `raw=True` argument in `parse()`. > marks, use the `-r` command-line argument or the `raw=True` argument in
> `parse()`.
Usage (cli): Usage (cli):
@ -91,4 +92,4 @@ Returns:
### Parser Information ### Parser Information
Compatibility: linux, darwin, cygwin, win32, aix, freebsd Compatibility: linux, darwin, cygwin, win32, aix, freebsd
Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com) Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com)

115
docs/parsers/postconf.md Normal file
View File

@ -0,0 +1,115 @@
[Home](https://kellyjonbrazil.github.io/jc/)
<a id="jc.parsers.postconf"></a>
# jc.parsers.postconf
jc - JSON Convert `postconf -M` command output parser
Usage (cli):
$ postconf -M | jc --postconf
or
$ jc postconf -M
Usage (module):
import jc
result = jc.parse('postconf', postconf_command_output)
Schema:
[
{
"service_name": string,
"service_type": string,
"private": boolean/null, # [0]
"unprivileged": boolean/null, # [0]
"chroot": boolean/null, # [0]
"wake_up_time": integer/null, # [0]
"no_wake_up_before_first_use": boolean/null, # [1]
"process_limit": integer/null, # [0]
"command": string
}
]
[0] '-' converted to null/None
[1] null/None if `wake_up_time` is null/None
Examples:
$ postconf -M | jc --postconf -p
[
{
"service_name": "smtp",
"service_type": "inet",
"private": false,
"unprivileged": null,
"chroot": true,
"wake_up_time": null,
"process_limit": null,
"command": "smtpd",
"no_wake_up_before_first_use": null
},
{
"service_name": "pickup",
"service_type": "unix",
"private": false,
"unprivileged": null,
"chroot": true,
"wake_up_time": 60,
"process_limit": 1,
"command": "pickup",
"no_wake_up_before_first_use": false
}
]
$ postconf -M | jc --postconf -p -r
[
{
"service_name": "smtp",
"service_type": "inet",
"private": "n",
"unprivileged": "-",
"chroot": "y",
"wake_up_time": "-",
"process_limit": "-",
"command": "smtpd"
},
{
"service_name": "pickup",
"service_type": "unix",
"private": "n",
"unprivileged": "-",
"chroot": "y",
"wake_up_time": "60",
"process_limit": "1",
"command": "pickup"
}
]
<a id="jc.parsers.postconf.parse"></a>
### parse
```python
def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict]
```
Main text parsing function
Parameters:
data: (string) text data to parse
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
Returns:
List of Dictionaries. Raw or processed structured data.
### Parser Information
Compatibility: linux
Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

88
jc/cli.py
View File

@ -9,10 +9,13 @@ import textwrap
import signal import signal
import shlex import shlex
import subprocess import subprocess
from typing import List, Dict
from .lib import (__version__, parser_info, all_parser_info, parsers, from .lib import (__version__, parser_info, all_parser_info, parsers,
_get_parser, _parser_is_streaming, standard_parser_mod_list, _get_parser, _parser_is_streaming, standard_parser_mod_list,
plugin_parser_mod_list, streaming_parser_mod_list) plugin_parser_mod_list, streaming_parser_mod_list)
from . import utils from . import utils
from .cli_data import long_options_map
from .shell_completions import bash_completion, zsh_completion
from . import tracebackplus from . import tracebackplus
from .exceptions import LibraryNotInstalled, ParseError from .exceptions import LibraryNotInstalled, ParseError
@ -166,6 +169,22 @@ def parsers_text(indent=0, pad=0):
return ptext return ptext
def options_text(indent=0, pad=0):
"""Return the argument and description information from each option"""
otext = ''
padding_char = ' '
for option in long_options_map:
o_short = '-' + long_options_map[option][0]
o_desc = long_options_map[option][1]
o_combined = o_short + ', ' + option
padding = pad - len(o_combined)
indent_text = padding_char * indent
padding_text = padding_char * padding
otext += indent_text + o_combined + padding_text + o_desc + '\n'
return otext
def about_jc(): def about_jc():
"""Return jc info and the contents of each parser.info as a dictionary""" """Return jc info and the contents of each parser.info as a dictionary"""
return { return {
@ -189,43 +208,35 @@ def about_jc():
def helptext(): def helptext():
"""Return the help text with the list of parsers""" """Return the help text with the list of parsers"""
parsers_string = parsers_text(indent=12, pad=17) parsers_string = parsers_text(indent=4, pad=20)
options_string = options_text(indent=4, pad=20)
helptext_string = f'''\ helptext_string = f'''\
jc converts the output of many commands and file-types to JSON jc converts the output of many commands and file-types to JSON or YAML
Usage: COMMAND | jc PARSER [OPTIONS] Usage:
COMMAND | jc PARSER [OPTIONS]
or magic syntax: or magic syntax:
jc [OPTIONS] COMMAND jc [OPTIONS] COMMAND
Parsers: Parsers:
{parsers_string} {parsers_string}
Options: Options:
-a about jc {options_string}
-C force color output even when using pipes (overrides -m) Examples:
-d debug (-dd for verbose debug) Standard Syntax:
-h help (-h --parser_name for parser documentation) $ dig www.google.com | jc --dig --pretty
-m monochrome output
-p pretty print output
-q quiet - suppress parser warnings (-qq to ignore streaming errors)
-r raw JSON output
-u unbuffer output
-v version info
-y YAML output
Examples: Magic Syntax:
Standard Syntax: $ jc --pretty dig www.google.com
$ dig www.google.com | jc --dig -p
Magic Syntax: Parser Documentation:
$ jc -p dig www.google.com $ jc --help --dig
'''
Parser Documentation: return helptext_string
$ jc -h --dig
'''
return textwrap.dedent(helptext_string)
def help_doc(options): def help_doc(options):
@ -285,7 +296,7 @@ def yaml_out(data, pretty=False, env_colors=None, mono=False, piped_out=False, a
# ruamel.yaml versions prior to 0.17.0 the use of __file__ in the # ruamel.yaml versions prior to 0.17.0 the use of __file__ in the
# plugin code is incompatible with the pyoxidizer packager # plugin code is incompatible with the pyoxidizer packager
YAML.official_plug_ins = lambda a: [] YAML.official_plug_ins = lambda a: []
yaml=YAML() yaml = YAML()
yaml.default_flow_style = False yaml.default_flow_style = False
yaml.explicit_start = True yaml.explicit_start = True
yaml.allow_unicode = not ascii_only yaml.allow_unicode = not ascii_only
@ -381,7 +392,7 @@ def magic_parser(args):
jc_options (list) list of jc options jc_options (list) list of jc options
""" """
# bail immediately if there are no args or a parser is defined # bail immediately if there are no args or a parser is defined
if len(args) <= 1 or args[1].startswith('--'): if len(args) <= 1 or (args[1].startswith('--') and args[1] not in long_options_map):
return False, None, None, [] return False, None, None, []
args_given = args[1:] args_given = args[1:]
@ -389,6 +400,12 @@ def magic_parser(args):
# find the options # find the options
for arg in list(args_given): for arg in list(args_given):
# long option found - populate option list
if arg in long_options_map:
options.extend(long_options_map[arg][0])
args_given.pop(0)
continue
# parser found - use standard syntax # parser found - use standard syntax
if arg.startswith('--'): if arg.startswith('--'):
return False, None, None, [] return False, None, None, []
@ -483,6 +500,9 @@ def main():
# find options if magic_parser did not find a command # find options if magic_parser did not find a command
if not valid_command: if not valid_command:
for opt in sys.argv: for opt in sys.argv:
if opt in long_options_map:
options.extend(long_options_map[opt][0])
if opt.startswith('-') and not opt.startswith('--'): if opt.startswith('-') and not opt.startswith('--'):
options.extend(opt[1:]) options.extend(opt[1:])
@ -499,6 +519,8 @@ def main():
unbuffer = 'u' in options unbuffer = 'u' in options
version_info = 'v' in options version_info = 'v' in options
yaml_out = 'y' in options yaml_out = 'y' in options
bash_comp = 'B' in options
zsh_comp = 'Z' in options
if verbose_debug: if verbose_debug:
tracebackplus.enable(context=11) tracebackplus.enable(context=11)
@ -523,6 +545,14 @@ def main():
utils._safe_print(versiontext()) utils._safe_print(versiontext())
sys.exit(0) sys.exit(0)
if bash_comp:
utils._safe_print(bash_completion())
sys.exit(0)
if zsh_comp:
utils._safe_print(zsh_completion())
sys.exit(0)
# if magic syntax used, try to run the command and error if it's not found, etc. # if magic syntax used, try to run the command and error if it's not found, etc.
magic_stdout, magic_stderr, magic_exit_code = None, None, 0 magic_stdout, magic_stderr, magic_exit_code = None, None, 0
if run_command: if run_command:

18
jc/cli_data.py Normal file
View File

@ -0,0 +1,18 @@
"""jc - JSON Convert cli_data module"""
from typing import List, Dict
long_options_map: Dict[str, List[str]] = {
'--about': ['a', 'about jc'],
'--force-color': ['C', 'force color output even when using pipes (overrides -m)'],
'--debug': ['d', 'debug (double for verbose debug)'],
'--help': ['h', 'help (--help --parser_name for parser documentation)'],
'--monochrome': ['m', 'monochrome output'],
'--pretty': ['p', 'pretty print output'],
'--quiet': ['q', 'suppress warnings (double to ignore streaming errors)'],
'--raw': ['r', 'raw output'],
'--unbuffer': ['u', 'unbuffer output'],
'--version': ['v', 'version info'],
'--yaml-out': ['y', 'YAML output'],
'--bash-comp': ['B', 'gen Bash completion: jc -B > /etc/bash_completion.d/jc'],
'--zsh-comp': ['Z', 'gen Zsh completion: jc -Z > "${fpath[1]}/_jc"']
}

3
jc/lib.py
View File

@ -6,7 +6,7 @@ import importlib
from typing import Dict, List, Iterable, Union, Iterator from typing import Dict, List, Iterable, Union, Iterator
from jc import appdirs from jc import appdirs
__version__ = '1.20.0' __version__ = '1.20.1'
parsers = [ parsers = [
'acpi', 'acpi',
@ -73,6 +73,7 @@ parsers = [
'ping-s', 'ping-s',
'pip-list', 'pip-list',
'pip-show', 'pip-show',
'postconf',
'ps', 'ps',
'route', 'route',
'rpm-qi', 'rpm-qi',

18
jc/parsers/asciitable.py
View File

@ -54,6 +54,9 @@ etc...
Headers (keys) are converted to snake-case. All values are returned as Headers (keys) are converted to snake-case. All values are returned as
strings, except empty strings, which are converted to None/null. strings, except empty strings, which are converted to None/null.
> Note: To preserve the case of the keys use the `-r` cli option or
> `raw=True` argument in `parse()`.
Usage (cli): Usage (cli):
$ cat table.txt | jc --asciitable $ cat table.txt | jc --asciitable
@ -122,7 +125,7 @@ from jc.parsers.universal import sparse_table_parse
class info(): class info():
"""Provides parser metadata (version, author, etc.)""" """Provides parser metadata (version, author, etc.)"""
version = '1.1' version = '1.2'
description = 'ASCII and Unicode table parser' description = 'ASCII and Unicode table parser'
author = 'Kelly Brazil' author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com' author_email = 'kellyjonbrazil@gmail.com'
@ -144,6 +147,12 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
List of Dictionaries. Structured to conform to the schema. List of Dictionaries. Structured to conform to the schema.
""" """
# normalize keys: convert to lowercase
for item in proc_data:
for key in item.copy():
k_new = key.lower()
item[k_new] = item.pop(key)
return proc_data return proc_data
@ -227,12 +236,11 @@ def _is_separator(line: str) -> bool:
def _snake_case(line: str) -> str: def _snake_case(line: str) -> str:
""" """
Replace spaces between words and special characters with an underscore Replace spaces between words and special characters with an underscore.
and set to lowercase. Ignore the replacement char (�) used for header Ignore the replacement char (�) used for header padding.
padding.
""" """
line = re.sub(r'[^a-zA-Z0-9� ]', '_', line) # special characters line = re.sub(r'[^a-zA-Z0-9� ]', '_', line) # special characters
line = re.sub(r'\b \b', '_', line).lower() # spaces betwee words line = re.sub(r'\b \b', '_', line) # spaces between words
return line return line

16
jc/parsers/asciitable_m.py
View File

@ -24,6 +24,9 @@ Headers (keys) are converted to snake-case and newlines between multi-line
headers are joined with an underscore. All values are returned as strings, headers are joined with an underscore. All values are returned as strings,
except empty strings, which are converted to None/null. except empty strings, which are converted to None/null.
> Note: To preserve the case of the keys use the `-r` cli option or
> `raw=True` argument in `parse()`.
> Note: table column separator characters (e.g. `|`) cannot be present > Note: table column separator characters (e.g. `|`) cannot be present
> inside the cell data. If detected, a warning message will be printed to > inside the cell data. If detected, a warning message will be printed to
> `STDERR` and the line will be skipped. The warning message can be > `STDERR` and the line will be skipped. The warning message can be
@ -107,7 +110,7 @@ from jc.exceptions import ParseError
class info(): class info():
"""Provides parser metadata (version, author, etc.)""" """Provides parser metadata (version, author, etc.)"""
version = '1.1' version = '1.2'
description = 'multi-line ASCII and Unicode table parser' description = 'multi-line ASCII and Unicode table parser'
author = 'Kelly Brazil' author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com' author_email = 'kellyjonbrazil@gmail.com'
@ -129,6 +132,12 @@ def _process(proc_data: List[Dict]) -> List[Dict]:
List of Dictionaries. Structured to conform to the schema. List of Dictionaries. Structured to conform to the schema.
""" """
# normalize keys: convert to lowercase
for item in proc_data:
for key in item.copy():
k_new = key.lower()
item[k_new] = item.pop(key)
return proc_data return proc_data
@ -233,12 +242,11 @@ def _is_separator(line: str) -> bool:
def _snake_case(line: str) -> str: def _snake_case(line: str) -> str:
""" """
replace spaces between words and special characters with an underscore replace spaces between words and special characters with an underscore.
and set to lowercase
""" """
# must include all column separator characters in regex # must include all column separator characters in regex
line = re.sub(r'[^a-zA-Z0-9 |│┃┆┇┊┋╎╏║]', '_', line) line = re.sub(r'[^a-zA-Z0-9 |│┃┆┇┊┋╎╏║]', '_', line)
return re.sub(r'\b \b', '_', line).lower() return re.sub(r'\b \b', '_', line)
def _fixup_separators(line: str) -> str: def _fixup_separators(line: str) -> str:

15
jc/parsers/id.py
View File

@ -105,7 +105,7 @@ import jc.utils
class info(): class info():
"""Provides parser metadata (version, author, etc.)""" """Provides parser metadata (version, author, etc.)"""
version = '1.4' version = '1.5'
description = '`id` command parser' description = '`id` command parser'
author = 'Kelly Brazil' author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com' author_email = 'kellyjonbrazil@gmail.com'
@ -144,6 +144,13 @@ def _process(proc_data):
return proc_data return proc_data
def _get_item(my_list, index, default=None):
if index < len(my_list):
return my_list[index]
return default
def parse(data, raw=False, quiet=False): def parse(data, raw=False, quiet=False):
""" """
Main text parsing function Main text parsing function
@ -174,14 +181,14 @@ def parse(data, raw=False, quiet=False):
uid_parsed = uid_parsed.split('=') uid_parsed = uid_parsed.split('=')
raw_output['uid'] = {} raw_output['uid'] = {}
raw_output['uid']['id'] = uid_parsed[1] raw_output['uid']['id'] = uid_parsed[1]
raw_output['uid']['name'] = uid_parsed[2] raw_output['uid']['name'] = _get_item(uid_parsed, 2)
if section.startswith('gid'): if section.startswith('gid'):
gid_parsed = section.replace('(', '=').replace(')', '=') gid_parsed = section.replace('(', '=').replace(')', '=')
gid_parsed = gid_parsed.split('=') gid_parsed = gid_parsed.split('=')
raw_output['gid'] = {} raw_output['gid'] = {}
raw_output['gid']['id'] = gid_parsed[1] raw_output['gid']['id'] = gid_parsed[1]
raw_output['gid']['name'] = gid_parsed[2] raw_output['gid']['name'] = _get_item(gid_parsed, 2)
if section.startswith('groups'): if section.startswith('groups'):
groups_parsed = section.replace('(', '=').replace(')', '=') groups_parsed = section.replace('(', '=').replace(')', '=')
@ -193,7 +200,7 @@ def parse(data, raw=False, quiet=False):
group_dict = {} group_dict = {}
grp_parsed = group.split('=') grp_parsed = group.split('=')
group_dict['id'] = grp_parsed[0] group_dict['id'] = grp_parsed[0]
group_dict['name'] = grp_parsed[1] group_dict['name'] = _get_item(grp_parsed, 1)
raw_output['groups'].append(group_dict) raw_output['groups'].append(group_dict)
if section.startswith('context'): if section.startswith('context'):

21
jc/parsers/ini.py
View File

@ -6,9 +6,10 @@ Parses standard `INI` files and files containing simple key/value pairs.
- Comment prefix can be `#` or `;`. Comments must be on their own line. - Comment prefix can be `#` or `;`. Comments must be on their own line.
- If duplicate keys are found, only the last value will be used. - If duplicate keys are found, only the last value will be used.
> Note: Values starting and ending with quotation marks will have the marks > Note: Values starting and ending with double or single quotation marks
> removed. If you would like to keep the quotation marks, use the `-r` > will have the marks removed. If you would like to keep the quotation
> command-line argument or the `raw=True` argument in `parse()`. > marks, use the `-r` command-line argument or the `raw=True` argument in
> `parse()`.
Usage (cli): Usage (cli):
@ -69,7 +70,7 @@ import configparser
class info(): class info():
"""Provides parser metadata (version, author, etc.)""" """Provides parser metadata (version, author, etc.)"""
version = '1.6' version = '1.7'
description = 'INI file parser' description = 'INI file parser'
author = 'Kelly Brazil' author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com' author_email = 'kellyjonbrazil@gmail.com'
@ -99,17 +100,21 @@ def _process(proc_data):
for key, value in proc_data[heading].items(): for key, value in proc_data[heading].items():
if value is not None and value.startswith('"') and value.endswith('"'): if value is not None and value.startswith('"') and value.endswith('"'):
proc_data[heading][key] = value.lstrip('"').rstrip('"') proc_data[heading][key] = value.lstrip('"').rstrip('"')
elif value is not None and value.startswith("'") and value.endswith("'"):
proc_data[heading][key] = value.lstrip("'").rstrip("'")
elif value is None: elif value is None:
proc_data[heading][key] = '' proc_data[heading][key] = ''
# simple key/value files with no headers # simple key/value files with no headers
else: else:
if (proc_data[heading] is not None and if proc_data[heading] is not None and proc_data[heading].startswith('"') and proc_data[heading].endswith('"'):
proc_data[heading].startswith('"') and
proc_data[heading].endswith('"')):
proc_data[heading] = proc_data[heading].lstrip('"').rstrip('"') proc_data[heading] = proc_data[heading].lstrip('"').rstrip('"')
elif proc_data[heading] is not None and proc_data[heading].startswith("'") and proc_data[heading].endswith("'"):
proc_data[heading] = proc_data[heading].lstrip("'").rstrip("'")
elif proc_data[heading] is None: elif proc_data[heading] is None:
proc_data[heading] = '' proc_data[heading] = ''

173
jc/parsers/postconf.py Normal file
View File

@ -0,0 +1,173 @@
"""jc - JSON Convert `postconf -M` command output parser
Usage (cli):
$ postconf -M | jc --postconf
or
$ jc postconf -M
Usage (module):
import jc
result = jc.parse('postconf', postconf_command_output)
Schema:
[
{
"service_name": string,
"service_type": string,
"private": boolean/null, # [0]
"unprivileged": boolean/null, # [0]
"chroot": boolean/null, # [0]
"wake_up_time": integer/null, # [0]
"no_wake_up_before_first_use": boolean/null, # [1]
"process_limit": integer/null, # [0]
"command": string
}
]
[0] '-' converted to null/None
[1] null/None if `wake_up_time` is null/None
Examples:
$ postconf -M | jc --postconf -p
[
{
"service_name": "smtp",
"service_type": "inet",
"private": false,
"unprivileged": null,
"chroot": true,
"wake_up_time": null,
"process_limit": null,
"command": "smtpd",
"no_wake_up_before_first_use": null
},
{
"service_name": "pickup",
"service_type": "unix",
"private": false,
"unprivileged": null,
"chroot": true,
"wake_up_time": 60,
"process_limit": 1,
"command": "pickup",
"no_wake_up_before_first_use": false
}
]
$ postconf -M | jc --postconf -p -r
[
{
"service_name": "smtp",
"service_type": "inet",
"private": "n",
"unprivileged": "-",
"chroot": "y",
"wake_up_time": "-",
"process_limit": "-",
"command": "smtpd"
},
{
"service_name": "pickup",
"service_type": "unix",
"private": "n",
"unprivileged": "-",
"chroot": "y",
"wake_up_time": "60",
"process_limit": "1",
"command": "pickup"
}
]
"""
from typing import List, Dict
import jc.utils
from jc.parsers.universal import simple_table_parse
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.0'
description = '`postconf -M` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
compatible = ['linux']
magic_commands = ['postconf -M']
__version__ = info.version
def _process(proc_data: List[Dict]) -> List[Dict]:
"""
Final processing to conform to the schema.
Parameters:
proc_data: (List of Dictionaries) raw structured data to process
Returns:
List of Dictionaries. Structured to conform to the schema.
"""
keys = ['private', 'unprivileged', 'chroot', 'wake_up_time', 'process_limit']
bools = ['private', 'unprivileged', 'chroot']
integers = ['wake_up_time', 'process_limit']
for item in proc_data:
if item['wake_up_time'].endswith('?'):
item['no_wake_up_before_first_use'] = True
elif item['wake_up_time'] == '-':
item['no_wake_up_before_first_use'] = None
else:
item['no_wake_up_before_first_use'] = False
for key in keys:
if item[key] == '-':
item[key] = None
for key in bools:
if item[key] is not None:
item[key] = jc.utils.convert_to_bool(item[key])
for key in integers:
if item[key] is not None:
item[key] = jc.utils.convert_to_int(item[key])
return proc_data
def parse(
data: str,
raw: bool = False,
quiet: bool = False
) -> List[Dict]:
"""
Main text parsing function
Parameters:
data: (string) text data to parse
raw: (boolean) unprocessed output if True
quiet: (boolean) suppress warning messages if True
Returns:
List of Dictionaries. Raw or processed structured data.
"""
jc.utils.compatibility(__name__, info.compatible, quiet)
jc.utils.input_type_check(data)
raw_output: List = []
if jc.utils.has_data(data):
table = ['service_name service_type private unprivileged chroot wake_up_time process_limit command']
data_list = list(filter(None, data.splitlines()))
table.extend(data_list)
raw_output = simple_table_parse(table)
return raw_output if raw else _process(raw_output)

348
jc/shell_completions.py Normal file
View File

@ -0,0 +1,348 @@
"""jc - JSON Convert shell_completions module"""
from string import Template
from .cli_data import long_options_map
from .lib import all_parser_info
bash_template = Template('''\
_jc()
{
local cur prev words cword jc_commands jc_parsers jc_options \\
jc_about_options jc_about_mod_options jc_help_options jc_special_options
jc_commands=(${bash_commands})
jc_parsers=(${bash_parsers})
jc_options=(${bash_options})
jc_about_options=(${bash_about_options})
jc_about_mod_options=(${bash_about_mod_options})
jc_help_options=(${bash_help_options})
jc_special_options=(${bash_special_options})
COMPREPLY=()
_get_comp_words_by_ref cur prev words cword
# if jc_about_options are found anywhere in the line, then only complete from jc_about_mod_options
for i in "$${words[@]::$${#words[@]}-1}"; do
if [[ " $${jc_about_options[*]} " =~ " $${i} " ]]; then
COMPREPLY=( $$( compgen -W "$${jc_about_mod_options[*]}" \\
-- "$${cur}" ) )
return 0
fi
done
# if jc_help_options and a parser are found anywhere in the line, then no more completions
if
(
for i in "$${words[@]::$${#words[@]}-1}"; do
if [[ " $${jc_help_options[*]} " =~ " $${i} " ]]; then
return 0
fi
done
return 1
) && (
for i in "$${words[@]::$${#words[@]}-1}"; do
if [[ " $${jc_parsers[*]} " =~ " $${i} " ]]; then
return 0
fi
done
return 1
); then
return 0
fi
# if jc_help_options are found anywhere in the line, then only complete with parsers
for i in "$${words[@]::$${#words[@]}-1}"; do
if [[ " $${jc_help_options[*]} " =~ " $${i} " ]]; then
COMPREPLY=( $$( compgen -W "$${jc_parsers[*]}" \\
-- "$${cur}" ) )
return 0
fi
done
# if special options are found anywhere in the line, then no more completions
for i in "$${words[@]::$${#words[@]}-1}"; do
if [[ " $${jc_special_options[*]} " =~ " $${i} " ]]; then
return 0
fi
done
# if magic command is found anywhere in the line, use called command's autocompletion
for i in "$${words[@]::$${#words[@]}-1}"; do
if [[ " $${jc_commands[*]} " =~ " $${i} " ]]; then
_command
return 0
fi
done
# if a parser arg is found anywhere in the line, only show options and help options
for i in "$${words[@]::$${#words[@]}-1}"; do
if [[ " $${jc_parsers[*]} " =~ " $${i} " ]]; then
COMPREPLY=( $$( compgen -W "$${jc_options[*]} $${jc_help_options[*]}" \\
-- "$${cur}" ) )
return 0
fi
done
# default completion
COMPREPLY=( $$( compgen -W "$${jc_options[*]} $${jc_about_options[*]} $${jc_help_options[*]} $${jc_special_options[*]} $${jc_parsers[*]} $${jc_commands[*]}" \\
-- "$${cur}" ) )
} &&
complete -F _jc jc
''')
zsh_template = Template('''\
#compdef jc
_jc() {
local -a jc_commands jc_commands_describe \\
jc_parsers jc_parsers_describe \\
jc_options jc_options_describe \\
jc_about_options jc_about_options_describe \\
jc_about_mod_options jc_about_mod_options_describe \\
jc_help_options jc_help_options_describe \\
jc_special_options jc_special_options_describe
jc_commands=(${zsh_commands})
jc_commands_describe=(
${zsh_commands_describe}
)
jc_parsers=(${zsh_parsers})
jc_parsers_describe=(
${zsh_parsers_describe}
)
jc_options=(${zsh_options})
jc_options_describe=(
${zsh_options_describe}
)
jc_about_options=(${zsh_about_options})
jc_about_options_describe=(
${zsh_about_options_describe}
)
jc_about_mod_options=(${zsh_about_mod_options})
jc_about_mod_options_describe=(
${zsh_about_mod_options_describe}
)
jc_help_options=(${zsh_help_options})
jc_help_options_describe=(
${zsh_help_options_describe}
)
jc_special_options=(${zsh_special_options})
jc_special_options_describe=(
${zsh_special_options_describe}
)
# if jc_about_options are found anywhere in the line, then only complete from jc_about_mod_options
for i in $${words:0:-1}; do
if (( $$jc_about_options[(Ie)$${i}] )); then
_describe 'commands' jc_about_mod_options_describe
return 0
fi
done
# if jc_help_options and a parser are found anywhere in the line, then no more completions
if
(
for i in $${words:0:-1}; do
if (( $$jc_help_options[(Ie)$${i}] )); then
return 0
fi
done
return 1
) && (
for i in $${words:0:-1}; do
if (( $$jc_parsers[(Ie)$${i}] )); then
return 0
fi
done
return 1
); then
return 0
fi
# if jc_help_options are found anywhere in the line, then only complete with parsers
for i in $${words:0:-1}; do
if (( $$jc_help_options[(Ie)$${i}] )); then
_describe 'commands' jc_parsers_describe
return 0
fi
done
# if special options are found anywhere in the line, then no more completions
for i in $${words:0:-1}; do
if (( $$jc_special_options[(Ie)$${i}] )); then
return 0
fi
done
# if magic command is found anywhere in the line, use called command's autocompletion
for i in $${words:0:-1}; do
if (( $$jc_commands[(Ie)$${i}] )); then
# hack to remove options between jc and the magic command
shift $$(( $${#words} - 2 )) words
words[1,0]=(jc)
CURRENT=$${#words}
# run the magic command's completions
_arguments '*::arguments:_normal'
return 0
fi
done
# if a parser arg is found anywhere in the line, only show options and help options
for i in $${words:0:-1}; do
if (( $$jc_parsers[(Ie)$${i}] )); then
_describe 'commands' jc_options_describe -- jc_help_options_describe
return 0
fi
done
# default completion
_describe 'commands' jc_options_describe -- jc_about_options_describe -- jc_help_options_describe -- jc_special_options_describe -- jc_parsers_describe -- jc_commands_describe
}
_jc
''')
about_options = ['--about', '-a']
about_mod_options = ['--pretty', '-p', '--yaml-out', '-y', '--monochrome', '-m', '--force-color', '-C']
help_options = ['--help', '-h']
special_options = ['--version', '-v', '--bash-comp', '-B', '--zsh-comp', '-Z']
def get_commands():
command_list = []
for cmd in all_parser_info():
if 'magic_commands' in cmd:
command_list.extend(cmd['magic_commands'])
return sorted(list(set([i.split()[0] for i in command_list])))
def get_options():
options_list = []
for opt in long_options_map:
options_list.append(opt)
options_list.append('-' + long_options_map[opt][0])
return options_list
def get_parsers():
p_list = []
for cmd in all_parser_info():
if 'argument' in cmd:
p_list.append(cmd['argument'])
return p_list
def get_parsers_descriptions():
pd_list = []
for p in all_parser_info():
if 'description' in p:
pd_list.append(f"'{p['argument']}:{p['description']}'")
return pd_list
def get_zsh_command_descriptions(command_list):
zsh_commands = []
for cmd in command_list:
zsh_commands.append(f"""'{cmd}:run "{cmd}" command with magic syntax.'""")
return zsh_commands
def get_descriptions(opt_list):
"""Return a list of options:description items."""
opt_desc_list = []
for item in opt_list:
# get long options
if item in long_options_map:
opt_desc_list.append(f"'{item}:{long_options_map[item][1]}'")
continue
# get short options
for k, v in long_options_map.items():
if item[1:] == v[0]:
opt_desc_list.append(f"'{item}:{v[1]}'")
continue
return opt_desc_list
def bash_completion():
parsers_str = ' '.join(get_parsers())
opts_no_special = get_options()
for s_option in special_options:
opts_no_special.remove(s_option)
for a_option in about_options:
opts_no_special.remove(a_option)
for h_option in help_options:
opts_no_special.remove(h_option)
options_str = ' '.join(opts_no_special)
about_options_str = ' '.join(about_options)
about_mod_options_str = ' '.join(about_mod_options)
help_options_str = ' '.join(help_options)
special_options_str = ' '.join(special_options)
commands_str = ' '.join(get_commands())
return bash_template.substitute(
bash_parsers=parsers_str,
bash_special_options=special_options_str,
bash_about_options=about_options_str,
bash_about_mod_options=about_mod_options_str,
bash_help_options=help_options_str,
bash_options=options_str,
bash_commands=commands_str
)
def zsh_completion():
parsers_str = ' '.join(get_parsers())
parsers_describe = '\n '.join(get_parsers_descriptions())
opts_no_special = get_options()
for s_option in special_options:
opts_no_special.remove(s_option)
for a_option in about_options:
opts_no_special.remove(a_option)
for h_option in help_options:
opts_no_special.remove(h_option)
options_str = ' '.join(opts_no_special)
options_describe = '\n '.join(get_descriptions(opts_no_special))
about_options_str = ' '.join(about_options)
about_options_describe = '\n '.join(get_descriptions(about_options))
about_mod_options_str = ' '.join(about_mod_options)
about_mod_options_describe = '\n '.join(get_descriptions(about_mod_options))
help_options_str = ' '.join(help_options)
help_options_describe = '\n '.join(get_descriptions(help_options))
special_options_str = ' '.join(special_options)
special_options_describe = '\n '.join(get_descriptions(special_options))
commands_str = ' '.join(get_commands())
commands_describe = '\n '.join(get_zsh_command_descriptions(get_commands()))
return zsh_template.substitute(
zsh_parsers=parsers_str,
zsh_parsers_describe=parsers_describe,
zsh_special_options=special_options_str,
zsh_special_options_describe=special_options_describe,
zsh_about_options=about_options_str,
zsh_about_options_describe=about_options_describe,
zsh_about_mod_options=about_mod_options_str,
zsh_about_mod_options_describe=about_mod_options_describe,
zsh_help_options=help_options_str,
zsh_help_options_describe=help_options_describe,
zsh_options=options_str,
zsh_options_describe=options_describe,
zsh_commands=commands_str,
zsh_commands_describe=commands_describe
)

57
man/jc.1
View File

@ -1,4 +1,4 @@
.TH jc 1 2022-05-31 1.20.0 "JSON Convert" .TH jc 1 2022-06-15 1.20.1 "JSON Convert"
.SH NAME .SH NAME
\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types
.SH SYNOPSIS .SH SYNOPSIS
@ -337,6 +337,11 @@ Key/Value file parser
\fB--pip-show\fP \fB--pip-show\fP
`pip show` command parser `pip show` command parser
.TP
.B
\fB--postconf\fP
`postconf -M` command parser
.TP .TP
.B .B
\fB--ps\fP \fB--ps\fP
@ -536,48 +541,56 @@ Options:
.TP .TP
.B .B
\fB-a\fP \fB-a\fP, \fB--about\fP
about \fBjc\fP (JSON or YAML output) About \fBjc\fP (JSON or YAML output)
.TP .TP
.B .B
\fB-C\fP \fB-C\fP, \fB--force-color\fP
force color output even when using pipes (overrides \fB-m\fP and the \fBNO_COLOR\fP env variable) Force color output even when using pipes (overrides \fB-m\fP and the \fBNO_COLOR\fP env variable)
.TP .TP
.B .B
\fB-d\fP \fB-d\fP, \fB--debug\fP
debug - show traceback (use \fB-dd\fP for verbose traceback) Debug - show traceback (use \fB-dd\fP for verbose traceback)
.TP .TP
.B .B
\fB-h\fP \fB-h\fP, \fB--help\fP
help (\fB-h --parser_name\fP for parser documentation) Help (\fB--help --parser_name\fP for parser documentation)
.TP .TP
.B .B
\fB-m\fP \fB-m\fP, \fB--monochrome\fP
monochrome output Monochrome output
.TP .TP
.B .B
\fB-p\fP \fB-p\fP, \fB--pretty\fP
pretty print output Pretty print output
.TP .TP
.B .B
\fB-q\fP \fB-q\fP, \fB--quiet\fP
quiet - suppress warnings (use \fB-qq\fP to ignore streaming parser errors) Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming parser errors)
.TP .TP
.B .B
\fB-r\fP \fB-r\fP, \fB--raw\fP
raw JSON output Raw output. Provides more literal output, typically with string values and no additional semantic processing
.TP .TP
.B .B
\fB-u\fP \fB-u\fP, \fB--unbuffer\fP
unbuffer output (useful for slow streaming data with streaming parsers) Unbuffer output (useful for slow streaming data with streaming parsers)
.TP .TP
.B .B
\fB-v\fP \fB-v\fP, \fB--version\fP
version information Version information
.TP .TP
.B .B
\fB-y\fP \fB-y\fP, \fB--yaml-out\fP
YAML output YAML output
.TP
.B
\fB-B\fP, \fB--bash-comp\fP
Generate Bash shell completion script
.TP
.B
\fB-Z\fP, \fB--zsh-comp\fP
Generate Zsh shell completion script
.SH EXIT CODES .SH EXIT CODES
Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP. When using the "Magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store the exit code of the program being parsed and add it to the \fBjc\fP exit code. This way it is easier to determine if an error was from the parsed program or \fBjc\fP. Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP. When using the "Magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store the exit code of the program being parsed and add it to the \fBjc\fP exit code. This way it is easier to determine if an error was from the parsed program or \fBjc\fP.

2
setup.py
View File

@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
setuptools.setup( setuptools.setup(
name='jc', name='jc',
version='1.20.0', version='1.20.1',
author='Kelly Brazil', author='Kelly Brazil',
author_email='kellyjonbrazil@gmail.com', author_email='kellyjonbrazil@gmail.com',
description='Converts the output of popular command-line tools and file-types to JSON.', description='Converts the output of popular command-line tools and file-types to JSON.',

50
templates/manpage_template
View File

@ -31,48 +31,56 @@ Options:
.TP .TP
.B .B
\fB-a\fP \fB-a\fP, \fB--about\fP
about \fBjc\fP (JSON or YAML output) About \fBjc\fP (JSON or YAML output)
.TP .TP
.B .B
\fB-C\fP \fB-C\fP, \fB--force-color\fP
force color output even when using pipes (overrides \fB-m\fP and the \fBNO_COLOR\fP env variable) Force color output even when using pipes (overrides \fB-m\fP and the \fBNO_COLOR\fP env variable)
.TP .TP
.B .B
\fB-d\fP \fB-d\fP, \fB--debug\fP
debug - show traceback (use \fB-dd\fP for verbose traceback) Debug - show traceback (use \fB-dd\fP for verbose traceback)
.TP .TP
.B .B
\fB-h\fP \fB-h\fP, \fB--help\fP
help (\fB-h --parser_name\fP for parser documentation) Help (\fB--help --parser_name\fP for parser documentation)
.TP .TP
.B .B
\fB-m\fP \fB-m\fP, \fB--monochrome\fP
monochrome output Monochrome output
.TP .TP
.B .B
\fB-p\fP \fB-p\fP, \fB--pretty\fP
pretty print output Pretty print output
.TP .TP
.B .B
\fB-q\fP \fB-q\fP, \fB--quiet\fP
quiet - suppress warnings (use \fB-qq\fP to ignore streaming parser errors) Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming parser errors)
.TP .TP
.B .B
\fB-r\fP \fB-r\fP, \fB--raw\fP
raw JSON output Raw output. Provides more literal output, typically with string values and no additional semantic processing
.TP .TP
.B .B
\fB-u\fP \fB-u\fP, \fB--unbuffer\fP
unbuffer output (useful for slow streaming data with streaming parsers) Unbuffer output (useful for slow streaming data with streaming parsers)
.TP .TP
.B .B
\fB-v\fP \fB-v\fP, \fB--version\fP
version information Version information
.TP .TP
.B .B
\fB-y\fP \fB-y\fP, \fB--yaml-out\fP
YAML output YAML output
.TP
.B
\fB-B\fP, \fB--bash-comp\fP
Generate Bash shell completion script
.TP
.B
\fB-Z\fP, \fB--zsh-comp\fP
Generate Zsh shell completion script
.SH EXIT CODES .SH EXIT CODES
Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP. When using the "Magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store the exit code of the program being parsed and add it to the \fBjc\fP exit code. This way it is easier to determine if an error was from the parsed program or \fBjc\fP. Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP. When using the "Magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store the exit code of the program being parsed and add it to the \fBjc\fP exit code. This way it is easier to determine if an error was from the parsed program or \fBjc\fP.

31
templates/readme_template
View File

@ -149,22 +149,21 @@ option.
| `{{ "{:>15}".format(parser.argument) }}` | {{ "{:<55}".format(parser.description) }} | {{ "{:<70}".format("[📃](https://kellyjonbrazil.github.io/jc/docs/parsers/" + parser.name + ")") }} |{% endfor %} | `{{ "{:>15}".format(parser.argument) }}` | {{ "{:<55}".format(parser.description) }} | {{ "{:<70}".format("[📃](https://kellyjonbrazil.github.io/jc/docs/parsers/" + parser.name + ")") }} |{% endfor %}
### Options ### Options
- `-a` about `jc`. Prints information about `jc` and the parsers (in JSON or | Short | Long | Description |
YAML, of course!) |-------|-----------------|--------------------------------------------------------------------------------------------------------------|
- `-C` force color output even when using pipes (overrides `-m` and the | `-a` | `--about` | About `jc`. Prints information about `jc` and the parsers (in JSON or YAML, of course!) |
`NO_COLOR` env variable) | `-C` | `--force-color` | Force color output even when using pipes (overrides `-m` and the `NO_COLOR` env variable) |
- `-d` debug mode. Prints trace messages if parsing issues are encountered (use | `-d` | `--debug` | Debug mode. Prints trace messages if parsing issues are encountered (use`-dd` for verbose debugging) |
`-dd` for verbose debugging) | `-h` | `--help` | Help. Use `jc -h --parser_name` for parser documentation |
- `-h` help. Use `jc -h --parser_name` for parser documentation | `-m` | `--monochrome` | Monochrome output |
- `-m` monochrome JSON output | `-p` | `--pretty` | Pretty format the JSON output |
- `-p` pretty format the JSON output | `-q` | `--quiet` | Quiet mode. Suppresses parser warning messages (use `-qq` to ignore streaming parser errors) |
- `-q` quiet mode. Suppresses parser warning messages (use `-qq` to ignore | `-r` | `--raw` | Raw output. Provides more literal output, typically with string values and no additional semantic processing |
streaming parser errors) | `-u` | `--unbuffer` | Unbuffer output |
- `-r` raw output. Provides a more literal JSON output, typically with string | `-v` | `--version` | Version information |
values and no additional semantic processing | `-y` | `--yaml-out` | YAML output |
- `-u` unbuffer output | `-B` | `--bash-comp` | Generate Bash shell completion script |
- `-v` version information | `-Z` | `--zsh-comp` | Generate Zsh shell completion script |
- `-y` YAML output
### Exit Codes ### Exit Codes
Any fatal errors within `jc` will generate an exit code of `100`, otherwise the Any fatal errors within `jc` will generate an exit code of `100`, otherwise the

4
tests/fixtures/generic/ini-double-quote.ini vendored Normal file
View File

@ -0,0 +1,4 @@
[client]
user=foo
host=localhost
password="bar"

1
tests/fixtures/generic/ini-double-quote.json vendored Normal file
View File

@ -0,0 +1 @@
{"client":{"user":"foo","host":"localhost","password":"bar"}}

4
tests/fixtures/generic/ini-single-quote.ini vendored Normal file
View File

@ -0,0 +1,4 @@
[client]
user=foo
host=localhost
password='bar'

1
tests/fixtures/generic/ini-single-quote.json vendored Normal file
View File

@ -0,0 +1 @@
{"client":{"user":"foo","host":"localhost","password":"bar"}}

1
tests/fixtures/generic/postconf-M.json vendored Normal file
View File

File diff suppressed because one or more lines are too long

31
tests/fixtures/generic/postconf-M.out vendored Normal file
View File

@ -0,0 +1,31 @@
smtp inet n - y - - smtpd
pickup unix n - y 60 1 pickup
cleanup unix n - y - 0 cleanup
qmgr unix n - n 300 1 qmgr
tlsmgr unix - - y 1000? 1 tlsmgr
rewrite unix - - y - - trivial-rewrite
bounce unix - - y - 0 bounce
defer unix - - y - 0 bounce
trace unix - - y - 0 bounce
verify unix - - y - 1 verify
flush unix n - y 1000? 0 flush
proxymap unix - - n - - proxymap
proxywrite unix - - n - 1 proxymap
smtp unix - - y - - smtp
relay unix - - y - - smtp -o syslog_name=postfix/$service_name
showq unix n - y - - showq
error unix - - y - - error
retry unix - - y - - error
discard unix - - y - - discard
local unix - n n - - local
virtual unix - n n - - virtual
lmtp unix - - y - - lmtp
anvil unix - - y - 1 anvil
scache unix - - y - 1 scache
postlog unix-dgram n - n - 1 postlogd
maildrop unix - n n - - pipe flags=DRXhu user=vmail argv=/usr/bin/maildrop -d ${recipient}
uucp unix - n n - - pipe flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient)
ifmail unix - n n - - pipe flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient)
bsmtp unix - n n - - pipe flags=Fq. user=bsmtp argv=/usr/lib/bsmtp/bsmtp -t$nexthop -f$sender $recipient
scalemail-backend unix - n n - 2 pipe flags=R user=scalemail argv=/usr/lib/scalemail/bin/scalemail-store ${nexthop} ${user} ${extension}
mailman unix - n n - - pipe flags=FRX user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py ${nexthop} ${user}

43
tests/test_asciitable.py
View File

@ -344,6 +344,49 @@ Internet 10.12.13.4 198 0950.5C8A.5c41 ARPA GigabitEthernet2.17
self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected) self.assertEqual(jc.parsers.asciitable.parse(input, quiet=True), expected)
def test_asciitable_no_lower_raw(self):
"""
Test 'asciitable' with a pure ASCII table that has special
characters and mixed case in the header. These should be converted to underscores
and no trailing or consecutive underscores should end up in the
resulting key names. Using `raw` in this test to preserve case. (no lower)
"""
input = '''
Protocol Address Age (min) Hardware Addr Type Interface
Internet 10.12.13.1 98 0950.5785.5cd1 ARPA FastEthernet2.13
Internet 10.12.13.3 131 0150.7685.14d5 ARPA GigabitEthernet2.13
Internet 10.12.13.4 198 0950.5C8A.5c41 ARPA GigabitEthernet2.17
'''
expected = [
{
"Protocol": "Internet",
"Address": "10.12.13.1",
"Age_min": "98",
"Hardware_Addr": "0950.5785.5cd1",
"Type": "ARPA",
"Interface": "FastEthernet2.13"
},
{
"Protocol": "Internet",
"Address": "10.12.13.3",
"Age_min": "131",
"Hardware_Addr": "0150.7685.14d5",
"Type": "ARPA",
"Interface": "GigabitEthernet2.13"
},
{
"Protocol": "Internet",
"Address": "10.12.13.4",
"Age_min": "198",
"Hardware_Addr": "0950.5C8A.5c41",
"Type": "ARPA",
"Interface": "GigabitEthernet2.17"
}
]
self.assertEqual(jc.parsers.asciitable.parse(input, raw=True, quiet=True), expected)
def test_asciitable_centered_col_header(self): def test_asciitable_centered_col_header(self):
""" """
Test 'asciitable' with long centered column header which can break Test 'asciitable' with long centered column header which can break

28
tests/test_asciitable_m.py
View File

@ -270,6 +270,34 @@ class MyTests(unittest.TestCase):
self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected) self.assertEqual(jc.parsers.asciitable_m.parse(input, quiet=True), expected)
def test_asciitable_no_lower_raw(self):
"""
Test 'asciitable_m' with a pure ASCII table that has special
characters and mixed case in the header. These should be converted to underscores
and no trailing or consecutive underscores should end up in the
resulting key names. Using `raw` in this test to preserve case. (no lower)
"""
input = '''
+----------+------------+-----------+----------------+-------+--------------------+
| Protocol | Address | Age (min) | Hardware Addr | Type | Interface |
| | | of int | | | |
+----------+------------+-----------+----------------+-------+--------------------+
| Internet | 10.12.13.1 | 98 | 0950.5785.5cd1 | ARPA | FastEthernet2.13 |
+----------+------------+-----------+----------------+-------+--------------------+
'''
expected = [
{
"Protocol": "Internet",
"Address": "10.12.13.1",
"Age_min_of_int": "98",
"Hardware_Addr": "0950.5785.5cd1",
"Type": "ARPA",
"Interface": "FastEthernet2.13"
}
]
self.assertEqual(jc.parsers.asciitable_m.parse(input, raw=True, quiet=True), expected)
def test_asciitable_m_sep_char_in_cell(self): def test_asciitable_m_sep_char_in_cell(self):
""" """
Test 'asciitable_m' with a column separator character inside the data Test 'asciitable_m' with a column separator character inside the data

5
tests/test_cli.py
View File

@ -25,7 +25,10 @@ class MyTests(unittest.TestCase):
'jc -h': (False, None, None, []), 'jc -h': (False, None, None, []),
'jc -h --arp': (False, None, None, []), 'jc -h --arp': (False, None, None, []),
'jc -h arp': (False, None, None, []), 'jc -h arp': (False, None, None, []),
'jc -h arp -a': (False, None, None, []) 'jc -h arp -a': (False, None, None, []),
'jc --pretty dig': (True, ['dig'], '--dig', ['p']),
'jc --pretty --monochrome --quiet --raw dig': (True, ['dig'], '--dig', ['p', 'm', 'q', 'r']),
'jc --about --yaml-out': (False, None, None, [])
} }
for command, expected_command in commands.items(): for command, expected_command in commands.items():

14
tests/test_id.py
View File

@ -29,6 +29,20 @@ class MyTests(unittest.TestCase):
""" """
self.assertEqual(jc.parsers.id.parse('', quiet=True), {}) self.assertEqual(jc.parsers.id.parse('', quiet=True), {})
def test_id_no_name(self):
"""
Test 'id' with no name
"""
self.assertEqual(
jc.parsers.id.parse('uid=1000 gid=1000 groups=1000,10', quiet=True),
{'uid': {'id': 1000, 'name': None}, 'gid': {'id': 1000, 'name': None}, 'groups': [{'id': 1000, 'name': None}, {'id': 10, 'name': None}]}
)
self.assertEqual(
jc.parsers.id.parse('uid=1000(user) gid=1000 groups=1000,10(wheel)', quiet=True),
{'uid': {'id': 1000, 'name': 'user'}, 'gid': {'id': 1000, 'name': None}, 'groups': [{'id': 1000, 'name': None}, {'id': 10, 'name': 'wheel'}]}
)
def test_id_centos_7_7(self): def test_id_centos_7_7(self):
""" """
Test 'id' on Centos 7.7 Test 'id' on Centos 7.7

25
tests/test_ini.py
View File

@ -16,6 +16,12 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ini-iptelserver.ini'), 'r', encoding='utf-8') as f: with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ini-iptelserver.ini'), 'r', encoding='utf-8') as f:
self.generic_ini_iptelserver = f.read() self.generic_ini_iptelserver = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ini-double-quote.ini'), 'r', encoding='utf-8') as f:
self.generic_ini_double_quote = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ini-single-quote.ini'), 'r', encoding='utf-8') as f:
self.generic_ini_single_quote = f.read()
# output # output
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ini-test.json'), 'r', encoding='utf-8') as f: with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ini-test.json'), 'r', encoding='utf-8') as f:
self.generic_ini_test_json = json.loads(f.read()) self.generic_ini_test_json = json.loads(f.read())
@ -23,6 +29,12 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ini-iptelserver.json'), 'r', encoding='utf-8') as f: with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ini-iptelserver.json'), 'r', encoding='utf-8') as f:
self.generic_ini_iptelserver_json = json.loads(f.read()) self.generic_ini_iptelserver_json = json.loads(f.read())
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ini-double-quote.json'), 'r', encoding='utf-8') as f:
self.generic_ini_double_quote_json = json.loads(f.read())
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ini-single-quote.json'), 'r', encoding='utf-8') as f:
self.generic_ini_single_quote_json = json.loads(f.read())
def test_ini_nodata(self): def test_ini_nodata(self):
""" """
Test the test ini file with no data Test the test ini file with no data
@ -53,6 +65,19 @@ duplicate_key = value2
expected = {'duplicate_key': 'value2', 'another_key': 'foo'} expected = {'duplicate_key': 'value2', 'another_key': 'foo'}
self.assertEqual(jc.parsers.ini.parse(data, quiet=True), expected) self.assertEqual(jc.parsers.ini.parse(data, quiet=True), expected)
def test_ini_doublequote(self):
"""
Test ini file with double quotes around a value
"""
self.assertEqual(jc.parsers.ini.parse(self.generic_ini_double_quote, quiet=True), self.generic_ini_double_quote_json)
def test_ini_singlequote(self):
"""
Test ini file with single quotes around a value
"""
self.assertEqual(jc.parsers.ini.parse(self.generic_ini_single_quote, quiet=True), self.generic_ini_single_quote_json)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

35
tests/test_postconf.py Normal file
View File

@ -0,0 +1,35 @@
import os
import unittest
import json
import jc.parsers.postconf
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
class MyTests(unittest.TestCase):
def setUp(self):
# input
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/postconf-M.out'), 'r', encoding='utf-8') as f:
self.generic_postconf_m = f.read()
# output
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/postconf-M.json'), 'r', encoding='utf-8') as f:
self.generic_postconf_m_json = json.loads(f.read())
def test_postconf_nodata(self):
"""
Test 'postconf' with no data
"""
self.assertEqual(jc.parsers.postconf.parse('', quiet=True), [])
def test_postconf(self):
"""
Test 'postconf -M'
"""
self.assertEqual(jc.parsers.postconf.parse(self.generic_postconf_m, quiet=True), self.generic_postconf_m_json)
if __name__ == '__main__':
unittest.main()