From 4f21c7b7b4b0f815fc340cfa1404a4b249cfc684 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 21 Aug 2022 12:38:55 -0700 Subject: [PATCH] change --time-out to --meta-out. add more meta fields --- README.md | 2 +- completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 6 ++-- jc/cli.py | 37 +++++++++++++++++-------- jc/cli_data.py | 2 +- man/jc.1 | 10 +++---- templates/manpage_template | 8 +++--- templates/readme_template | 2 +- tests/{test_cli.py => test_jc_cli.py} | 40 ++++++++++++++++----------- 9 files changed, 65 insertions(+), 44 deletions(-) rename tests/{test_cli.py => test_jc_cli.py} (86%) diff --git a/README.md b/README.md index e4b0f32d..91352111 100644 --- a/README.md +++ b/README.md @@ -280,10 +280,10 @@ option. | `-d` | `--debug` | Debug mode. Prints trace messages if parsing issues are encountered (use`-dd` for verbose debugging) | | `-h` | `--help` | Help. Use `jc -h --parser_name` for parser documentation | | `-m` | `--monochrome` | Monochrome output | +| `-M` | `--meta-out` | Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. | | | `-p` | `--pretty` | Pretty format the JSON output | | `-q` | `--quiet` | Quiet mode. Suppresses parser warning messages (use `-qq` to ignore streaming parser errors) | | `-r` | `--raw` | Raw output. Provides more literal output, typically with string values and no additional semantic processing | -| `-t` | `--time-out` | Add UTC Unix timestamp information to output | | `-u` | `--unbuffer` | Unbuffer output | | `-v` | `--version` | Version information | | `-y` | `--yaml-out` | YAML output | diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index 8b2901d6..3fa432e0 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -5,7 +5,7 @@ _jc() jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --cef-s --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) - jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --time-out -t --unbuffer -u --yaml-out -y) + jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) jc_help_options=(--help -h) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index d89e4c7c..ec87a73b 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -218,7 +218,7 @@ _jc() { '--yaml:YAML file parser' '--zipinfo:`zipinfo` command parser' ) - jc_options=(--force-color -C --debug -d --monochrome -m --pretty -p --quiet -q --raw -r --time-out -t --unbuffer -u --yaml-out -y) + jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_options_describe=( '--force-color:force color output even when using pipes (overrides -m)' '-C:force color output even when using pipes (overrides -m)' @@ -226,14 +226,14 @@ _jc() { '-d:debug (double for verbose debug)' '--monochrome:monochrome output' '-m:monochrome output' + '--meta-out:add metadata to output including timestamp, etc.' + '-M:add metadata to output including timestamp, etc.' '--pretty:pretty print output' '-p:pretty print output' '--quiet:suppress warnings (double to ignore streaming errors)' '-q:suppress warnings (double to ignore streaming errors)' '--raw:raw output' '-r:raw output' - '--time-out:add UTC Unix timestamp information to output' - '-t:add UTC Unix timestamp information to output' '--unbuffer:unbuffer output' '-u:unbuffer output' '--yaml-out:YAML output' diff --git a/jc/cli.py b/jc/cli.py index cffd4c14..b3ffff99 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -445,28 +445,41 @@ def combined_exit_code(program_exit=0, jc_exit=0): return exit_code -def add_timestamp_to(list_or_dict, runtime, magic_exit_code): +def add_metadata_to(list_or_dict, + runtime=None, + run_command=None, + magic_exit_code=None, + parser_name=None): """ This function mutates a list or dict in place. If the _jc_meta field - does not already exist, it will be created with the timestamp field. If - the _jc_meta field already exists, the timestamp will be added to the - existing object. + does not already exist, it will be created with the metadata fields. If + the _jc_meta field already exists, the metadata fields will be added to + the existing object. """ run_timestamp = runtime.timestamp() - timestamp_obj = {'timestamp': run_timestamp} + + if not run_command: + magic_exit_code = None + + meta_obj = { + 'parser': parser_name, + 'magic_command': run_command, + 'magic_command_exit': magic_exit_code, + 'timestamp': run_timestamp + } if isinstance(list_or_dict, dict): if '_jc_meta' not in list_or_dict: list_or_dict['_jc_meta'] = {} - list_or_dict['_jc_meta'].update(timestamp_obj) + list_or_dict['_jc_meta'].update(meta_obj) elif isinstance(list_or_dict, list): for item in list_or_dict: if '_jc_meta' not in item: item['_jc_meta'] = {} - item['_jc_meta'].update(timestamp_obj) + item['_jc_meta'].update(meta_obj) else: utils.error_message(['Parser returned an unsupported object type.']) @@ -517,7 +530,7 @@ def main(): quiet = 'q' in options ignore_exceptions = options.count('q') > 1 raw = 'r' in options - timestamp = 't' in options + meta_out = 'M' in options unbuffer = 'u' in options version_info = 'v' in options yaml_out = 'y' in options @@ -632,9 +645,9 @@ def main(): ignore_exceptions=ignore_exceptions) for line in result: - if timestamp: + if meta_out: run_dt_utc = datetime.now(timezone.utc) - add_timestamp_to(line, run_dt_utc, magic_exit_code) + add_metadata_to(line, run_dt_utc, run_command, magic_exit_code, parser_name) safe_print_out(line, pretty=pretty, @@ -661,9 +674,9 @@ def main(): raw=raw, quiet=quiet) - if timestamp: + if meta_out: run_dt_utc = datetime.now(timezone.utc) - add_timestamp_to(result, run_dt_utc, magic_exit_code) + add_metadata_to(result, run_dt_utc, run_command, magic_exit_code, parser_name) safe_print_out(result, pretty=pretty, diff --git a/jc/cli_data.py b/jc/cli_data.py index 489f2126..7147e897 100644 --- a/jc/cli_data.py +++ b/jc/cli_data.py @@ -7,10 +7,10 @@ long_options_map: Dict[str, List[str]] = { '--debug': ['d', 'debug (double for verbose debug)'], '--help': ['h', 'help (--help --parser_name for parser documentation)'], '--monochrome': ['m', 'monochrome output'], + '--meta-out': ['M', 'add metadata to output including timestamp, etc.'], '--pretty': ['p', 'pretty print output'], '--quiet': ['q', 'suppress warnings (double to ignore streaming errors)'], '--raw': ['r', 'raw output'], - '--time-out': ['t', 'add UTC Unix timestamp information to output'], '--unbuffer': ['u', 'unbuffer output'], '--version': ['v', 'version info'], '--yaml-out': ['y', 'YAML output'], diff --git a/man/jc.1 b/man/jc.1 index 002d11a0..2e9f1711 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-20 1.21.0 "JSON Convert" +.TH jc 1 2022-08-21 1.21.0 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -646,6 +646,10 @@ Help (\fB--help --parser_name\fP for parser documentation) Monochrome output .TP .B +\fB-M\fP, \fB--meta-out\fP +Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. +.TP +.B \fB-p\fP, \fB--pretty\fP Pretty print output .TP @@ -658,10 +662,6 @@ Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming pars Raw output. Provides more literal output, typically with string values and no additional semantic processing .TP .B -\fB-t\fP, \fB--time-out\fP -Add UTC Unix timestamp information to output -.TP -.B \fB-u\fP, \fB--unbuffer\fP Unbuffer output (useful for slow streaming data with streaming parsers) .TP diff --git a/templates/manpage_template b/templates/manpage_template index 26a97307..7c6c7d33 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -51,6 +51,10 @@ Help (\fB--help --parser_name\fP for parser documentation) Monochrome output .TP .B +\fB-M\fP, \fB--meta-out\fP +Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. +.TP +.B \fB-p\fP, \fB--pretty\fP Pretty print output .TP @@ -63,10 +67,6 @@ Quiet mode. Suppresses parser warning messages (use -qq to ignore streaming pars Raw output. Provides more literal output, typically with string values and no additional semantic processing .TP .B -\fB-t\fP, \fB--time-out\fP -Add UTC Unix timestamp information to output -.TP -.B \fB-u\fP, \fB--unbuffer\fP Unbuffer output (useful for slow streaming data with streaming parsers) .TP diff --git a/templates/readme_template b/templates/readme_template index 32c5a0aa..c4e64d89 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -161,10 +161,10 @@ option. | `-d` | `--debug` | Debug mode. Prints trace messages if parsing issues are encountered (use`-dd` for verbose debugging) | | `-h` | `--help` | Help. Use `jc -h --parser_name` for parser documentation | | `-m` | `--monochrome` | Monochrome output | +| `-M` | `--meta-out` | Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. | | | `-p` | `--pretty` | Pretty format the JSON output | | `-q` | `--quiet` | Quiet mode. Suppresses parser warning messages (use `-qq` to ignore streaming parser errors) | | `-r` | `--raw` | Raw output. Provides more literal output, typically with string values and no additional semantic processing | -| `-t` | `--time-out` | Add UTC Unix timestamp information to output | | `-u` | `--unbuffer` | Unbuffer output | | `-v` | `--version` | Version information | | `-y` | `--yaml-out` | YAML output | diff --git a/tests/test_cli.py b/tests/test_jc_cli.py similarity index 86% rename from tests/test_cli.py rename to tests/test_jc_cli.py index 3368f346..48a642b3 100644 --- a/tests/test_cli.py +++ b/tests/test_jc_cli.py @@ -255,39 +255,47 @@ class MyTests(unittest.TestCase): self.assertGreaterEqual(jc.cli.about_jc()['parser_count'], 55) self.assertEqual(jc.cli.about_jc()['parser_count'], len(jc.cli.about_jc()['parsers'])) - def test_add_timestamp_to_simple_dict(self): + def test_add_meta_to_simple_dict(self): list_or_dict = {'a': 1, 'b': 2} runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) - magic_exit_code = 0 - expected = {'a': 1, 'b': 2, '_jc_meta': {'timestamp': 1659659829.273349}} - jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + magic_exit_code = 2 + run_command = ['ping', '-c3', '192.168.1.123'] + parser_name = 'ping' + expected = {'a': 1, 'b': 2, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}} + jc.cli.add_metadata_to(list_or_dict, runtime, run_command, magic_exit_code, parser_name) self.assertEqual(list_or_dict, expected) - def test_add_timestamp_to_simple_list(self): + def test_add_meta_to_simple_list(self): list_or_dict = [{'a': 1, 'b': 2},{'a': 3, 'b': 4}] runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) - magic_exit_code = 0 - expected = [{'a': 1, 'b': 2, '_jc_meta': {'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'timestamp': 1659659829.273349}}] - jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + magic_exit_code = 2 + run_command = ['ping', '-c3', '192.168.1.123'] + parser_name = 'ping' + expected = [{'a': 1, 'b': 2, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}] + jc.cli.add_metadata_to(list_or_dict, runtime, run_command, magic_exit_code, parser_name) self.assertEqual(list_or_dict, expected) - def test_add_timestamp_to_dict_existing_meta(self): + def test_add_meta_to_dict_existing_meta(self): list_or_dict = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar'}} runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) - magic_exit_code = 0 - expected = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'timestamp': 1659659829.273349}} - jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + magic_exit_code = 2 + run_command = ['ping', '-c3', '192.168.1.123'] + parser_name = 'ping' + expected = {'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}} + jc.cli.add_metadata_to(list_or_dict, runtime, run_command, magic_exit_code, parser_name) self.assertEqual(list_or_dict, expected) - def test_add_timestamp_to_list_existing_meta(self): + def test_add_meta_to_list_existing_meta(self): list_or_dict = [{'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar'}},{'a': 3, 'b': 4, '_jc_meta': {'foo': 'bar'}}] runtime = datetime(2022, 8, 5, 0, 37, 9, 273349, tzinfo=timezone.utc) - magic_exit_code = 0 - expected = [{'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'foo': 'bar', 'timestamp': 1659659829.273349}}] - jc.cli.add_timestamp_to(list_or_dict, runtime, magic_exit_code) + magic_exit_code = 2 + run_command = ['ping', '-c3', '192.168.1.123'] + parser_name = 'ping' + expected = [{'a': 1, 'b': 2, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}, {'a': 3, 'b': 4, '_jc_meta': {'foo': 'bar', 'parser': 'ping', 'magic_command': ['ping', '-c3', '192.168.1.123'], 'magic_command_exit': 2, 'timestamp': 1659659829.273349}}] + jc.cli.add_metadata_to(list_or_dict, runtime, run_command, magic_exit_code, parser_name) self.assertEqual(list_or_dict, expected)