Merge pull request #353 from kellyjonbrazil/dev

Dev v1.22.5

.github/workflows/pythonapp.yml

@@ -13,8 +13,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        os: [macos-latest, ubuntu-20.04, windows-latest]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3

CHANGELOG

@@ -1,5 +1,41 @@
 jc changelog
 
+20230111 v1.22.5
+- Add TOML file parser
+- Add INI with duplicate key support file parser
+- Add AIX support for the `arp` command parser
+- Add AIX support for the `mount` command parser
+- Fix `lsusb` command parser when extra hub port status information is output
+- Refactor `lsusb` command parser for more code reuse
+- Fix INI file parser to include top-level values with no section header
+- Fix INI file parser to not specially handle the [DEFAULT] section
+- Fix INI file and Key/Value parsers to only remove one quotation mark from the
+  beginning and end of values.
+- Update copyright dates
+
+20221230 v1.22.4
+- Add `iwconfig` command parser
+- Add NeXTSTEP format support to the PLIST file parser
+- Fix `proc` parser magic signature detection for `/proc/pid/stat` hacks
+- Fix `x509-cert` parser for string serial numbers
+- Add category tags to parser metadata: generic, standard, file, string, binary, command
+- Add "list parsers by category" view to help
+- Fix python 3.6-related issues
+- Add python 3.6 to automated tests
+
+20221216 v1.22.3
+- Add Common Log Format and Combined Log Format file parser (standard and streaming)
+- Add PostgreSQL password file parser
+- Add openvpn-status.log file parser
+- Add `cbt` command parser (Google Big Table)
+- Enhance `ifconfig` parser with interface lane information on BSD
+- Enhance `ifconfig` parser with additional IPv6 `scope_id` info for BSD
+- Fix `ifconfig` parser to capture some IPv6 addresses missed on BSD
+- Fix `git-log` and `git-log-s` parsers for failure on empty author name
+- Update `os-prober` parser with split EFI partition fields
+- Add ISO string attribute (`.iso`) to `jc.utils.timestamp()`
+- Fix several documentation typos
+
 20221107 v1.22.2
 - add `sshd_conf` parser for `sshd` configuration files and `sshd -T` output
 - add `findmnt` command parser
@@ -581,7 +617,7 @@ jc changelog
 
 20200211 v1.7.3
 - Add alternative 'magic' syntax: e.g. `jc ls -al`
-- Options can now be condensed (e.g. -prq is equivalant to -p -r -q)
+- Options can now be condensed (e.g. -prq is equivalent to -p -r -q)
 
 20200208 v1.7.2
 - Include test fixtures in wheel and sdist

EXAMPLES.md

@@ -1636,21 +1636,21 @@ cat example.ini | jc --ini -p
 ```
 ```json
 {
-  "bitbucket.org": {
-    "ServeraLiveInterval": "45",
+  "DEFAULT": {
+    "ServerAliveInterval": "45",
     "Compression": "yes",
     "CompressionLevel": "9",
-    "ForwardX11": "yes",
+    "ForwardX11": "yes"
+  },
+  "bitbucket.org": {
     "User": "hg"
   },
   "topsecret.server.com": {
-    "ServeraLiveInterval": "45",
-    "Compression": "yes",
-    "CompressionLevel": "9",
-    "ForwardX11": "no",
-    "Port": "50022"
+    "Port": "50022",
+    "ForwardX11": "no"
   }
 }
+
 ```
 ### iostat
 ```bash

README.md

@@ -161,10 +161,13 @@ option.
 | `   --asciitable` | ASCII and Unicode table parser                          | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable)     |
 | ` --asciitable-m` | multi-line ASCII and Unicode table parser               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/asciitable_m)   |
 | `        --blkid` | `blkid` command parser                                  | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/blkid)          |
+| `          --cbt` | `cbt` (Google Bigtable) command parser                  | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cbt)            |
 | `          --cef` | CEF string parser                                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef)            |
 | `        --cef-s` | CEF string streaming parser                             | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cef_s)          |
 | `        --chage` | `chage --list` command parser                           | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/chage)          |
 | `        --cksum` | `cksum` and `sum` command parser                        | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/cksum)          |
+| `          --clf` | Common and Combined Log Format file parser              | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/clf)            |
+| `        --clf-s` | Common and Combined Log Format file streaming parser    | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/clf_s)          |
 | `      --crontab` | `crontab` command and file parser                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/crontab)        |
 | `    --crontab-u` | `crontab` file parser with user support                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/crontab_u)      |
 | `          --csv` | CSV file parser                                         | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/csv)            |
@@ -198,15 +201,17 @@ option.
 | `           --id` | `id` command parser                                     | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/id)             |
 | `     --ifconfig` | `ifconfig` command parser                               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ifconfig)       |
 | `          --ini` | INI file parser                                         | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini)            |
+| `      --ini-dup` | INI with duplicate key file parser                      | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini_dup)        |
 | `       --iostat` | `iostat` command parser                                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat)         |
 | `     --iostat-s` | `iostat` command streaming parser                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat_s)       |
 | `   --ip-address` | IPv4 and IPv6 Address string parser                     | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ip_address)     |
 | `     --iptables` | `iptables` command parser                               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iptables)       |
 | `      --iw-scan` | `iw dev [device] scan` command parser                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iw_scan)        |
+| `     --iwconfig` | `iwconfig` command parser                               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iwconfig)       |
 | ` --jar-manifest` | Java MANIFEST.MF file parser                            | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jar_manifest)   |
 | `         --jobs` | `jobs` command parser                                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jobs)           |
 | `          --jwt` | JWT string parser                                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jwt)            |
-| `           --kv` | Key/Value file parser                                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/kv)             |
+| `           --kv` | Key/Value file and string parser                        | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/kv)             |
 | `         --last` | `last` and `lastb` command parser                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/last)           |
 | `           --ls` | `ls` command parser                                     | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ls)             |
 | `         --ls-s` | `ls` command streaming parser                           | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ls_s)           |
@@ -223,9 +228,11 @@ option.
 | `      --netstat` | `netstat` command parser                                | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/netstat)        |
 | `        --nmcli` | `nmcli` command parser                                  | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/nmcli)          |
 | `         --ntpq` | `ntpq -p` command parser                                | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ntpq)           |
+| `      --openvpn` | openvpn-status.log file parser                          | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/openvpn)        |
 | `    --os-prober` | `os-prober` command parser                              | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/os_prober)      |
 | `       --passwd` | `/etc/passwd` file parser                               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/passwd)         |
 | `      --pci-ids` | `pci.ids` file parser                                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pci_ids)        |
+| `       --pgpass` | PostgreSQL password file parser                         | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pgpass)         |
 | `      --pidstat` | `pidstat -H` command parser                             | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pidstat)        |
 | `    --pidstat-s` | `pidstat -H` command streaming parser                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pidstat_s)      |
 | `         --ping` | `ping` and `ping6` command parser                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ping)           |
@@ -260,6 +267,7 @@ option.
 | `         --time` | `/usr/bin/time` command parser                          | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/time)           |
 | `  --timedatectl` | `timedatectl status` command parser                     | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timedatectl)    |
 | `    --timestamp` | Unix Epoch Timestamp string parser                      | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/timestamp)      |
+| `         --toml` | TOML file parser                                        | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/toml)           |
 | `          --top` | `top -b` command parser                                 | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/top)            |
 | `        --top-s` | `top -b` command streaming parser                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/top_s)          |
 | `    --tracepath` | `tracepath` and `tracepath6` command parser             | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/tracepath)      |
@@ -390,7 +398,7 @@ option.
 ### Streaming Parsers
 Most parsers load all of the data from `STDIN`, parse it, then output the entire
 JSON document serially. There are some streaming parsers (e.g. `ls-s` and
-`ping-s`) that immediately start processing and outputing the data line-by-line
+`ping-s`) that immediately start processing and outputting the data line-by-line
 as [JSON Lines](https://jsonlines.org/) (aka [NDJSON](http://ndjson.org/)) while
 it is being received from `STDIN`. This can significantly reduce the amount of
 memory required to parse large amounts of command output (e.g. `ls -lR /`) and
@@ -750,37 +758,31 @@ ifconfig | jc -p --ifconfig          # or:  jc -p ifconfig
 cat example.ini
 ```
 ```
-[DEFAULT]
-ServerAliveInterval = 45
-Compression = yes
-CompressionLevel = 9
-ForwardX11 = yes
+foo = fiz
+bar = buz
 
-[bitbucket.org]
-User = hg
+[section1]
+fruit = apple
+color = blue
 
-[topsecret.server.com]
-Port = 50022
-ForwardX11 = no
+[section2]
+fruit = pear
+color = green
 ```
 ```bash
 cat example.ini | jc -p --ini
 ```
 ```json
 {
-  "bitbucket.org": {
-    "ServeraLiveInterval": "45",
-    "Compression": "yes",
-    "CompressionLevel": "9",
-    "ForwardX11": "yes",
-    "User": "hg"
+  "foo": "fiz",
+  "bar": "buz",
+  "section1": {
+    "fruit": "apple",
+    "color": "blue"
   },
-  "topsecret.server.com": {
-    "ServeraLiveInterval": "45",
-    "Compression": "yes",
-    "CompressionLevel": "9",
-    "ForwardX11": "no",
-    "Port": "50022"
+  "section2": {
+    "fruit": "pear",
+    "color": "green"
   }
 }
 ```
@@ -1236,4 +1238,4 @@ cat istio.yaml | jc -p --yaml
 ]
 ```
 
-© 2019-2022 Kelly Brazil
+© 2019-2023 Kelly Brazil

completions/jc_bash_completion.sh

@@ -3,8 +3,8 @@ _jc()
     local cur prev words cword jc_commands jc_parsers jc_options \
           jc_about_options jc_about_mod_options jc_help_options jc_special_options
 
-    jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo)
-    jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --cef-s --chage --cksum --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --os-prober --passwd --pci-ids --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo)
+    jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo)
+    jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo)
     jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y)
     jc_about_options=(--about -a)
     jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C)

completions/jc_zsh_completion.sh

@@ -9,12 +9,13 @@ _jc() {
              jc_help_options jc_help_options_describe \
              jc_special_options jc_special_options_describe
 
-    jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo)
+    jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo)
     jc_commands_describe=(
         'acpi:run "acpi" command with magic syntax.'
         'airport:run "airport" command with magic syntax.'
         'arp:run "arp" command with magic syntax.'
         'blkid:run "blkid" command with magic syntax.'
+        'cbt:run "cbt" command with magic syntax.'
         'chage:run "chage" command with magic syntax.'
         'cksum:run "cksum" command with magic syntax.'
         'crontab:run "crontab" command with magic syntax.'
@@ -37,6 +38,7 @@ _jc() {
         'iostat:run "iostat" command with magic syntax.'
         'iptables:run "iptables" command with magic syntax.'
         'iw:run "iw" command with magic syntax.'
+        'iwconfig:run "iwconfig" command with magic syntax.'
         'jobs:run "jobs" command with magic syntax.'
         'last:run "last" command with magic syntax.'
         'lastb:run "lastb" command with magic syntax.'
@@ -100,7 +102,7 @@ _jc() {
         'xrandr:run "xrandr" command with magic syntax.'
         'zipinfo:run "zipinfo" command with magic syntax.'
     )
-    jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --cef-s --chage --cksum --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --os-prober --passwd --pci-ids --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo)
+    jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo)
     jc_parsers_describe=(
         '--acpi:`acpi` command parser'
         '--airport:`airport -I` command parser'
@@ -109,10 +111,13 @@ _jc() {
         '--asciitable:ASCII and Unicode table parser'
         '--asciitable-m:multi-line ASCII and Unicode table parser'
         '--blkid:`blkid` command parser'
+        '--cbt:`cbt` (Google Bigtable) command parser'
         '--cef:CEF string parser'
         '--cef-s:CEF string streaming parser'
         '--chage:`chage --list` command parser'
         '--cksum:`cksum` and `sum` command parser'
+        '--clf:Common and Combined Log Format file parser'
+        '--clf-s:Common and Combined Log Format file streaming parser'
         '--crontab:`crontab` command and file parser'
         '--crontab-u:`crontab` file parser with user support'
         '--csv:CSV file parser'
@@ -146,15 +151,17 @@ _jc() {
         '--id:`id` command parser'
         '--ifconfig:`ifconfig` command parser'
         '--ini:INI file parser'
+        '--ini-dup:INI with duplicate key file parser'
         '--iostat:`iostat` command parser'
         '--iostat-s:`iostat` command streaming parser'
         '--ip-address:IPv4 and IPv6 Address string parser'
         '--iptables:`iptables` command parser'
         '--iw-scan:`iw dev [device] scan` command parser'
+        '--iwconfig:`iwconfig` command parser'
         '--jar-manifest:Java MANIFEST.MF file parser'
         '--jobs:`jobs` command parser'
         '--jwt:JWT string parser'
-        '--kv:Key/Value file parser'
+        '--kv:Key/Value file and string parser'
         '--last:`last` and `lastb` command parser'
         '--ls:`ls` command parser'
         '--ls-s:`ls` command streaming parser'
@@ -171,9 +178,11 @@ _jc() {
         '--netstat:`netstat` command parser'
         '--nmcli:`nmcli` command parser'
         '--ntpq:`ntpq -p` command parser'
+        '--openvpn:openvpn-status.log file parser'
         '--os-prober:`os-prober` command parser'
         '--passwd:`/etc/passwd` file parser'
         '--pci-ids:`pci.ids` file parser'
+        '--pgpass:PostgreSQL password file parser'
         '--pidstat:`pidstat -H` command parser'
         '--pidstat-s:`pidstat -H` command streaming parser'
         '--ping:`ping` and `ping6` command parser'
@@ -257,6 +266,7 @@ _jc() {
         '--time:`/usr/bin/time` command parser'
         '--timedatectl:`timedatectl status` command parser'
         '--timestamp:Unix Epoch Timestamp string parser'
+        '--toml:TOML file parser'
         '--top:`top -b` command parser'
         '--top-s:`top -b` command streaming parser'
         '--tracepath:`tracepath` and `tracepath6` command parser'
@@ -283,8 +293,8 @@ _jc() {
     )
     jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y)
     jc_options_describe=(
-        '--force-color:force color output even when using pipes (overrides -m)'
-        '-C:force color output even when using pipes (overrides -m)'
+        '--force-color:force color output (overrides -m)'
+        '-C:force color output (overrides -m)'
         '--debug:debug (double for verbose debug)'
         '-d:debug (double for verbose debug)'
         '--monochrome:monochrome output'
@@ -315,8 +325,8 @@ _jc() {
         '-y:YAML output'
         '--monochrome:monochrome output'
         '-m:monochrome output'
-        '--force-color:force color output even when using pipes (overrides -m)'
-        '-C:force color output even when using pipes (overrides -m)'
+        '--force-color:force color output (overrides -m)'
+        '-C:force color output (overrides -m)'
     )
     jc_help_options=(--help -h)
     jc_help_options_describe=(

docs/parsers/arp.md

@@ -140,4 +140,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, aix, freebsd, darwin
 
-Version 1.11 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.12 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/cbt.md

@@ -0,0 +1,125 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+<a id="jc.parsers.cbt"></a>
+
+# jc.parsers.cbt
+
+jc - JSON Convert `cbt` command output parser (Google Bigtable)
+
+Parses the human-, but not machine-, friendly output of the cbt command (for
+Google's Bigtable).
+
+No effort is made to convert the data types of the values in the cells.
+
+The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on
+the local time of the system the parser is run on)
+
+The `timestamp_epoch_utc` calculated timestamp field is timezone-aware and
+is only available if the timestamp has a UTC timezone.
+
+The `timestamp_iso` calculated timestamp field will only include UTC
+timezone information if the timestamp has a UTC timezone.
+
+Raw output contains all cells for each column (including timestamps), while
+the normal output contains only the latest value for each column.
+
+Usage (cli):
+
+    $ cbt | jc --cbt
+
+or
+
+    $ jc cbt
+
+Usage (module):
+
+    import jc
+    result = jc.parse('cbt', cbt_command_output)
+
+Schema:
+
+    [
+      {
+        "key":                      string,
+        "cells": {
+          <string>: {                         # column family
+            <string>:               string    # column: value
+          }
+        }
+      }
+    ]
+
+Schema (raw):
+
+    [
+      {
+        "key":                      string,
+        "cells": [
+          {
+            "column_family":        string,
+            "column":               string,
+            "value":                string,
+            "timestamp_iso":        string,
+            "timestamp_epoch":      integer,
+            "timestamp_epoch_utc":  integer
+          }
+        ]
+      }
+    ]
+
+Examples:
+
+    $ cbt -project=$PROJECT -instance=$INSTANCE lookup $TABLE foo | jc --cbt -p
+    [
+      {
+        "key": "foo",
+        "cells": {
+          "foo": {
+            "bar": "baz"
+          }
+        }
+      }
+    ]
+
+    $ cbt -project=$PROJECT -instance=$INSTANCE lookup $TABLE foo | jc --cbt -p -r
+    [
+      {
+        "key": "foo",
+        "cells": [
+          {
+            "column_family": "foo",
+            "column": "bar",
+            "value": "baz1",
+            "timestamp_iso": "1970-01-01T01:00:00",
+            "timestamp_epoch": 32400,
+            "timestamp_epoch_utc": null
+          }
+        ]
+      }
+    ]
+
+<a id="jc.parsers.cbt.parse"></a>
+
+### parse
+
+```python
+def parse(data: str,
+          raw: bool = False,
+          quiet: bool = False) -> List[JSONDictType]
+```
+
+Main text parsing function
+
+Parameters:
+
+    data:        (string)  text data to parse
+    raw:         (boolean) unprocessed output if True
+    quiet:       (boolean) suppress warning messages if True
+
+Returns:
+
+    List of Dictionaries. Raw or processed structured data.
+
+### Parser Information
+Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
+
+Version 1.0 by Andreas Weiden (andreas.weiden@gmail.com)

docs/parsers/clf.md

@@ -0,0 +1,199 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+<a id="jc.parsers.clf"></a>
+
+# jc.parsers.clf
+
+jc - JSON Convert Common Log Format file parser
+
+This parser will handle the Common Log Format standard as specified at
+https://www.w3.org/Daemon/User/Config/Logging.html#common-logfile-format.
+
+Combined Log Format is also supported. (Referer and User Agent fields added)
+
+Extra fields may be present and will be enclosed in the `extra` field as
+a single string.
+
+If a log line cannot be parsed, an object with an `unparsable` field will
+be present with a value of the original line.
+
+The `epoch` calculated timestamp field is naive. (i.e. based on the
+local time of the system the parser is run on)
+
+The `epoch_utc` calculated timestamp field is timezone-aware and is
+only available if the timezone field is UTC.
+
+Usage (cli):
+
+    $ cat file.log | jc --clf
+
+Usage (module):
+
+    import jc
+    result = jc.parse('clf', common_log_file_output)
+
+Schema:
+
+Empty strings and `-` values are converted to `null`/`None`.
+
+    [
+      {
+        "host":                         string,
+        "ident":                        string,
+        "authuser":                     string,
+        "date":                         string,
+        "day":                          integer,
+        "month":                        string,
+        "year":                         integer,
+        "hour":                         integer,
+        "minute":                       integer,
+        "second":                       integer,
+        "tz":                           string,
+        "request":                      string,
+        "request_method":               string,
+        "request_url":                  string,
+        "request_version":              string,
+        "status":                       integer,
+        "bytes":                        integer,
+        "referer":                      string,
+        "user_agent":                   string,
+        "extra":                        string,
+        "epoch":                        integer,  # [0]
+        "epoch_utc":                    integer,  # [1]
+        "unparsable":                   string    # [2]
+      }
+    ]
+
+    [0] naive timestamp
+    [1] timezone-aware timestamp. Only available if timezone field is UTC
+    [2] exists if the line was not able to be parsed
+
+Examples:
+
+    $ cat file.log | jc --clf -p
+    [
+      {
+        "host": "127.0.0.1",
+        "ident": "user-identifier",
+        "authuser": "frank",
+        "date": "10/Oct/2000:13:55:36 -0700",
+        "day": 10,
+        "month": "Oct",
+        "year": 2000,
+        "hour": 13,
+        "minute": 55,
+        "second": 36,
+        "tz": "-0700",
+        "request": "GET /apache_pb.gif HTTPS/1.0",
+        "status": 200,
+        "bytes": 2326,
+        "referer": null,
+        "user_agent": null,
+        "extra": null,
+        "request_method": "GET",
+        "request_url": "/apache_pb.gif",
+        "request_version": "HTTPS/1.0",
+        "epoch": 971211336,
+        "epoch_utc": null
+      },
+      {
+        "host": "1.1.1.2",
+        "ident": null,
+        "authuser": null,
+        "date": "11/Nov/2016:03:04:55 +0100",
+        "day": 11,
+        "month": "Nov",
+        "year": 2016,
+        "hour": 3,
+        "minute": 4,
+        "second": 55,
+        "tz": "+0100",
+        "request": "GET /",
+        "status": 200,
+        "bytes": 83,
+        "referer": null,
+        "user_agent": null,
+        "extra": "- 9221 1.1.1.1",
+        "request_method": "GET",
+        "request_url": "/",
+        "request_version": null,
+        "epoch": 1478862295,
+        "epoch_utc": null
+      },
+      ...
+    ]
+
+    $ cat file.log | jc --clf -p -r
+    [
+      {
+        "host": "127.0.0.1",
+        "ident": "user-identifier",
+        "authuser": "frank",
+        "date": "10/Oct/2000:13:55:36 -0700",
+        "day": "10",
+        "month": "Oct",
+        "year": "2000",
+        "hour": "13",
+        "minute": "55",
+        "second": "36",
+        "tz": "-0700",
+        "request": "GET /apache_pb.gif HTTPS/1.0",
+        "status": "200",
+        "bytes": "2326",
+        "referer": null,
+        "user_agent": null,
+        "extra": "",
+        "request_method": "GET",
+        "request_url": "/apache_pb.gif",
+        "request_version": "HTTPS/1.0"
+      },
+      {
+        "host": "1.1.1.2",
+        "ident": "-",
+        "authuser": "-",
+        "date": "11/Nov/2016:03:04:55 +0100",
+        "day": "11",
+        "month": "Nov",
+        "year": "2016",
+        "hour": "03",
+        "minute": "04",
+        "second": "55",
+        "tz": "+0100",
+        "request": "GET /",
+        "status": "200",
+        "bytes": "83",
+        "referer": "-",
+        "user_agent": "-",
+        "extra": "- 9221 1.1.1.1",
+        "request_method": "GET",
+        "request_url": "/",
+        "request_version": null
+      },
+      ...
+    ]
+
+<a id="jc.parsers.clf.parse"></a>
+
+### parse
+
+```python
+def parse(data: str,
+          raw: bool = False,
+          quiet: bool = False) -> List[JSONDictType]
+```
+
+Main text parsing function
+
+Parameters:
+
+    data:        (string)  text data to parse
+    raw:         (boolean) unprocessed output if True
+    quiet:       (boolean) suppress warning messages if True
+
+Returns:
+
+    List of Dictionaries. Raw or processed structured data.
+
+### Parser Information
+Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
+
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/clf_s.md

@@ -0,0 +1,117 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+<a id="jc.parsers.clf_s"></a>
+
+# jc.parsers.clf\_s
+
+jc - JSON Convert Common Log Format file streaming parser
+
+> This streaming parser outputs JSON Lines (cli) or returns an Iterable of
+> Dictionaries (module)
+
+This parser will handle the Common Log Format standard as specified at
+https://www.w3.org/Daemon/User/Config/Logging.html#common-logfile-format.
+
+Combined Log Format is also supported. (Referer and User Agent fields added)
+
+Extra fields may be present and will be enclosed in the `extra` field as
+a single string.
+
+If a log line cannot be parsed, an object with an `unparsable` field will
+be present with a value of the original line.
+
+The `epoch` calculated timestamp field is naive. (i.e. based on the
+local time of the system the parser is run on)
+
+The `epoch_utc` calculated timestamp field is timezone-aware and is
+only available if the timezone field is UTC.
+
+Usage (cli):
+
+    $ cat file.log | jc --clf-s
+
+Usage (module):
+
+    import jc
+
+    result = jc.parse('clf_s', common_log_file_output.splitlines())
+    for item in result:
+        # do something
+
+Schema:
+
+    Empty strings and `-` values are converted to `null`/`None`.
+
+    {
+      "host":                         string,
+      "ident":                        string,
+      "authuser":                     string,
+      "date":                         string,
+      "day":                          integer,
+      "month":                        string,
+      "year":                         integer,
+      "hour":                         integer,
+      "minute":                       integer,
+      "second":                       integer,
+      "tz":                           string,
+      "request":                      string,
+      "request_method":               string,
+      "request_url":                  string,
+      "request_version":              string,
+      "status":                       integer,
+      "bytes":                        integer,
+      "referer":                      string,
+      "user_agent":                   string,
+      "extra":                        string,
+      "epoch":                        integer,  # [0]
+      "epoch_utc":                    integer,  # [1]
+      "unparsable":                   string    # [2]
+    }
+
+    [0] naive timestamp
+    [1] timezone-aware timestamp. Only available if timezone field is UTC
+    [2] exists if the line was not able to be parsed
+
+Examples:
+
+    $ cat file.log | jc --clf-s
+    {"host":"127.0.0.1","ident":"user-identifier","authuser":"frank","...}
+    {"host":"1.1.1.2","ident":null,"authuser":null,"date":"11/Nov/2016...}
+    ...
+
+    $ cat file.log | jc --clf-s -r
+    {"host":"127.0.0.1","ident":"user-identifier","authuser":"frank","...}
+    {"host":"1.1.1.2","ident":"-","authuser":"-","date":"11/Nov/2016:0...}
+    ...
+
+<a id="jc.parsers.clf_s.parse"></a>
+
+### parse
+
+```python
+@add_jc_meta
+def parse(data: Iterable[str],
+          raw: bool = False,
+          quiet: bool = False,
+          ignore_exceptions: bool = False) -> StreamingOutputType
+```
+
+Main text parsing generator function. Returns an iterable object.
+
+Parameters:
+
+    data:              (iterable)  line-based text data to parse
+                                   (e.g. sys.stdin or str.splitlines())
+
+    raw:               (boolean)   unprocessed output if True
+    quiet:             (boolean)   suppress warning messages if True
+    ignore_exceptions: (boolean)   ignore parsing exceptions if True
+
+
+Returns:
+
+    Iterable of Dictionaries
+
+### Parser Information
+Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
+
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/datetime_iso.md

@@ -6,7 +6,7 @@
 jc - JSON Convert ISO 8601 Datetime string parser
 
 This parser supports standard ISO 8601 strings that include both date and
-time. If no timezone or offset information is available in the sring, then
+time. If no timezone or offset information is available in the string, then
 UTC timezone is used.
 
 Usage (cli):

docs/parsers/dig.md

@@ -106,7 +106,7 @@ Schema:
     ]
 
     [0] naive timestamp if "when" field is parsable, else null
-    [1] timezone aware timestamp availabe for UTC, else null
+    [1] timezone aware timestamp available for UTC, else null
 
 Examples:
 

docs/parsers/findmnt.md

@@ -72,7 +72,6 @@ Examples:
       ...
     ]
 
-
     $ findmnt | jc --findmnt -p -r
     [
       {
@@ -115,4 +114,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/git_log.md

@@ -40,13 +40,13 @@ Schema:
     [
       {
         "commit":               string,
-        "author":               string,
-        "author_email":         string,
+        "author":               string/null,
+        "author_email":         string/null,
         "date":                 string,
         "epoch":                integer,  # [0]
         "epoch_utc":            integer,  # [1]
-        "commit_by":            string,
-        "commit_by_email":      string,
+        "commit_by":            string/null,
+        "commit_by_email":      string/null,
         "commit_by_date":       string,
         "message":              string,
         "stats" : {
@@ -61,7 +61,7 @@ Schema:
     ]
 
     [0] naive timestamp if "date" field is parsable, else null
-    [1] timezone aware timestamp availabe for UTC, else null
+    [1] timezone aware timestamp available for UTC, else null
 
 Examples:
 
@@ -172,4 +172,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
-Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/git_log_s.md

@@ -41,13 +41,13 @@ Schema:
 
     {
       "commit":               string,
-      "author":               string,
-      "author_email":         string,
+      "author":               string/null,
+      "author_email":         string/null,
       "date":                 string,
       "epoch":                integer,  # [0]
       "epoch_utc":            integer,  # [1]
-      "commit_by":            string,
-      "commit_by_email":      string,
+      "commit_by":            string/null,
+      "commit_by_email":      string/null,
       "commit_by_date":       string,
       "message":              string,
       "stats" : {
@@ -68,7 +68,7 @@ Schema:
     }
 
     [0] naive timestamp if "date" field is parsable, else null
-    [1] timezone aware timestamp availabe for UTC, else null
+    [1] timezone aware timestamp available for UTC, else null
 
 Examples:
 
@@ -108,4 +108,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
-Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/git_ls_remote.md

@@ -71,7 +71,7 @@ Examples:
 ```python
 def parse(data: str,
           raw: bool = False,
-          quiet: bool = False) -> List[JSONDictType]
+          quiet: bool = False) -> Union[JSONDictType, List[JSONDictType]]
 ```
 
 Main text parsing function

docs/parsers/ifconfig.md

@@ -3,7 +3,7 @@
 
 # jc.parsers.ifconfig
 
-jc - JSON Convert `foo` command output parser
+jc - JSON Convert `ifconfig` command output parser
 
 No `ifconfig` options are supported.
 
@@ -42,6 +42,7 @@ Schema:
         "ipv6_addr":                string,    # [0]
         "ipv6_mask":                integer,   # [0]
         "ipv6_scope":               string,    # [0]
+        "ipv6_scope_id":            string,    # [0]
         "ipv6_type":                string,    # [0]
         "rx_packets":               integer,
         "rx_bytes":                 integer,
@@ -87,10 +88,19 @@ Schema:
         "ipv6: [
           {
             "address":              string,
+            "scope_id":             string,
             "mask":                 integer,
             "scope":                string,
             "type":                 string
           }
+        ],
+        "lanes": [
+          {
+            "lane":                 integer,
+            "rx_power_mw":          float,
+            "rx_power_dbm":         float,
+            "tx_bias_ma":           float
+          }
         ]
       }
     ]
@@ -147,6 +157,7 @@ Examples:
         "ipv6": [
           {
             "address": "fe80::c1cb:715d:bc3e:b8a0",
+            "scope_id": null,
             "mask": 64,
             "scope": "0x20",
             "type": "link"
@@ -195,6 +206,7 @@ Examples:
         "ipv6": [
           {
             "address": "fe80::c1cb:715d:bc3e:b8a0",
+            "scope_id": null,
             "mask": "64",
             "scope": "0x20",
             "type": "link"
@@ -228,4 +240,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, aix, freebsd, darwin
 
-Version 2.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 2.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/ini.md

@@ -3,13 +3,15 @@
 
 # jc.parsers.ini
 
-jc - JSON Convert `INI` file parser
+jc - JSON Convert INI file parser
 
-Parses standard `INI` files and files containing simple key/value pairs.
+Parses standard INI files.
 
 - Delimiter can be `=` or `:`. Missing values are supported.
 - Comment prefix can be `#` or `;`. Comments must be on their own line.
 - If duplicate keys are found, only the last value will be used.
+- If any section names have the same name as a top-level key, the top-level
+  key will be overwritten by the section data.
 
 > Note: Values starting and ending with double or single quotation marks
 > will have the marks removed. If you would like to keep the quotation
@@ -27,45 +29,47 @@ Usage (module):
 
 Schema:
 
-ini or key/value document converted to a dictionary - see the configparser
+INI document converted to a dictionary - see the python configparser
 standard library documentation for more details.
 
     {
-      "key1":       string,
-      "key2":       string
+      "<key1>":               string,
+      "<key2>":               string,
+      "<section1>": {
+        "<key1>":             string,
+        "<key2>":             string
+      },
+      "<section2>": {
+        "<key1>":             string,
+        "<key2>":             string
+      }
     }
 
 Examples:
 
     $ cat example.ini
-    [DEFAULT]
-    ServerAliveInterval = 45
-    Compression = yes
-    CompressionLevel = 9
-    ForwardX11 = yes
+    foo = fiz
+    bar = buz
 
-    [bitbucket.org]
-    User = hg
+    [section1]
+    fruit = apple
+    color = blue
 
-    [topsecret.server.com]
-    Port = 50022
-    ForwardX11 = no
+    [section2]
+    fruit = pear
+    color = green
 
     $ cat example.ini | jc --ini -p
     {
-      "bitbucket.org": {
-        "ServerAliveInterval": "45",
-        "Compression": "yes",
-        "CompressionLevel": "9",
-        "ForwardX11": "yes",
-        "User": "hg"
+      "foo": "fiz",
+      "bar": "buz",
+      "section1": {
+        "fruit": "apple",
+        "color": "blue"
       },
-      "topsecret.server.com": {
-        "ServerAliveInterval": "45",
-        "Compression": "yes",
-        "CompressionLevel": "9",
-        "ForwardX11": "no",
-        "Port": "50022"
+      "section2": {
+        "fruit": "pear",
+        "color": "green"
       }
     }
 
@@ -87,9 +91,9 @@ Parameters:
 
 Returns:
 
-    Dictionary representing the ini file
+    Dictionary representing the INI file.
 
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
-Version 1.8 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 2.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/ini_dup.md

@@ -0,0 +1,121 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+<a id="jc.parsers.ini_dup"></a>
+
+# jc.parsers.ini\_dup
+
+jc - JSON Convert INI with duplicate key file parser
+
+Parses standard INI files and preserves duplicate values. All values are
+contained in lists/arrays.
+
+- Delimiter can be `=` or `:`. Missing values are supported.
+- Comment prefix can be `#` or `;`. Comments must be on their own line.
+- If any section names have the same name as a top-level key, the top-level
+  key will be overwritten by the section data.
+- If multi-line values are used, each line will be a separate item in the
+  value list. Blank lines in multi-line values are not supported.
+
+> Note: Values starting and ending with double or single quotation marks
+> will have the marks removed. If you would like to keep the quotation
+> marks, use the `-r` command-line argument or the `raw=True` argument in
+> `parse()`.
+
+Usage (cli):
+
+    $ cat foo.ini | jc --ini
+
+Usage (module):
+
+    import jc
+    result = jc.parse('ini', ini_file_output)
+
+Schema:
+
+INI document converted to a dictionary - see the python configparser
+standard library documentation for more details.
+
+    {
+      "<key1>": [
+                            string
+      ],
+      "<key2>": [
+                            string
+      ],
+      "<section1>": {
+        "<key1>": [
+                            string
+        ],
+        "<key2>": [
+                            string
+        ]
+      }
+    }
+
+Examples:
+
+    $ cat example.ini
+    foo = fiz
+    bar = buz
+
+    [section1]
+    fruit = apple
+    color = blue
+    color = red
+
+    [section2]
+    fruit = pear
+    fruit = peach
+    color = green
+
+    $ cat example.ini | jc --ini -p
+    {
+      "foo": [
+        "fiz"
+      ],
+      "bar": [
+        "buz"
+      ],
+      "section1": {
+        "fruit": [
+          "apple"
+        ],
+        "color": [
+          "blue",
+          "red"
+        ]
+      },
+      "section2": {
+        "fruit": [
+          "pear",
+          "peach"
+        ],
+        "color": [
+          "green"
+        ]
+      }
+    }
+
+<a id="jc.parsers.ini_dup.parse"></a>
+
+### parse
+
+```python
+def parse(data, raw=False, quiet=False)
+```
+
+Main text parsing function
+
+Parameters:
+
+    data:        (string)  text data to parse
+    raw:         (boolean) unprocessed output if True
+    quiet:       (boolean) suppress warning messages if True
+
+Returns:
+
+    Dictionary representing the INI file.
+
+### Parser Information
+Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
+
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/iostat_s.md

@@ -130,4 +130,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/iwconfig.md

@@ -0,0 +1,111 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+<a id="jc.parsers.iwconfig"></a>
+
+# jc.parsers.iwconfig
+
+jc - JSON Convert `iwconfig` command output parser
+
+No `iwconfig` options are supported.
+
+Usage (cli):
+
+    $ iwconfig | jc --iwconfig
+
+or
+
+    $ jc iwconfig
+
+Usage (module):
+
+    import jc
+    result = jc.parse('iwconfig', iwconfig_command_output)
+
+Schema:
+
+    [
+        {
+            "name":                         string,
+            "protocol":                     string,
+            "essid":                        string,
+            "mode":                         string,
+            "frequency":                    float,
+            "frequency_unit":               string,
+            "access_point":                 string,
+            "bit_rate":                     float,
+            "bit_rate_unit":                string,
+            "tx_power":                     integer,
+            "tx_power_unit":                string,
+            "retry_short_limit":            integer,
+            "rts_threshold":                boolean,
+            "fragment_threshold":           boolean,
+            "power_management":             boolean,
+            "link_quality":                 string,
+            "signal_level":                 integer,
+            "signal_level_unit":            string,
+            "rx_invalid_nwid":              integer,
+            "rx_invalid_crypt":             integer,
+            "rx_invalid_frag":              integer,
+            "tx_excessive_retries":         integer,
+            "invalid_misc":                 integer,
+            "missed_beacon":                integer
+        }
+    ]
+
+Examples:
+
+    $  iwconfig | jc --iwconfig -p
+    [
+      {
+        "name": "wlp5s0",
+        "protocol": "IEEE 802.11",
+        "essid": "BLABLABLA",
+        "mode": "Managed",
+        "frequency": 5.18,
+        "frequency_unit": "GHz",
+        "access_point": "E6:64:DA:16:51:BF",
+        "bit_rate": 6.0,
+        "bit_rate_unit": "Mb/s",
+        "tx_power": 30,
+        "tx_power_unit": "dBm",
+        "retry_short_limit": 7,
+        "rts_threshold": false,
+        "fragment_threshold": false,
+        "power_management": true,
+        "link_quality": "61/70",
+        "signal_level": -49,
+        "signal_level_unit": "dBm",
+        "rx_invalid_nwid": 0,
+        "rx_invalid_crypt": 0,
+        "rx_invalid_frag": 0,
+        "tx_excessive_retries": 0,
+        "invalid_misc": 2095,
+        "missed_beacon": 0
+      }
+    ]
+
+<a id="jc.parsers.iwconfig.parse"></a>
+
+### parse
+
+```python
+def parse(data: str,
+          raw: bool = False,
+          quiet: bool = False) -> List[JSONDictType]
+```
+
+Main text parsing function
+
+Parameters:
+
+    data:        (string)  text data to parse
+    raw:         (boolean) unprocessed output if True
+    quiet:       (boolean) suppress warning messages if True
+
+Returns:
+
+    List of Dictionaries. Raw or processed structured data.
+
+### Parser Information
+Compatibility:  linux
+
+Version 1.0 by Thomas Vincent (vrince@gmail.com)

docs/parsers/kv.md

@@ -3,7 +3,7 @@
 
 # jc.parsers.kv
 
-jc - JSON Convert `Key/Value` file parser
+jc - JSON Convert `Key/Value` file and string parser
 
 Supports files containing simple key/value pairs.
 
@@ -26,8 +26,8 @@ Usage (module):
 
 Schema:
 
-key/value document converted to a dictionary - see the configparser standard
-library documentation for more details.
+Key/Value document converted to a dictionary - see the python configparser
+standard library documentation for more details.
 
     {
       "key1":       string,
@@ -41,6 +41,7 @@ Examples:
     name = John Doe
     address=555 California Drive
     age: 34
+
     ; comments can include # or ;
     # delimiter can be = or :
     # quoted values have quotation marks stripped by default
@@ -65,8 +66,6 @@ def parse(data, raw=False, quiet=False)
 
 Main text parsing function
 
-    Note: this is just a wrapper for jc.parsers.ini
-
 Parameters:
 
     data:        (string)  text data to parse
@@ -75,9 +74,9 @@ Parameters:
 
 Returns:
 
-    Dictionary representing the key/value file
+    Dictionary representing a Key/Value pair document.
 
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
-Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 2.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/ls_s.md

@@ -99,4 +99,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, aix, freebsd
 
-Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/lsusb.md

@@ -102,6 +102,28 @@ Schema:
                     ]
                   }
                 },
+                "videocontrol_descriptors": [
+                  {
+                    "<item>": {
+                      "value":                string,
+                      "description":          string,
+                      "attributes": [
+                                              string
+                      ]
+                    }
+                  }
+                ],
+                "videostreaming_descriptors": [
+                  {
+                    "<item>": {
+                      "value":                string,
+                      "description":          string,
+                      "attributes": [
+                                              string
+                      ]
+                    }
+                  }
+                ],
                 "endpoint_descriptors": [
                   {
                     "<item>": {
@@ -290,4 +312,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/mount.md

@@ -25,7 +25,7 @@ Schema:
         "filesystem":       string,
         "mount_point":      string,
         "type":             string,
-        "access": [
+        "options": [
                             string
         ]
       }
@@ -39,7 +39,7 @@ Example:
         "filesystem": "sysfs",
         "mount_point": "/sys",
         "type": "sysfs",
-        "access": [
+        "options": [
           "rw",
           "nosuid",
           "nodev",
@@ -51,7 +51,7 @@ Example:
         "filesystem": "proc",
         "mount_point": "/proc",
         "type": "proc",
-        "access": [
+        "options": [
           "rw",
           "nosuid",
           "nodev",
@@ -63,7 +63,7 @@ Example:
         "filesystem": "udev",
         "mount_point": "/dev",
         "type": "devtmpfs",
-        "access": [
+        "options": [
           "rw",
           "nosuid",
           "relatime",
@@ -96,6 +96,6 @@ Returns:
     List of Dictionaries. Raw or processed structured data.
 
 ### Parser Information
-Compatibility:  linux, darwin, freebsd
+Compatibility:  linux, darwin, freebsd, aix
 
-Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.8 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/openvpn.md

@@ -0,0 +1,178 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+<a id="jc.parsers.openvpn"></a>
+
+# jc.parsers.openvpn
+
+jc - JSON Convert openvpn-status.log file parser
+
+The `*_epoch` calculated timestamp fields are naive. (i.e. based on
+the local time of the system the parser is run on)
+
+Usage (cli):
+
+    $ cat openvpn-status.log | jc --openvpn
+
+Usage (module):
+
+    import jc
+    result = jc.parse('openvpn', openvpn_status_log_file_output)
+
+Schema:
+
+    {
+      "clients": [
+        {
+          "common_name":                        string,
+          "real_address":                       string,
+          "real_address_prefix":                integer,  # [0]
+          "real_address_port":                  integer,  # [0]
+          "bytes_received":                     integer,
+          "bytes_sent":                         integer,
+          "connected_since":                    string,
+          "connected_since_epoch":              integer,
+          "updated":                            string,
+          "updated_epoch":                      integer,
+        }
+      ],
+      "routing_table": [
+        {
+          "virtual_address":                    string,
+          "virtual_address_prefix":             integer,  # [0]
+          "virtual_address_port":               integer,  # [0]
+          "common_name":                        string,
+          "real_address":                       string,
+          "real_address_prefix":                integer,  # [0]
+          "real_address_port":                  integer,  # [0]
+          "last_reference":                     string,
+          "last_reference_epoch":               integer,
+        }
+      ],
+      "global_stats": {
+        "max_bcast_mcast_queue_len":            integer
+      }
+    }
+
+    [0] null/None if not found
+
+Examples:
+
+    $ cat openvpn-status.log | jc --openvpn -p
+    {
+      "clients": [
+        {
+          "common_name": "foo@example.com",
+          "real_address": "10.10.10.10",
+          "bytes_received": 334948,
+          "bytes_sent": 1973012,
+          "connected_since": "Thu Jun 18 04:23:03 2015",
+          "updated": "Thu Jun 18 08:12:15 2015",
+          "real_address_prefix": null,
+          "real_address_port": 49502,
+          "connected_since_epoch": 1434626583,
+          "updated_epoch": 1434640335
+        },
+        {
+          "common_name": "foo@example.com",
+          "real_address": "10.10.10.10",
+          "bytes_received": 334948,
+          "bytes_sent": 1973012,
+          "connected_since": "Thu Jun 18 04:23:03 2015",
+          "updated": "Thu Jun 18 08:12:15 2015",
+          "real_address_prefix": null,
+          "real_address_port": 49503,
+          "connected_since_epoch": 1434626583,
+          "updated_epoch": 1434640335
+        }
+      ],
+      "routing_table": [
+        {
+          "virtual_address": "192.168.255.118",
+          "common_name": "baz@example.com",
+          "real_address": "10.10.10.10",
+          "last_reference": "Thu Jun 18 08:12:09 2015",
+          "virtual_address_prefix": null,
+          "virtual_address_port": null,
+          "real_address_prefix": null,
+          "real_address_port": 63414,
+          "last_reference_epoch": 1434640329
+        },
+        {
+          "virtual_address": "10.200.0.0",
+          "common_name": "baz@example.com",
+          "real_address": "10.10.10.10",
+          "last_reference": "Thu Jun 18 08:12:09 2015",
+          "virtual_address_prefix": 16,
+          "virtual_address_port": null,
+          "real_address_prefix": null,
+          "real_address_port": 63414,
+          "last_reference_epoch": 1434640329
+        }
+      ],
+      "global_stats": {
+        "max_bcast_mcast_queue_len": 0
+      }
+    }
+
+    $ cat openvpn-status.log | jc --openvpn -p -r
+    {
+      "clients": [
+        {
+          "common_name": "foo@example.com",
+          "real_address": "10.10.10.10:49502",
+          "bytes_received": "334948",
+          "bytes_sent": "1973012",
+          "connected_since": "Thu Jun 18 04:23:03 2015",
+          "updated": "Thu Jun 18 08:12:15 2015"
+        },
+        {
+          "common_name": "foo@example.com",
+          "real_address": "10.10.10.10:49503",
+          "bytes_received": "334948",
+          "bytes_sent": "1973012",
+          "connected_since": "Thu Jun 18 04:23:03 2015",
+          "updated": "Thu Jun 18 08:12:15 2015"
+        }
+      ],
+      "routing_table": [
+        {
+          "virtual_address": "192.168.255.118",
+          "common_name": "baz@example.com",
+          "real_address": "10.10.10.10:63414",
+          "last_reference": "Thu Jun 18 08:12:09 2015"
+        },
+        {
+          "virtual_address": "10.200.0.0/16",
+          "common_name": "baz@example.com",
+          "real_address": "10.10.10.10:63414",
+          "last_reference": "Thu Jun 18 08:12:09 2015"
+        }
+      ],
+      "global_stats": {
+        "max_bcast_mcast_queue_len": "0"
+      }
+    }
+
+<a id="jc.parsers.openvpn.parse"></a>
+
+### parse
+
+```python
+def parse(data: str, raw: bool = False, quiet: bool = False) -> JSONDictType
+```
+
+Main text parsing function
+
+Parameters:
+
+    data:        (string)  text data to parse
+    raw:         (boolean) unprocessed output if True
+    quiet:       (boolean) suppress warning messages if True
+
+Returns:
+
+    Dictionary. Raw or processed structured data.
+
+### Parser Information
+Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
+
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/os_prober.md

@@ -21,12 +21,15 @@ Usage (module):
 Schema:
 
     {
-      'partition':              string,
-      'name':                   string,
-      'short_name':             string,
-      'type':                   string
+      "partition":              string,
+      "efi_bootmgr":            string,  # [0]
+      "name":                   string,
+      "short_name":             string,
+      "type":                   string
     }
 
+    [0] only exists if an EFI boot manager is detected
+
 Examples:
 
     $ os-prober | jc --os-prober -p
@@ -60,4 +63,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/pgpass.md

@@ -0,0 +1,75 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+<a id="jc.parsers.pgpass"></a>
+
+# jc.parsers.pgpass
+
+jc - JSON Convert PostgreSQL password file parser
+
+Usage (cli):
+
+    $ cat /var/lib/postgresql/.pgpass | jc --pgpass
+
+Usage (module):
+
+    import jc
+    result = jc.parse('pgpass', postgres_password_file)
+
+Schema:
+
+    [
+      {
+        "hostname":               string,
+        "port":                   string,
+        "database":               string,
+        "username":               string,
+        "password":               string
+      }
+    ]
+
+Examples:
+
+    $ cat /var/lib/postgresql/.pgpass | jc --pgpass -p
+    [
+      {
+        "hostname": "dbserver",
+        "port": "*",
+        "database": "db1",
+        "username": "dbuser",
+        "password": "pwd123"
+      },
+      {
+        "hostname": "dbserver2",
+        "port": "8888",
+        "database": "inventory",
+        "username": "joe:user",
+        "password": "abc123"
+      },
+      ...
+    ]
+
+<a id="jc.parsers.pgpass.parse"></a>
+
+### parse
+
+```python
+def parse(data: str,
+          raw: bool = False,
+          quiet: bool = False) -> List[JSONDictType]
+```
+
+Main text parsing function
+
+Parameters:
+
+    data:        (string)  text data to parse
+    raw:         (boolean) unprocessed output if True
+    quiet:       (boolean) suppress warning messages if True
+
+Returns:
+
+    List of Dictionaries. Raw or processed structured data.
+
+### Parser Information
+Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
+
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/ping_s.md

@@ -106,4 +106,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, freebsd
 
-Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/plist.md

@@ -5,7 +5,7 @@
 
 jc - JSON Convert PLIST file parser
 
-Converts binary and XML PLIST files.
+Converts binary, XML, and NeXTSTEP PLIST files.
 
 Binary values are converted into an ASCII hex representation.
 
@@ -69,9 +69,9 @@ Parameters:
 
 Returns:
 
-    List of Dictionaries. Raw or processed structured data.
+    Dictionary. Raw or processed structured data.
 
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/proc.md

@@ -6,7 +6,7 @@
 jc - JSON Convert Proc file output parser
 
 This parser automatically identifies the Proc file and calls the
-corresponding parser to peform the parsing.
+corresponding parser to perform the parsing.
 
 Magic syntax for converting `/proc` files is also supported by running
 `jc /proc/<path to file>`. Any `jc` options must be specified before the
@@ -139,4 +139,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/rsync_s.md

@@ -114,4 +114,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, freebsd
 
-Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/sshd_conf.md

@@ -161,7 +161,7 @@ Schema:
 
 Examples:
 
-    $ sshd -T | jc --sshd_conf -p
+    $ sshd -T | jc --sshd-conf -p
     {
       "acceptenv": [
         "LANG",
@@ -376,7 +376,7 @@ Examples:
       "subsystem_command": "/usr/lib/openssh/sftp-server"
     }
 
-    $ sshd -T | jc --sshd_conf -p -r
+    $ sshd -T | jc --sshd-conf -p -r
     {
       "acceptenv": [
         "LANG",

docs/parsers/stat_s.md

@@ -107,4 +107,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, freebsd
 
-Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.3 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/syslog.md

@@ -53,7 +53,7 @@ Blank values converted to `null`/`None`.
     ]
 
     [0] naive timestamp if "timestamp" field is parsable, else null
-    [1] timezone aware timestamp availabe for UTC, else null
+    [1] timezone aware timestamp available for UTC, else null
     [2] this field exists if the syslog line is not parsable. The value
         is the original syslog line.
 

docs/parsers/syslog_s.md

@@ -64,7 +64,7 @@ Blank values converted to `null`/`None`.
     }
 
     [0] naive timestamp if "timestamp" field is parsable, else null
-    [1] timezone aware timestamp availabe for UTC, else null
+    [1] timezone aware timestamp available for UTC, else null
     [2] this field exists if the syslog line is not parsable. The value
         is the original syslog line.
 

docs/parsers/toml.md

@@ -0,0 +1,81 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+<a id="jc.parsers.toml"></a>
+
+# jc.parsers.toml
+
+jc - JSON Convert TOML file parser
+
+Usage (cli):
+
+    $ cat file.toml | jc --toml
+
+Usage (module):
+
+    import jc
+    result = jc.parse('toml', toml_file_output)
+
+Schema:
+
+TOML Document converted to a Dictionary.
+See https://toml.io/en/ for details.
+
+    {
+      "key1":     string/int/float/boolean/null/array/object,
+      "key2":     string/int/float/boolean/null/array/object
+    }
+
+Examples:
+
+    $ cat file.toml
+    title = "TOML Example"
+
+    [owner]
+    name = "Tom Preston-Werner"
+    dob = 1979-05-27T07:32:00-08:00
+
+    [database]
+    enabled = true
+    ports = [ 8000, 8001, 8002 ]
+
+    $ cat file.toml | jc --toml -p
+    {
+      "title": "TOML Example",
+      "owner": {
+        "name": "Tom Preston-Werner",
+        "dob": 296667120,
+        "dob_iso": "1979-05-27T07:32:00-08:00"
+      },
+      "database": {
+        "enabled": true,
+        "ports": [
+          8000,
+          8001,
+          8002
+        ]
+      }
+    }
+
+<a id="jc.parsers.toml.parse"></a>
+
+### parse
+
+```python
+def parse(data: str, raw: bool = False, quiet: bool = False) -> JSONDictType
+```
+
+Main text parsing function
+
+Parameters:
+
+    data:        (string)  text data to parse
+    raw:         (boolean) unprocessed output if True
+    quiet:       (boolean) suppress warning messages if True
+
+Returns:
+
+    Dictionary. Raw or processed structured data.
+
+### Parser Information
+Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
+
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/vmstat_s.md

@@ -123,4 +123,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/x509_cert.md

@@ -32,6 +32,7 @@ Schema:
         "tbs_certificate": {
           "version":                      string,
           "serial_number":                string,  # [0]
+          "serial_number_str":            string,
           "signature": {
             "algorithm":                  string,
             "parameters":                 string/null,
@@ -43,7 +44,9 @@ Schema:
             "organization_name":          array/string,
             "organizational_unit_name":   array/string,
             "common_name":                string,
-            "email_address":              string
+            "email_address":              string,
+            "serial_number":              string,   # [0]
+            "serial_number_str":          string
           },
           "validity": {
             "not_before":                 integer,  # [1]
@@ -58,7 +61,9 @@ Schema:
             "organization_name":          array/string,
             "organizational_unit_name":   array/string,
             "common_name":                string,
-            "email_address":              string
+            "email_address":              string,
+            "serial_number":              string,   # [0]
+            "serial_number_str":          string
           },
           "subject_public_key_info": {
             "algorithm": {
@@ -428,4 +433,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/yaml.md

@@ -18,8 +18,8 @@ Usage (module):
 
 Schema:
 
-    YAML Document converted to a Dictionary
-    See https://pypi.org/project/ruamel.yaml for details
+YAML Document converted to a Dictionary.
+See https://pypi.org/project/ruamel.yaml for details.
 
     [
       {
@@ -30,7 +30,7 @@ Schema:
 
 Examples:
 
-    $ cat istio-mtls-permissive.yaml
+    $ cat file.yaml
     apiVersion: "authentication.istio.io/v1alpha1"
     kind: "Policy"
     metadata:
@@ -51,7 +51,7 @@ Examples:
         tls:
           mode: ISTIO_MUTUAL
 
-    $ cat istio-mtls-permissive.yaml | jc --yaml -p
+    $ cat file.yaml | jc --yaml -p
     [
       {
         "apiVersion": "authentication.istio.io/v1alpha1",

docs/parsers/zipinfo.md

@@ -99,4 +99,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin
 
-Version 1.1 by Matt J (https://github.com/listuser)
+Version 1.2 by Matt J (https://github.com/listuser)

docs/streaming.md

@@ -51,7 +51,7 @@ Add `_jc_meta` object to output line if `ignore_exceptions=True`
 ### stream\_error
 
 ```python
-def stream_error(e: BaseException, line: str) -> Dict[str, MetadataType]
+def stream_error(e: BaseException, line: str) -> JSONDictType
 ```
 
 Return an error `_jc_meta` field.

docs/utils.md

@@ -91,7 +91,7 @@ Parameters:
                   the parser. compatible options:
                   linux, darwin, cygwin, win32, aix, freebsd
 
-    quiet:        (bool) supress compatibility message if True
+    quiet:        (bool) suppress compatibility message if True
 
 Returns:
 
@@ -233,3 +233,6 @@ Returns a timestamp object with the following attributes:
     utc  (int | None):  aware timestamp only if UTC timezone
         detected in datetime string. None if conversion fails.
 
+    iso (str | None):  ISO string - timezone information is output
+        only if UTC timezone is detected in the datetime string.
+

jc/cli.py

@@ -9,13 +9,13 @@ from datetime import datetime, timezone
 import textwrap
 import shlex
 import subprocess
-from typing import List, Union, Optional, TextIO
+from typing import List, Dict, Union, Optional, TextIO
 from types import ModuleType
 from .lib import (
     __version__, parser_info, all_parser_info, parsers, _get_parser, _parser_is_streaming,
     parser_mod_list, standard_parser_mod_list, plugin_parser_mod_list, streaming_parser_mod_list
 )
-from .jc_types import JSONDictType, AboutJCType, MetadataType, CustomColorType, ParserInfoType
+from .jc_types import JSONDictType, CustomColorType, ParserInfoType
 from . import utils
 from .cli_data import (
     long_options_map, new_pygments_colors, old_pygments_colors, helptext_preamble_string,
@@ -48,7 +48,7 @@ class info():
     author: str = 'Kelly Brazil'
     author_email: str = 'kellyjonbrazil@gmail.com'
     website: str = 'https://github.com/kellyjonbrazil/jc'
-    copyright: str = '© 2019-2022 Kelly Brazil'
+    copyright: str = '© 2019-2023 Kelly Brazil'
     license: str = 'MIT License'
 
 
@@ -64,17 +64,17 @@ if PYGMENTS_INSTALLED:
 class JcCli():
     __slots__ = (
         'data_in', 'data_out', 'options', 'args', 'parser_module', 'parser_name', 'indent', 'pad',
-        'custom_colors', 'show_hidden', 'ascii_only', 'json_separators', 'json_indent',
-        'run_timestamp', 'about', 'debug', 'verbose_debug', 'force_color', 'mono', 'help_me',
-        'pretty', 'quiet', 'ignore_exceptions', 'raw', 'meta_out', 'unbuffer', 'version_info',
-        'yaml_output', 'bash_comp', 'zsh_comp', 'magic_found_parser', 'magic_options',
-        'magic_run_command', 'magic_run_command_str', 'magic_stdout', 'magic_stderr',
-        'magic_returncode'
+        'custom_colors', 'show_hidden', 'show_categories', 'ascii_only', 'json_separators',
+        'json_indent', 'run_timestamp', 'about', 'debug', 'verbose_debug', 'force_color', 'mono',
+        'help_me', 'pretty', 'quiet', 'ignore_exceptions', 'raw', 'meta_out', 'unbuffer',
+        'version_info', 'yaml_output', 'bash_comp', 'zsh_comp', 'magic_found_parser',
+        'magic_options', 'magic_run_command', 'magic_run_command_str', 'magic_stdout',
+        'magic_stderr', 'magic_returncode'
     )
 
     def __init__(self) -> None:
         self.data_in: Optional[Union[str, bytes, TextIO]] = None
-        self.data_out: Optional[Union[List[JSONDictType], JSONDictType, AboutJCType]] = None
+        self.data_out: Optional[Union[List[JSONDictType], JSONDictType]] = None
         self.options: List[str] = []
         self.args: List[str] = []
         self.parser_module: Optional[ModuleType] = None
@@ -83,6 +83,7 @@ class JcCli():
         self.pad: int = 0
         self.custom_colors: CustomColorType = {}
         self.show_hidden: bool = False
+        self.show_categories: bool = False
         self.ascii_only: bool = False
         self.json_separators: Optional[tuple[str, str]] = (',', ':')
         self.json_indent: Optional[int] = None
@@ -198,6 +199,41 @@ class JcCli():
 
         return ptext
 
+    def parser_categories_text(self) -> str:
+        """Return lists of parsers by category"""
+        category_text: str = ''
+        padding_char: str = ' '
+        all_parsers = all_parser_info(show_hidden=True, show_deprecated=False)
+        generic = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'generic' in x['tags']]
+        standard = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'standard' in x['tags']]
+        command = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'command' in x['tags']]
+        file_str_bin = [
+            {'arg': x['argument'], 'desc': x['description']} for x in all_parsers
+                if 'file' in x['tags'] or
+                'string' in x['tags'] or
+                'binary' in x['tags']
+        ]
+        streaming = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if x.get('streaming')]
+        categories: Dict = {
+            'Generic Parsers:': generic,
+            'Standard Spec Parsers:': standard,
+            'File/String/Binary Parsers:': file_str_bin,
+            'Streaming Parsers:': streaming,
+            'Command Parsers:': command
+        }
+
+        for cat, cat_objs in categories.items():
+            category_text += f'{cat}  ({len(cat_objs)})\n'
+            for p in cat_objs:
+                parser_arg: str = p.get('arg', 'UNKNOWN')
+                parser_desc: str = p.get('desc', 'No description available.')
+                padding: int = self.pad - len(parser_arg)
+                padding_text: str = padding_char * padding
+                category_text += f'{parser_arg}{padding_text}{parser_desc}\n'
+            category_text += '\n'
+
+        return category_text[:-1]
+
     def options_text(self) -> str:
         """Return the argument and description information from each option"""
         otext: str = ''
@@ -214,7 +250,7 @@ class JcCli():
         return otext
 
     @staticmethod
-    def about_jc() -> AboutJCType:
+    def about_jc() -> JSONDictType:
         """Return jc info and the contents of each parser.info as a dictionary"""
         return {
             'name': 'jc',
@@ -236,8 +272,6 @@ class JcCli():
 
     def helptext(self) -> str:
         """Return the help text with the list of parsers"""
-        self.indent = 4
-        self.pad = 20
         parsers_string: str = self.parsers_text()
         options_string: str = self.options_text()
         helptext_string: str = f'{helptext_preamble_string}{parsers_string}\nOptions:\n{options_string}\n{helptext_end_string}'
@@ -248,6 +282,13 @@ class JcCli():
         Pages the parser documentation if a parser is found in the arguments,
         otherwise the general help text is printed.
         """
+        self.indent = 4
+        self.pad = 22
+
+        if self.show_categories:
+            utils._safe_print(self.parser_categories_text())
+            return
+
         for arg in self.args:
             parser_name: str = self.parser_shortname(arg)
 
@@ -598,7 +639,7 @@ class JcCli():
         even if there are no results.
         """
         if self.run_timestamp:
-            meta_obj: MetadataType = {
+            meta_obj: JSONDictType = {
                 'parser': self.parser_name,
                 'timestamp': self.run_timestamp.timestamp()
             }
@@ -609,9 +650,9 @@ class JcCli():
 
             if isinstance(self.data_out, dict):
                 if '_jc_meta' not in self.data_out:
-                    self.data_out['_jc_meta'] = {}  # type: ignore
+                    self.data_out['_jc_meta'] = {}
 
-                self.data_out['_jc_meta'].update(meta_obj)  # type: ignore
+                self.data_out['_jc_meta'].update(meta_obj)
 
             elif isinstance(self.data_out, list):
                 if not self.data_out:
@@ -622,7 +663,7 @@ class JcCli():
                         if '_jc_meta' not in item:
                             item['_jc_meta'] = {}
 
-                        item['_jc_meta'].update(meta_obj)  # type: ignore
+                        item['_jc_meta'].update(meta_obj)
 
             else:
                 utils.error_message(['Parser returned an unsupported object type.'])
@@ -655,6 +696,7 @@ class JcCli():
         self.force_color = 'C' in self.options
         self.help_me = 'h' in self.options
         self.show_hidden = self.options.count('h') > 1   # verbose help
+        self.show_categories = self.options.count('h') > 2
         self.pretty = 'p' in self.options
         self.quiet = 'q' in self.options
         self.ignore_exceptions = self.options.count('q') > 1

jc/cli_data.py

@@ -3,7 +3,7 @@ from typing import List, Dict
 
 long_options_map: Dict[str, List[str]] = {
     '--about': ['a', 'about jc'],
-    '--force-color': ['C', 'force color output even when using pipes (overrides -m)'],
+    '--force-color': ['C', 'force color output (overrides -m)'],
     '--debug': ['d', 'debug (double for verbose debug)'],
     '--help': ['h', 'help (--help --parser_name for parser documentation)'],
     '--monochrome': ['m', 'monochrome output'],
@@ -91,6 +91,7 @@ Examples:
     Parser Documentation:
         $ jc --help --dig
 
-    Show Hidden Parsers:
-        $ jc -hh
+    More Help:
+        $ jc -hh          # show hidden parsers
+        $ jc -hhh         # list parsers by category tags
 '''

jc/jc_types.py

@@ -1,11 +1,9 @@
 """jc - JSON Convert lib module"""
 
 import sys
-from datetime import datetime
-from typing import Dict, List, Tuple, Iterator, Optional, Union
+from typing import Any, Dict, List, Tuple, Iterator, Optional, Union
 
-JSONDictType = Dict[str, Union[str, int, float, bool, List, Dict, None]]
-MetadataType = Dict[str, Optional[Union[str, int, float, List[str], datetime]]]
+JSONDictType = Dict[str, Any]
 StreamingOutputType = Iterator[Union[JSONDictType, Tuple[BaseException, str]]]
 
 if sys.version_info >= (3, 8):
@@ -22,6 +20,7 @@ if sys.version_info >= (3, 8):
             "author_email": str,
             "compatible": List[str],
             "magic_commands": List[str],
+            "tags": List[str],
             "documentation": str,
             "streaming": bool,
             "plugin": bool,
@@ -45,9 +44,6 @@ else:
     TimeStampFormatType = Dict
 
 
-AboutJCType = Dict[str, Union[str, int, List[ParserInfoType]]]
-
-
 try:
     from pygments.token import (Name, Number, String, Keyword)
     CustomColorType = Dict[Union[Name.Tag, Number, String, Keyword], str]

jc/lib.py

@@ -9,7 +9,7 @@ from .jc_types import ParserInfoType, JSONDictType
 from jc import appdirs
 
 
-__version__ = '1.22.2'
+__version__ = '1.22.5'
 
 parsers: List[str] = [
     'acpi',
@@ -19,10 +19,13 @@ parsers: List[str] = [
     'asciitable',
     'asciitable-m',
     'blkid',
+    'cbt',
     'cef',
     'cef-s',
     'chage',
     'cksum',
+    'clf',
+    'clf-s',
     'crontab',
     'crontab-u',
     'csv',
@@ -56,12 +59,14 @@ parsers: List[str] = [
     'id',
     'ifconfig',
     'ini',
+    'ini-dup',
     'iostat',
     'iostat-s',
     'ip-address',
     'iptables',
     'iso-datetime',
     'iw-scan',
+    'iwconfig',
     'jar-manifest',
     'jobs',
     'jwt',
@@ -82,9 +87,11 @@ parsers: List[str] = [
     'netstat',
     'nmcli',
     'ntpq',
+    'openvpn',
     'os-prober',
     'passwd',
     'pci-ids',
+    'pgpass',
     'pidstat',
     'pidstat-s',
     'ping',
@@ -168,6 +175,7 @@ parsers: List[str] = [
     'time',
     'timedatectl',
     'timestamp',
+    'toml',
     'top',
     'top-s',
     'tracepath',

jc/parsers/acpi.py

@@ -233,6 +233,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['acpi']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/airport.py

@@ -86,6 +86,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['darwin']
     magic_commands = ['airport -I']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/airport_s.py

@@ -115,6 +115,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['darwin']
     magic_commands = ['airport -s']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/arp.py

@@ -119,12 +119,13 @@ import jc.parsers.universal
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.11'
+    version = '1.12'
     description = '`arp` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'aix', 'freebsd', 'darwin']
     magic_commands = ['arp']
+    tags = ['command']
 
 
 __version__ = info.version
@@ -221,14 +222,26 @@ def parse(
         else:
             for line in cleandata:
                 splitline = line.split()
-                if '<incomplete>' not in splitline:
+
+                # Ignore AIX bucket information
+                if 'bucket:' in splitline[0]:
+                    continue
+                elif 'There' in splitline[0] and 'are' in splitline[1]:
+                    continue
+
+                # AIX uses (incomplete)
+                elif '<incomplete>' not in splitline and '(incomplete)' not in splitline:
                     output_line = {
                         'name': splitline[0],
                         'address': splitline[1].lstrip('(').rstrip(')'),
                         'hwtype': splitline[4].lstrip('[').rstrip(']'),
                         'hwaddress': splitline[3],
-                        'iface': splitline[6],
                     }
+                    # Handle permanence and ignore interface in AIX
+                    if 'permanent' in splitline:
+                        output_line['permanent'] = True
+                    elif 'in' not in splitline[6]: # AIX doesn't show interface
+                        output_line['iface'] = splitline[6]
 
                 else:
                     output_line = {
@@ -236,8 +249,10 @@ def parse(
                         'address': splitline[1].lstrip('(').rstrip(')'),
                         'hwtype': None,
                         'hwaddress': None,
-                        'iface': splitline[5],
                     }
+                    # AIX doesn't show interface
+                    if len(splitline) >= 5:
+                        output_line['iface'] = splitline[5]
 
                 raw_output.append(output_line)
 

jc/parsers/asciitable.py

@@ -130,6 +130,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['generic', 'string']
 
 
 __version__ = info.version

jc/parsers/asciitable_m.py

@@ -115,6 +115,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['generic', 'string']
 
 
 __version__ = info.version

jc/parsers/asn1crypto/algos.py

@@ -567,7 +567,7 @@ class DSASignature(Sequence):
     @classmethod
     def from_p1363(cls, data):
         """
-        Reads a signature from a byte string encoding accordint to IEEE P1363,
+        Reads a signature from a byte string encoding according to IEEE P1363,
         which is used by Microsoft's BCryptSignHash() function.
 
         :param data:

jc/parsers/asn1crypto/core.py

@@ -247,7 +247,7 @@ class Asn1Value(object):
 
         :param no_explicit:
             If explicit tagging info should be removed from this instance.
-            Used internally to allow contructing the underlying value that
+            Used internally to allow constructing the underlying value that
             has been wrapped in an explicit tag.
 
         :param tag_type:
@@ -697,7 +697,7 @@ class Castable(object):
         if other_class.tag != self.__class__.tag:
             raise TypeError(unwrap(
                 '''
-                Can not covert a value from %s object to %s object since they
+                Can not convert a value from %s object to %s object since they
                 use different tags: %d versus %d
                 ''',
                 type_name(other_class),
@@ -1349,7 +1349,7 @@ class Choice(Asn1Value):
 
 class Concat(object):
     """
-    A class that contains two or more encoded child values concatentated
+    A class that contains two or more encoded child values concatenated
     together. THIS IS NOT PART OF THE ASN.1 SPECIFICATION! This exists to handle
     the x509.TrustedCertificate() class for OpenSSL certificates containing
     extra information.
@@ -3757,7 +3757,7 @@ class Sequence(Asn1Value):
 
     def _make_value(self, field_name, field_spec, value_spec, field_params, value):
         """
-        Contructs an appropriate Asn1Value object for a field
+        Constructs an appropriate Asn1Value object for a field
 
         :param field_name:
             A unicode string of the field name
@@ -3766,7 +3766,7 @@ class Sequence(Asn1Value):
             An Asn1Value class that is the field spec
 
         :param value_spec:
-            An Asn1Value class that is the vaue spec
+            An Asn1Value class that is the value spec
 
         :param field_params:
             None or a dict of params for the field spec

jc/parsers/blkid.py

@@ -127,6 +127,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['blkid']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/cbt.py

@@ -0,0 +1,195 @@
+"""jc - JSON Convert `cbt` command output parser (Google Bigtable)
+
+Parses the human-, but not machine-, friendly output of the cbt command (for
+Google's Bigtable).
+
+No effort is made to convert the data types of the values in the cells.
+
+The `timestamp_epoch` calculated timestamp field is naive. (i.e. based on
+the local time of the system the parser is run on)
+
+The `timestamp_epoch_utc` calculated timestamp field is timezone-aware and
+is only available if the timestamp has a UTC timezone.
+
+The `timestamp_iso` calculated timestamp field will only include UTC
+timezone information if the timestamp has a UTC timezone.
+
+Raw output contains all cells for each column (including timestamps), while
+the normal output contains only the latest value for each column.
+
+Usage (cli):
+
+    $ cbt | jc --cbt
+
+or
+
+    $ jc cbt
+
+Usage (module):
+
+    import jc
+    result = jc.parse('cbt', cbt_command_output)
+
+Schema:
+
+    [
+      {
+        "key":                      string,
+        "cells": {
+          <string>: {                         # column family
+            <string>:               string    # column: value
+          }
+        }
+      }
+    ]
+
+Schema (raw):
+
+    [
+      {
+        "key":                      string,
+        "cells": [
+          {
+            "column_family":        string,
+            "column":               string,
+            "value":                string,
+            "timestamp_iso":        string,
+            "timestamp_epoch":      integer,
+            "timestamp_epoch_utc":  integer
+          }
+        ]
+      }
+    ]
+
+Examples:
+
+    $ cbt -project=$PROJECT -instance=$INSTANCE lookup $TABLE foo | jc --cbt -p
+    [
+      {
+        "key": "foo",
+        "cells": {
+          "foo": {
+            "bar": "baz"
+          }
+        }
+      }
+    ]
+
+    $ cbt -project=$PROJECT -instance=$INSTANCE lookup $TABLE foo | jc --cbt -p -r
+    [
+      {
+        "key": "foo",
+        "cells": [
+          {
+            "column_family": "foo",
+            "column": "bar",
+            "value": "baz1",
+            "timestamp_iso": "1970-01-01T01:00:00",
+            "timestamp_epoch": 32400,
+            "timestamp_epoch_utc": null
+          }
+        ]
+      }
+    ]
+"""
+from itertools import groupby
+from typing import List, Dict
+from jc.jc_types import JSONDictType
+import jc.utils
+
+
+class info():
+    """Provides parser metadata (version, author, etc.)"""
+    version = '1.0'
+    description = '`cbt` (Google Bigtable) command parser'
+    author = 'Andreas Weiden'
+    author_email = 'andreas.weiden@gmail.com'
+    compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    magic_commands = ['cbt']
+    tags = ['command']
+
+
+__version__ = info.version
+
+
+def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]:
+    """
+    Final processing to conform to the schema.
+
+    Parameters:
+
+        proc_data:   (List of Dictionaries) raw structured data to process
+
+    Returns:
+
+        List of Dictionaries. Structured to conform to the schema.
+    """
+    out_data = []
+    for row in proc_data:
+        cells: Dict = {}
+        key_func = lambda cell: (cell["column_family"], cell["column"])
+        all_cells = sorted(row["cells"], key=key_func)
+        for (column_family, column), group in groupby(all_cells, key=key_func):
+            group_list = sorted(group, key=lambda cell: cell["timestamp_iso"], reverse=True)
+            if column_family not in cells:
+                cells[column_family] = {}
+            cells[column_family][column] = group_list[0]["value"]
+        row["cells"] = cells
+        out_data.append(row)
+    return out_data
+
+
+def parse(
+        data: str,
+        raw: bool = False,
+        quiet: bool = False
+) -> List[JSONDictType]:
+    """
+    Main text parsing function
+
+    Parameters:
+
+        data:        (string)  text data to parse
+        raw:         (boolean) unprocessed output if True
+        quiet:       (boolean) suppress warning messages if True
+
+    Returns:
+
+        List of Dictionaries. Raw or processed structured data.
+    """
+    jc.utils.compatibility(__name__, info.compatible, quiet)
+    jc.utils.input_type_check(data)
+
+    raw_output: List[Dict] = []
+
+    if jc.utils.has_data(data):
+        for line in filter(None, data.split("-" * 40)):
+            key = None
+            cells = []
+            column_name = ""
+            timestamp = ""
+            value_next = False
+            for field in line.splitlines():
+                if not field.strip():
+                    continue
+                if field.startswith(" " * 4):
+                    value = field.strip(' "')
+                    if value_next:
+                        dt = jc.utils.timestamp(timestamp, format_hint=(1750, 1755))
+                        cells.append({
+                            "column_family": column_name.split(":", 1)[0],
+                            "column": column_name.split(":", 1)[1],
+                            "value": value,
+                            "timestamp_iso": dt.iso,
+                            "timestamp_epoch": dt.naive,
+                            "timestamp_epoch_utc": dt.utc
+                        })
+                elif field.startswith(" " * 2):
+                    column_name, timestamp = map(str.strip, field.split("@"))
+                    value_next = True
+                else:
+                    key = field
+            if key is not None:
+                raw_output.append({"key": key, "cells": cells})
+
+    return raw_output if raw else _process(raw_output)

jc/parsers/cef.py

@@ -129,6 +129,8 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using the pycef library at https://github.com/DavidJBianco/pycef/releases/tag/v1.11-2'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
+
 
 __version__ = info.version
 
@@ -185,7 +187,7 @@ def _pycef_parse(str_input):
 
         # If the input entry had any blanks in the required headers, that's wrong
         # and we should return.  Note we explicitly don't check the last item in the
-        # split list becuase the header ends in a '|' which means the last item
+        # split list because the header ends in a '|' which means the last item
         # will always be an empty string (it doesn't exist, but the delimiter does).
         if "" in spl[0:-1]:
             raise ParseError('Blank field(s) in CEF header. Is it valid CEF format?')

jc/parsers/cef_s.py

@@ -103,6 +103,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using the pycef library at https://github.com/DavidJBianco/pycef/releases/tag/v1.11-2'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
     streaming = True
 
 

jc/parsers/chage.py

@@ -63,6 +63,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['chage --list', 'chage -l']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/cksum.py

@@ -60,6 +60,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
     magic_commands = ['cksum', 'sum']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/clf.py

@@ -0,0 +1,299 @@
+"""jc - JSON Convert Common Log Format file parser
+
+This parser will handle the Common Log Format standard as specified at
+https://www.w3.org/Daemon/User/Config/Logging.html#common-logfile-format.
+
+Combined Log Format is also supported. (Referer and User Agent fields added)
+
+Extra fields may be present and will be enclosed in the `extra` field as
+a single string.
+
+If a log line cannot be parsed, an object with an `unparsable` field will
+be present with a value of the original line.
+
+The `epoch` calculated timestamp field is naive. (i.e. based on the
+local time of the system the parser is run on)
+
+The `epoch_utc` calculated timestamp field is timezone-aware and is
+only available if the timezone field is UTC.
+
+Usage (cli):
+
+    $ cat file.log | jc --clf
+
+Usage (module):
+
+    import jc
+    result = jc.parse('clf', common_log_file_output)
+
+Schema:
+
+Empty strings and `-` values are converted to `null`/`None`.
+
+    [
+      {
+        "host":                         string,
+        "ident":                        string,
+        "authuser":                     string,
+        "date":                         string,
+        "day":                          integer,
+        "month":                        string,
+        "year":                         integer,
+        "hour":                         integer,
+        "minute":                       integer,
+        "second":                       integer,
+        "tz":                           string,
+        "request":                      string,
+        "request_method":               string,
+        "request_url":                  string,
+        "request_version":              string,
+        "status":                       integer,
+        "bytes":                        integer,
+        "referer":                      string,
+        "user_agent":                   string,
+        "extra":                        string,
+        "epoch":                        integer,  # [0]
+        "epoch_utc":                    integer,  # [1]
+        "unparsable":                   string    # [2]
+      }
+    ]
+
+    [0] naive timestamp
+    [1] timezone-aware timestamp. Only available if timezone field is UTC
+    [2] exists if the line was not able to be parsed
+
+Examples:
+
+    $ cat file.log | jc --clf -p
+    [
+      {
+        "host": "127.0.0.1",
+        "ident": "user-identifier",
+        "authuser": "frank",
+        "date": "10/Oct/2000:13:55:36 -0700",
+        "day": 10,
+        "month": "Oct",
+        "year": 2000,
+        "hour": 13,
+        "minute": 55,
+        "second": 36,
+        "tz": "-0700",
+        "request": "GET /apache_pb.gif HTTPS/1.0",
+        "status": 200,
+        "bytes": 2326,
+        "referer": null,
+        "user_agent": null,
+        "extra": null,
+        "request_method": "GET",
+        "request_url": "/apache_pb.gif",
+        "request_version": "HTTPS/1.0",
+        "epoch": 971211336,
+        "epoch_utc": null
+      },
+      {
+        "host": "1.1.1.2",
+        "ident": null,
+        "authuser": null,
+        "date": "11/Nov/2016:03:04:55 +0100",
+        "day": 11,
+        "month": "Nov",
+        "year": 2016,
+        "hour": 3,
+        "minute": 4,
+        "second": 55,
+        "tz": "+0100",
+        "request": "GET /",
+        "status": 200,
+        "bytes": 83,
+        "referer": null,
+        "user_agent": null,
+        "extra": "- 9221 1.1.1.1",
+        "request_method": "GET",
+        "request_url": "/",
+        "request_version": null,
+        "epoch": 1478862295,
+        "epoch_utc": null
+      },
+      ...
+    ]
+
+    $ cat file.log | jc --clf -p -r
+    [
+      {
+        "host": "127.0.0.1",
+        "ident": "user-identifier",
+        "authuser": "frank",
+        "date": "10/Oct/2000:13:55:36 -0700",
+        "day": "10",
+        "month": "Oct",
+        "year": "2000",
+        "hour": "13",
+        "minute": "55",
+        "second": "36",
+        "tz": "-0700",
+        "request": "GET /apache_pb.gif HTTPS/1.0",
+        "status": "200",
+        "bytes": "2326",
+        "referer": null,
+        "user_agent": null,
+        "extra": "",
+        "request_method": "GET",
+        "request_url": "/apache_pb.gif",
+        "request_version": "HTTPS/1.0"
+      },
+      {
+        "host": "1.1.1.2",
+        "ident": "-",
+        "authuser": "-",
+        "date": "11/Nov/2016:03:04:55 +0100",
+        "day": "11",
+        "month": "Nov",
+        "year": "2016",
+        "hour": "03",
+        "minute": "04",
+        "second": "55",
+        "tz": "+0100",
+        "request": "GET /",
+        "status": "200",
+        "bytes": "83",
+        "referer": "-",
+        "user_agent": "-",
+        "extra": "- 9221 1.1.1.1",
+        "request_method": "GET",
+        "request_url": "/",
+        "request_version": null
+      },
+      ...
+    ]
+"""
+import re
+from typing import List, Dict
+from jc.jc_types import JSONDictType
+import jc.utils
+
+
+class info():
+    """Provides parser metadata (version, author, etc.)"""
+    version = '1.0'
+    description = 'Common and Combined Log Format file parser'
+    author = 'Kelly Brazil'
+    author_email = 'kellyjonbrazil@gmail.com'
+    compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
+
+
+__version__ = info.version
+
+
+def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]:
+    """
+    Final processing to conform to the schema.
+
+    Parameters:
+
+        proc_data:   (List of Dictionaries) raw structured data to process
+
+    Returns:
+
+        List of Dictionaries. Structured to conform to the schema.
+    """
+    int_list = {'day', 'year', 'hour', 'minute', 'second', 'status', 'bytes'}
+
+    for log in proc_data:
+        for key, val in log.items():
+
+            # integer conversions
+            if key in int_list:
+                log[key] = jc.utils.convert_to_int(val)
+
+            # convert `-` and blank values to None
+            if val == '-' or val == '':
+                log[key] = None
+
+        # add unix timestamps
+        if 'date' in log:
+            ts = jc.utils.timestamp(log['date'], format_hint=(1800,))
+            log['epoch'] = ts.naive
+            log['epoch_utc'] = ts.utc
+
+    return proc_data
+
+
+def parse(
+    data: str,
+    raw: bool = False,
+    quiet: bool = False
+) -> List[JSONDictType]:
+    """
+    Main text parsing function
+
+    Parameters:
+
+        data:        (string)  text data to parse
+        raw:         (boolean) unprocessed output if True
+        quiet:       (boolean) suppress warning messages if True
+
+    Returns:
+
+        List of Dictionaries. Raw or processed structured data.
+    """
+    jc.utils.compatibility(__name__, info.compatible, quiet)
+    jc.utils.input_type_check(data)
+
+    raw_output: List[Dict] = []
+    output_line: Dict = {}
+
+    clf_pattern = re.compile(r'''
+        ^(?P<host>-|\S+)\s
+        (?P<ident>-|\S+)\s
+        (?P<authuser>-|\S+)\s
+        \[
+        (?P<date>
+            (?P<day>\d+)/
+            (?P<month>\S\S\S)/
+            (?P<year>\d\d\d\d):
+            (?P<hour>\d\d):
+            (?P<minute>\d\d):
+            (?P<second>\d\d)\s
+            (?P<tz>\S+)
+        )
+        \]\s
+        \"(?P<request>.*?)\"\s
+        (?P<status>-|\d\d\d)\s
+        (?P<bytes>-|\d+)\s?
+        (?:\"(?P<referer>.*?)\"\s?)?
+        (?:\"(?P<user_agent>.*?)\"\s?)?
+        (?P<extra>.*)
+        ''', re.VERBOSE
+    )
+
+    request_pattern = re.compile(r'''
+        (?P<request_method>\S+)\s
+        (?P<request_url>.*?(?=\sHTTPS?/|$))\s?  # positive lookahead for HTTP(S)/ or end of string
+        (?P<request_version>HTTPS?/[\d\.]+)?
+    ''', re.VERBOSE
+    )
+
+    if jc.utils.has_data(data):
+
+        for line in filter(None, data.splitlines()):
+            output_line = {}
+            clf_match = re.match(clf_pattern, line)
+
+            if clf_match:
+                output_line = clf_match.groupdict()
+
+                if output_line.get('request', None):
+                    request_string = output_line['request']
+                    request_match = re.match(request_pattern, request_string)
+                    if request_match:
+                         output_line.update(request_match.groupdict())
+
+                raw_output.append(output_line)
+
+            else:
+                raw_output.append(
+                    {"unparsable": line}
+                )
+
+    return raw_output if raw else _process(raw_output)

jc/parsers/clf_s.py

@@ -0,0 +1,224 @@
+"""jc - JSON Convert Common Log Format file streaming parser
+
+> This streaming parser outputs JSON Lines (cli) or returns an Iterable of
+> Dictionaries (module)
+
+This parser will handle the Common Log Format standard as specified at
+https://www.w3.org/Daemon/User/Config/Logging.html#common-logfile-format.
+
+Combined Log Format is also supported. (Referer and User Agent fields added)
+
+Extra fields may be present and will be enclosed in the `extra` field as
+a single string.
+
+If a log line cannot be parsed, an object with an `unparsable` field will
+be present with a value of the original line.
+
+The `epoch` calculated timestamp field is naive. (i.e. based on the
+local time of the system the parser is run on)
+
+The `epoch_utc` calculated timestamp field is timezone-aware and is
+only available if the timezone field is UTC.
+
+Usage (cli):
+
+    $ cat file.log | jc --clf-s
+
+Usage (module):
+
+    import jc
+
+    result = jc.parse('clf_s', common_log_file_output.splitlines())
+    for item in result:
+        # do something
+
+Schema:
+
+    Empty strings and `-` values are converted to `null`/`None`.
+
+    {
+      "host":                         string,
+      "ident":                        string,
+      "authuser":                     string,
+      "date":                         string,
+      "day":                          integer,
+      "month":                        string,
+      "year":                         integer,
+      "hour":                         integer,
+      "minute":                       integer,
+      "second":                       integer,
+      "tz":                           string,
+      "request":                      string,
+      "request_method":               string,
+      "request_url":                  string,
+      "request_version":              string,
+      "status":                       integer,
+      "bytes":                        integer,
+      "referer":                      string,
+      "user_agent":                   string,
+      "extra":                        string,
+      "epoch":                        integer,  # [0]
+      "epoch_utc":                    integer,  # [1]
+      "unparsable":                   string    # [2]
+    }
+
+    [0] naive timestamp
+    [1] timezone-aware timestamp. Only available if timezone field is UTC
+    [2] exists if the line was not able to be parsed
+
+Examples:
+
+    $ cat file.log | jc --clf-s
+    {"host":"127.0.0.1","ident":"user-identifier","authuser":"frank","...}
+    {"host":"1.1.1.2","ident":null,"authuser":null,"date":"11/Nov/2016...}
+    ...
+
+    $ cat file.log | jc --clf-s -r
+    {"host":"127.0.0.1","ident":"user-identifier","authuser":"frank","...}
+    {"host":"1.1.1.2","ident":"-","authuser":"-","date":"11/Nov/2016:0...}
+    ...
+"""
+import re
+from typing import Dict, Iterable
+import jc.utils
+from jc.streaming import (
+    add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield
+)
+from jc.jc_types import JSONDictType, StreamingOutputType
+from jc.exceptions import ParseError
+
+
+class info():
+    """Provides parser metadata (version, author, etc.)"""
+    version = '1.0'
+    description = 'Common and Combined Log Format file streaming parser'
+    author = 'Kelly Brazil'
+    author_email = 'kellyjonbrazil@gmail.com'
+    compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
+    streaming = True
+
+
+__version__ = info.version
+
+
+def _process(proc_data: JSONDictType) -> JSONDictType:
+    """
+    Final processing to conform to the schema.
+
+    Parameters:
+
+        proc_data:   (Dictionary) raw structured data to process
+
+    Returns:
+
+        Dictionary. Structured data to conform to the schema.
+    """
+    int_list = {'day', 'year', 'hour', 'minute', 'second', 'status', 'bytes'}
+
+    for key, val in proc_data.items():
+
+        # integer conversions
+        if key in int_list:
+            proc_data[key] = jc.utils.convert_to_int(val)
+
+        # convert `-` and blank values to None
+        if val == '-' or val == '':
+            proc_data[key] = None
+
+    # add unix timestamps
+    if 'date' in proc_data:
+        ts = jc.utils.timestamp(proc_data['date'], format_hint=(1800,))
+        proc_data['epoch'] = ts.naive
+        proc_data['epoch_utc'] = ts.utc
+
+    return proc_data
+
+
+@add_jc_meta
+def parse(
+    data: Iterable[str],
+    raw: bool = False,
+    quiet: bool = False,
+    ignore_exceptions: bool = False
+) -> StreamingOutputType:
+    """
+    Main text parsing generator function. Returns an iterable object.
+
+    Parameters:
+
+        data:              (iterable)  line-based text data to parse
+                                       (e.g. sys.stdin or str.splitlines())
+
+        raw:               (boolean)   unprocessed output if True
+        quiet:             (boolean)   suppress warning messages if True
+        ignore_exceptions: (boolean)   ignore parsing exceptions if True
+
+
+    Returns:
+
+        Iterable of Dictionaries
+    """
+    jc.utils.compatibility(__name__, info.compatible, quiet)
+    streaming_input_type_check(data)
+
+    clf_pattern = re.compile(r'''
+        ^(?P<host>-|\S+)\s
+        (?P<ident>-|\S+)\s
+        (?P<authuser>-|\S+)\s
+        \[
+        (?P<date>
+            (?P<day>\d+)/
+            (?P<month>\S\S\S)/
+            (?P<year>\d\d\d\d):
+            (?P<hour>\d\d):
+            (?P<minute>\d\d):
+            (?P<second>\d\d)\s
+            (?P<tz>\S+)
+        )
+        \]\s
+        \"(?P<request>.*?)\"\s
+        (?P<status>-|\d\d\d)\s
+        (?P<bytes>-|\d+)\s?
+        (?:\"(?P<referer>.*?)\"\s?)?
+        (?:\"(?P<user_agent>.*?)\"\s?)?
+        (?P<extra>.*)
+        ''', re.VERBOSE
+    )
+
+    request_pattern = re.compile(r'''
+        (?P<request_method>\S+)\s
+        (?P<request_url>.*?(?=\sHTTPS?/|$))\s?  # positive lookahead for HTTP(S)/ or end of string
+        (?P<request_version>HTTPS?/[\d\.]+)?
+    ''', re.VERBOSE
+    )
+
+    for line in data:
+        try:
+            streaming_line_input_type_check(line)
+            output_line: Dict = {}
+
+            if not line.strip():
+                continue
+
+            clf_match = re.match(clf_pattern, line)
+
+            if clf_match:
+                output_line = clf_match.groupdict()
+
+                if output_line.get('request', None):
+                    request_string = output_line['request']
+                    request_match = re.match(request_pattern, request_string)
+                    if request_match:
+                         output_line.update(request_match.groupdict())
+
+            else:
+                output_line = {"unparsable": line.strip()}
+
+            if output_line:
+                yield output_line if raw else _process(output_line)
+            else:
+                raise ParseError('Not Common Log Format data')
+
+        except Exception as e:
+            yield raise_or_yield(ignore_exceptions, e, line)

jc/parsers/crontab.py

@@ -180,6 +180,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
     magic_commands = ['crontab']
+    tags = ['file', 'command']
 
 
 __version__ = info.version

jc/parsers/crontab_u.py

@@ -176,6 +176,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
+    tags = ['file', 'command']
 
 
 __version__ = info.version

jc/parsers/csv.py

@@ -86,6 +86,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using the python standard csv library'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
 
 
 __version__ = info.version

jc/parsers/csv_s.py

@@ -69,6 +69,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using the python standard csv library'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
     streaming = True
 
 

jc/parsers/date.py

@@ -84,6 +84,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'freebsd']
     magic_commands = ['date']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/datetime_iso.py

@@ -1,7 +1,7 @@
 """jc - JSON Convert ISO 8601 Datetime string parser
 
 This parser supports standard ISO 8601 strings that include both date and
-time. If no timezone or offset information is available in the sring, then
+time. If no timezone or offset information is available in the string, then
 UTC timezone is used.
 
 Usage (cli):
@@ -75,6 +75,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using the pyiso8601 library from https://github.com/micktwomey/pyiso8601/releases/tag/1.0.2'
     compatible = ['linux', 'aix', 'freebsd', 'darwin', 'win32', 'cygwin']
+    tags = ['standard', 'string']
 
 
 __version__ = info.version

jc/parsers/df.py

@@ -105,6 +105,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'freebsd']
     magic_commands = ['df']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/dig.py

@@ -101,7 +101,7 @@ Schema:
     ]
 
     [0] naive timestamp if "when" field is parsable, else null
-    [1] timezone aware timestamp availabe for UTC, else null
+    [1] timezone aware timestamp available for UTC, else null
 
 Examples:
 
@@ -328,6 +328,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'aix', 'freebsd', 'darwin', 'win32', 'cygwin']
     magic_commands = ['dig']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/dir.py

@@ -126,6 +126,7 @@ class info():
     author = 'Rasheed Elsaleh'
     author_email = 'rasheed@rebelliondefense.com'
     compatible = ['win32']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/dmidecode.py

@@ -131,6 +131,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['dmidecode']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/dpkg_l.py

@@ -138,6 +138,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['dpkg -l']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/du.py

@@ -98,6 +98,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
     magic_commands = ['du']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/email_address.py

@@ -47,6 +47,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'string']
 
 
 __version__ = info.version

jc/parsers/env.py

@@ -78,6 +78,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
     magic_commands = ['env', 'printenv']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/file.py

@@ -69,6 +69,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'aix', 'freebsd', 'darwin']
     magic_commands = ['file']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/findmnt.py

@@ -67,7 +67,6 @@ Examples:
       ...
     ]
 
-
     $ findmnt | jc --findmnt -p -r
     [
       {
@@ -94,12 +93,13 @@ import jc.utils
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.0'
+    version = '1.1'
     description = '`findmnt` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['findmnt']
+    tags = ['command']
 
 
 __version__ = info.version
@@ -123,7 +123,7 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]:
         kv_options = {}
 
         if 'options' in item:
-            opt_list = item['options'].split(',')  # type: ignore
+            opt_list = item['options'].split(',')
 
             for option in opt_list:
                 if '=' in option:
@@ -141,7 +141,7 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]:
     return proc_data
 
 
-def _replace(matchobj: re.Match) -> str:
+def _replace(matchobj):
     if matchobj:
         matchlen = len(matchobj.group(1))
         return ' ' * matchlen + '/'

jc/parsers/finger.py

@@ -98,6 +98,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'freebsd']
     magic_commands = ['finger']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/foo.py

@@ -48,6 +48,9 @@ class info():
 
     # compatible options: linux, darwin, cygwin, win32, aix, freebsd
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+
+    # tags options: generic, standard, file, string, binary, command
+    tags = ['command']
     magic_commands = ['foo']
 
 

jc/parsers/foo_s.py

@@ -58,6 +58,9 @@ class info():
 
     # compatible options: linux, darwin, cygwin, win32, aix, freebsd
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+
+    # tags options: generic, standard, file, string, binary, command
+    tags = ['command']
     streaming = True
 
 
@@ -117,6 +120,10 @@ def parse(
             streaming_line_input_type_check(line)
             output_line: Dict = {}
 
+            # skip blank lines
+            if not line.strip():
+                continue
+
             # parse the content here
             # check out helper functions in jc.utils
             # and jc.parsers.universal

jc/parsers/free.py

@@ -79,6 +79,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['free']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/fstab.py

@@ -90,6 +90,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/git_log.py

@@ -35,13 +35,13 @@ Schema:
     [
       {
         "commit":               string,
-        "author":               string,
-        "author_email":         string,
+        "author":               string/null,
+        "author_email":         string/null,
         "date":                 string,
         "epoch":                integer,  # [0]
         "epoch_utc":            integer,  # [1]
-        "commit_by":            string,
-        "commit_by_email":      string,
+        "commit_by":            string/null,
+        "commit_by_email":      string/null,
         "commit_by_date":       string,
         "message":              string,
         "stats" : {
@@ -56,7 +56,7 @@ Schema:
     ]
 
     [0] naive timestamp if "date" field is parsable, else null
-    [1] timezone aware timestamp availabe for UTC, else null
+    [1] timezone aware timestamp available for UTC, else null
 
 Examples:
 
@@ -153,12 +153,13 @@ changes_pattern = re.compile(r'\s(?P<files>\d+)\s+(files? changed),\s+(?P<insert
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.2'
+    version = '1.3'
     description = '`git log` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
     magic_commands = ['git log']
+    tags = ['command']
 
 
 __version__ = info.version
@@ -202,6 +203,28 @@ def _is_commit_hash(hash_string: str) -> bool:
 
     return False
 
+def _parse_name_email(line):
+    values = line.rsplit(maxsplit=1)
+    name = None
+    email = None
+
+    if len(values) == 2:
+        name = values[0]
+        if values[1].startswith('<') and values[1].endswith('>'):
+            email = values[1][1:-1]
+    else:
+        if values[0].lstrip().startswith('<') and values[0].endswith('>'):
+            email = values[0].lstrip()[1:-1]
+        else:
+            name = values[0]
+
+    if not name:
+        name = None
+    if not email:
+        email = None # covers '<>' case turning into null, not ''
+
+    return name, email
+
 
 def parse(
     data: str,
@@ -271,9 +294,7 @@ def parse(
                 continue
 
             if line.startswith('Author: '):
-                values = line_list[1].rsplit(maxsplit=1)
-                output_line['author'] = values[0]
-                output_line['author_email'] = values[1].strip('<').strip('>')
+                output_line['author'], output_line['author_email'] = _parse_name_email(line_list[1])
                 continue
 
             if line.startswith('Date: '):
@@ -289,9 +310,7 @@ def parse(
                 continue
 
             if line.startswith('Commit: '):
-                values = line_list[1].rsplit(maxsplit=1)
-                output_line['commit_by'] = values[0]
-                output_line['commit_by_email'] = values[1].strip('<').strip('>')
+                output_line['commit_by'], output_line['commit_by_email'] = _parse_name_email(line_list[1])
                 continue
 
             if line.startswith('    '):

jc/parsers/git_log_s.py

@@ -36,13 +36,13 @@ Schema:
 
     {
       "commit":               string,
-      "author":               string,
-      "author_email":         string,
+      "author":               string/null,
+      "author_email":         string/null,
       "date":                 string,
       "epoch":                integer,  # [0]
       "epoch_utc":            integer,  # [1]
-      "commit_by":            string,
-      "commit_by_email":      string,
+      "commit_by":            string/null,
+      "commit_by_email":      string/null,
       "commit_by_date":       string,
       "message":              string,
       "stats" : {
@@ -63,7 +63,7 @@ Schema:
     }
 
     [0] naive timestamp if "date" field is parsable, else null
-    [1] timezone aware timestamp availabe for UTC, else null
+    [1] timezone aware timestamp available for UTC, else null
 
 Examples:
 
@@ -75,6 +75,7 @@ Examples:
 import re
 from typing import List, Dict, Iterable, Union
 import jc.utils
+from jc.parsers.git_log import _parse_name_email
 from jc.streaming import (
     add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield
 )
@@ -87,11 +88,12 @@ changes_pattern = re.compile(r'\s(?P<files>\d+)\s+(files? changed),\s+(?P<insert
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.2'
+    version = '1.3'
     description = '`git log` command streaming parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['command']
     streaming = True
 
 
@@ -215,9 +217,7 @@ def parse(
                 continue
 
             if line.startswith('Author: '):
-                values = line_list[1].rsplit(maxsplit=1)
-                output_line['author'] = values[0]
-                output_line['author_email'] = values[1].strip('<').strip('>')
+                output_line['author'], output_line['author_email'] = _parse_name_email(line_list[1])
                 continue
 
             if line.startswith('Date: '):
@@ -233,9 +233,7 @@ def parse(
                 continue
 
             if line.startswith('Commit: '):
-                values = line_list[1].rsplit(maxsplit=1)
-                output_line['commit_by'] = values[0]
-                output_line['commit_by_email'] = values[1].strip('<').strip('>')
+                output_line['commit_by'], output_line['commit_by_email'] = _parse_name_email(line_list[1])
                 continue
 
             if line.startswith('    '):

jc/parsers/git_ls_remote.py

@@ -59,7 +59,7 @@ Examples:
       ...
     ]
 """
-from typing import List, Dict
+from typing import List, Union
 from jc.jc_types import JSONDictType
 import jc.utils
 
@@ -72,6 +72,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
     magic_commands = ['git ls-remote']
+    tags = ['command']
 
 
 __version__ = info.version
@@ -94,7 +95,7 @@ def _process(proc_data: List[JSONDictType]) -> JSONDictType:
     for item in proc_data:
         new_dict.update(
             {
-                item['reference']: item['commit']  # type: ignore
+                item['reference']: item['commit']
             }
         )
 
@@ -105,7 +106,7 @@ def parse(
     data: str,
     raw: bool = False,
     quiet: bool = False
-) -> List[JSONDictType]:
+) -> Union[JSONDictType, List[JSONDictType]]:
     """
     Main text parsing function
 
@@ -122,7 +123,8 @@ def parse(
     jc.utils.compatibility(__name__, info.compatible, quiet)
     jc.utils.input_type_check(data)
 
-    raw_output: List[Dict] = []
+    raw_output: List[JSONDictType] = []
+    output_line: JSONDictType = {}
 
     if jc.utils.has_data(data):
 
@@ -135,4 +137,4 @@ def parse(
             }
             raw_output.append(output_line)
 
-    return raw_output if raw else _process(raw_output)  # type: ignore
+    return raw_output if raw else _process(raw_output)

jc/parsers/gpg.py

@@ -126,6 +126,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['gpg --with-colons']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/group.py

@@ -114,6 +114,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/gshadow.py

@@ -82,6 +82,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'aix', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/hash.py

@@ -43,6 +43,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/hashsum.py

@@ -76,6 +76,7 @@ class info():
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
     magic_commands = ['md5sum', 'md5', 'shasum', 'sha1sum', 'sha224sum',
                       'sha256sum', 'sha384sum', 'sha512sum']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/hciconfig.py

@@ -323,6 +323,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['hciconfig']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/history.py

@@ -69,6 +69,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Optimizations by https://github.com/philippeitis'
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/hosts.py

@@ -79,6 +79,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/id.py

@@ -112,6 +112,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
     magic_commands = ['id']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/ifconfig.py

@@ -1,4 +1,4 @@
-"""jc - JSON Convert `foo` command output parser
+"""jc - JSON Convert `ifconfig` command output parser
 
 No `ifconfig` options are supported.
 
@@ -37,6 +37,7 @@ Schema:
         "ipv6_addr":                string,    # [0]
         "ipv6_mask":                integer,   # [0]
         "ipv6_scope":               string,    # [0]
+        "ipv6_scope_id":            string,    # [0]
         "ipv6_type":                string,    # [0]
         "rx_packets":               integer,
         "rx_bytes":                 integer,
@@ -82,10 +83,19 @@ Schema:
         "ipv6: [
           {
             "address":              string,
+            "scope_id":             string,
             "mask":                 integer,
             "scope":                string,
             "type":                 string
           }
+        ],
+        "lanes": [
+          {
+            "lane":                 integer,
+            "rx_power_mw":          float,
+            "rx_power_dbm":         float,
+            "tx_bias_ma":           float
+          }
         ]
       }
     ]
@@ -142,6 +152,7 @@ Examples:
         "ipv6": [
           {
             "address": "fe80::c1cb:715d:bc3e:b8a0",
+            "scope_id": null,
             "mask": 64,
             "scope": "0x20",
             "type": "link"
@@ -190,6 +201,7 @@ Examples:
         "ipv6": [
           {
             "address": "fe80::c1cb:715d:bc3e:b8a0",
+            "scope_id": null,
             "mask": "64",
             "scope": "0x20",
             "type": "link"
@@ -200,25 +212,26 @@ Examples:
 """
 import re
 from ipaddress import IPv4Network
-from typing import List, Dict
+from typing import List, Dict, Optional
 from jc.jc_types import JSONDictType
 import jc.utils
 
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '2.0'
+    version = '2.2'
     description = '`ifconfig` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'aix', 'freebsd', 'darwin']
     magic_commands = ['ifconfig']
+    tags = ['command']
 
 
 __version__ = info.version
 
 
-def _convert_cidr_to_quad(string):
+def _convert_cidr_to_quad(string: str) -> str:
     return str(IPv4Network('0.0.0.0/' + string).netmask)
 
 
@@ -237,8 +250,9 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]:
     int_list = {
         'flags', 'mtu', 'ipv6_mask', 'rx_packets', 'rx_bytes', 'rx_errors', 'rx_dropped',
         'rx_overruns', 'rx_frame', 'tx_packets', 'tx_bytes', 'tx_errors', 'tx_dropped',
-        'tx_overruns', 'tx_carrier', 'tx_collisions', 'metric', 'nd6_options'
+        'tx_overruns', 'tx_carrier', 'tx_collisions', 'metric', 'nd6_options', 'lane'
     }
+    float_list = {'rx_power_mw', 'rx_power_dbm', 'tx_bias_ma'}
 
     for entry in proc_data:
         for key in entry:
@@ -248,58 +262,67 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]:
         # convert OSX-style subnet mask to dotted quad
         if 'ipv4_mask' in entry:
             try:
-                if entry['ipv4_mask'].startswith('0x'):  # type: ignore
+                if entry['ipv4_mask'].startswith('0x'):
                     new_mask = entry['ipv4_mask']
-                    new_mask = new_mask.lstrip('0x')  # type: ignore
+                    new_mask = new_mask.lstrip('0x')
                     new_mask = '.'.join(str(int(i, 16)) for i in [new_mask[i:i + 2] for i in range(0, len(new_mask), 2)])
                     entry['ipv4_mask'] = new_mask
             except (ValueError, TypeError, AttributeError):
                 pass
 
             # for new-style freebsd output convert CIDR mask to dotted-quad to match other output
-            if entry['ipv4_mask'] and not '.' in entry['ipv4_mask']:  # type: ignore
+            if entry['ipv4_mask'] and not '.' in entry['ipv4_mask']:
                 entry['ipv4_mask'] = _convert_cidr_to_quad(entry['ipv4_mask'])
 
         # convert state value to an array
         if 'state' in entry:
             try:
-                new_state = entry['state'].split(',')  # type: ignore
+                new_state = entry['state'].split(',')
                 entry['state'] = new_state
             except (ValueError, TypeError, AttributeError):
                 pass
 
         # conversions for list of ipv4 addresses
         if 'ipv4' in entry:
-            for ip_address in entry['ipv4']:  # type: ignore
+            for ip_address in entry['ipv4']:
                 if 'mask' in ip_address:
                     try:
-                        if ip_address['mask'].startswith('0x'):  # type: ignore
-                            new_mask = ip_address['mask']  # type: ignore
+                        if ip_address['mask'].startswith('0x'):
+                            new_mask = ip_address['mask']
                             new_mask = new_mask.lstrip('0x')
                             new_mask = '.'.join(str(int(i, 16)) for i in [new_mask[i:i + 2] for i in range(0, len(new_mask), 2)])
-                            ip_address['mask'] = new_mask  # type: ignore
+                            ip_address['mask'] = new_mask
                     except (ValueError, TypeError, AttributeError):
                         pass
 
                     # for new-style freebsd output convert CIDR mask to dotted-quad to match other output
-                    if ip_address['mask'] and not '.' in ip_address['mask']:  # type: ignore
-                        ip_address['mask'] = _convert_cidr_to_quad(ip_address['mask'])  # type: ignore
+                    if ip_address['mask'] and not '.' in ip_address['mask']:
+                        ip_address['mask'] = _convert_cidr_to_quad(ip_address['mask'])
 
         # conversions for list of ipv6 addresses
         if 'ipv6' in entry:
-            for ip_address in entry['ipv6']:  # type: ignore
+            for ip_address in entry['ipv6']:
                 if 'mask' in ip_address:
-                    ip_address['mask'] = jc.utils.convert_to_int(ip_address['mask'])  # type: ignore
+                    ip_address['mask'] = jc.utils.convert_to_int(ip_address['mask'])
+
+        # conversions for list of lanes
+        if 'lanes' in entry:
+            for lane_item in entry['lanes']:
+                for key in lane_item:
+                    if key in int_list:
+                        lane_item[key] = jc.utils.convert_to_int(lane_item[key])
+                    if key in float_list:
+                        lane_item[key] = jc.utils.convert_to_float(lane_item[key])
 
         # final conversions
         if entry.get('media_flags', None):
-            entry['media_flags'] = entry['media_flags'].split(',')  # type: ignore
+            entry['media_flags'] = entry['media_flags'].split(',')
 
         if entry.get('nd6_flags', None):
-            entry['nd6_flags'] = entry['nd6_flags'].split(',')  # type: ignore
+            entry['nd6_flags'] = entry['nd6_flags'].split(',')
 
         if entry.get('options_flags', None):
-            entry['options_flags'] = entry['options_flags'].split(',')  # type: ignore
+            entry['options_flags'] = entry['options_flags'].split(',')
 
     return proc_data
 
@@ -371,6 +394,7 @@ def parse(
     interface_item: Dict = interface_obj.copy()
     ipv4_info: List = []
     ipv6_info: List = []
+    lane_info: List = []
 
     # Below regular expression patterns are based off of the work of:
     # https://github.com/KnightWhoSayNi/ifconfig-parser
@@ -525,9 +549,10 @@ def parse(
         ''', re.IGNORECASE | re.VERBOSE
     )
     re_freebsd_ipv6 = re.compile(r'''
-        \s?inet6\s(?P<address>.*)(?:\%\w+\d+)\s
-        prefixlen\s(?P<mask>\d+)(?:\s\w+)?\s
-        scopeid\s(?P<scope>\w+x\w+)
+        \s?inet6\s(?P<address>.*?)
+        (?:\%(?P<scope_id>\w+\d+))?\s
+        prefixlen\s(?P<mask>\d+).*?(?=scopeid|$)  # positive lookahead for scopeid or end of line
+        (?:scopeid\s(?P<scope>0x\w+))?
         ''', re.IGNORECASE | re.VERBOSE
     )
     re_freebsd_details = re.compile(r'''
@@ -581,6 +606,15 @@ def parse(
         ''', re.IGNORECASE | re.VERBOSE
     )
 
+    # this pattern is special since it is used to build the lane_info list
+    re_freebsd_lane = re.compile(r'''
+        lane\s(?P<lane>\d+):\s
+        RX\spower:\s(?P<rx_power_mw>\S+)\smW\s
+        \((?P<rx_power_dbm>\S+)\sdBm\)\s
+        TX\sbias:\s(?P<tx_bias_ma>\S+)
+        ''', re.IGNORECASE | re.VERBOSE
+    )
+
     re_linux = [
         re_linux_interface, re_linux_ipv4, re_linux_ipv6, re_linux_state, re_linux_rx, re_linux_tx,
         re_linux_bytes, re_linux_tx_stats
@@ -611,10 +645,13 @@ def parse(
                         interface_item['ipv4'] = ipv4_info
                     if ipv6_info:
                         interface_item['ipv6'] = ipv6_info
+                    if lane_info:
+                        interface_item['lanes'] = lane_info
                     raw_output.append(interface_item)
                     interface_item = interface_obj.copy()
                     ipv4_info = []
                     ipv6_info = []
+                    lane_info = []
 
                 interface_item.update(interface_match.groupdict())
                 continue
@@ -646,6 +683,7 @@ def parse(
                     'mask': 'ipv6_mask',
                     'broadcast': 'ipv6_bcast',
                     'scope': 'ipv6_scope',
+                    'scope_id': 'ipv6_scope_id',
                     'type': 'ipv6_type'
                 }
                 for k, v in ipv6_dict.copy().items():
@@ -666,6 +704,12 @@ def parse(
                 ipv6_info.append(ipv6_match.groupdict())
                 continue
 
+            # lane information lines
+            lane_match = re.search(re_freebsd_lane, line)
+            if lane_match:
+                lane_info.append(lane_match.groupdict())
+                continue
+
             # All other lines
             other_match = _bundle_match(re_linux + re_openbsd + re_freebsd, line)
             if other_match:
@@ -677,6 +721,8 @@ def parse(
                 interface_item['ipv4'] = ipv4_info
             if ipv6_info:
                 interface_item['ipv6'] = ipv6_info
+            if lane_info:
+                interface_item['lanes'] = lane_info
             raw_output.append(interface_item)
 
     return raw_output if raw else _process(raw_output)

jc/parsers/ini.py

@@ -1,10 +1,12 @@
-"""jc - JSON Convert `INI` file parser
+"""jc - JSON Convert INI file parser
 
-Parses standard `INI` files and files containing simple key/value pairs.
+Parses standard INI files.
 
 - Delimiter can be `=` or `:`. Missing values are supported.
 - Comment prefix can be `#` or `;`. Comments must be on their own line.
 - If duplicate keys are found, only the last value will be used.
+- If any section names have the same name as a top-level key, the top-level
+  key will be overwritten by the section data.
 
 > Note: Values starting and ending with double or single quotation marks
 > will have the marks removed. If you would like to keep the quotation
@@ -22,65 +24,82 @@ Usage (module):
 
 Schema:
 
-ini or key/value document converted to a dictionary - see the configparser
+INI document converted to a dictionary - see the python configparser
 standard library documentation for more details.
 
     {
-      "key1":       string,
-      "key2":       string
+      "<key1>":               string,
+      "<key2>":               string,
+      "<section1>": {
+        "<key1>":             string,
+        "<key2>":             string
+      },
+      "<section2>": {
+        "<key1>":             string,
+        "<key2>":             string
+      }
     }
 
 Examples:
 
     $ cat example.ini
-    [DEFAULT]
-    ServerAliveInterval = 45
-    Compression = yes
-    CompressionLevel = 9
-    ForwardX11 = yes
+    foo = fiz
+    bar = buz
 
-    [bitbucket.org]
-    User = hg
+    [section1]
+    fruit = apple
+    color = blue
 
-    [topsecret.server.com]
-    Port = 50022
-    ForwardX11 = no
+    [section2]
+    fruit = pear
+    color = green
 
     $ cat example.ini | jc --ini -p
     {
-      "bitbucket.org": {
-        "ServerAliveInterval": "45",
-        "Compression": "yes",
-        "CompressionLevel": "9",
-        "ForwardX11": "yes",
-        "User": "hg"
+      "foo": "fiz",
+      "bar": "buz",
+      "section1": {
+        "fruit": "apple",
+        "color": "blue"
       },
-      "topsecret.server.com": {
-        "ServerAliveInterval": "45",
-        "Compression": "yes",
-        "CompressionLevel": "9",
-        "ForwardX11": "no",
-        "Port": "50022"
+      "section2": {
+        "fruit": "pear",
+        "color": "green"
       }
     }
 """
 import jc.utils
 import configparser
+import uuid
 
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.8'
+    version = '2.0'
     description = 'INI file parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
-    details = 'Using configparser from the standard library'
+    details = 'Using configparser from the python standard library'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
 
 
 __version__ = info.version
 
 
+def _remove_quotes(value):
+    if value is None:
+        value = ''
+
+    elif value.startswith('"') and value.endswith('"'):
+        value = value[1:-1]
+
+    elif value.startswith("'") and value.endswith("'"):
+        value = value[1:-1]
+
+    return value
+
+
 def _process(proc_data):
     """
     Final processing to conform to the schema.
@@ -91,32 +110,16 @@ def _process(proc_data):
 
     Returns:
 
-        Dictionary representing an ini or simple key/value pair document.
+        Dictionary representing the INI file.
     """
     # remove quotation marks from beginning and end of values
-    for heading in proc_data:
-        # standard ini files with headers
-        if isinstance(proc_data[heading], dict):
-            for key, value in proc_data[heading].items():
-                if value is not None and value.startswith('"') and value.endswith('"'):
-                    proc_data[heading][key] = value.lstrip('"').rstrip('"')
+    for k, v in proc_data.items():
+        if isinstance(v, dict):
+            for key, value in v.items():
+                v[key] = _remove_quotes(value)
+            continue
 
-                elif value is not None and value.startswith("'") and value.endswith("'"):
-                    proc_data[heading][key] = value.lstrip("'").rstrip("'")
-
-                elif value is None:
-                    proc_data[heading][key] = ''
-
-        # simple key/value files with no headers
-        else:
-            if proc_data[heading] is not None and proc_data[heading].startswith('"') and proc_data[heading].endswith('"'):
-                proc_data[heading] = proc_data[heading].lstrip('"').rstrip('"')
-
-            elif proc_data[heading] is not None and proc_data[heading].startswith("'") and proc_data[heading].endswith("'"):
-                proc_data[heading] = proc_data[heading].lstrip("'").rstrip("'")
-
-            elif proc_data[heading] is None:
-                proc_data[heading] = ''
+        proc_data[k] = _remove_quotes(v)
 
     return proc_data
 
@@ -133,7 +136,7 @@ def parse(data, raw=False, quiet=False):
 
     Returns:
 
-        Dictionary representing the ini file
+        Dictionary representing the INI file.
     """
     jc.utils.compatibility(__name__, info.compatible, quiet)
     jc.utils.input_type_check(data)
@@ -142,25 +145,36 @@ def parse(data, raw=False, quiet=False):
 
     if jc.utils.has_data(data):
 
-        ini = configparser.ConfigParser(allow_no_value=True,
-                                        interpolation=None,
-                                        strict=False)
+        ini_parser = configparser.ConfigParser(
+            allow_no_value=True,
+            interpolation=None,
+            default_section=None,
+            strict=False
+        )
 
         # don't convert keys to lower-case:
-        ini.optionxform = lambda option: option
+        ini_parser.optionxform = lambda option: option
 
         try:
-            ini.read_string(data)
-            raw_output = {s: dict(ini.items(s)) for s in ini.sections()}
+            ini_parser.read_string(data)
+            raw_output = {s: dict(ini_parser.items(s)) for s in ini_parser.sections()}
 
         except configparser.MissingSectionHeaderError:
-            data = '[data]\n' + data
-            ini.read_string(data)
-            output_dict = {s: dict(ini.items(s)) for s in ini.sections()}
-            for key, value in output_dict['data'].items():
-                raw_output[key] = value
+            # find a top-level section name that will not collide with any existing ones
+            while True:
+                my_uuid = str(uuid.uuid4())
+                if my_uuid not in data:
+                    break
+
+            data = f'[{my_uuid}]\n' + data
+            ini_parser.read_string(data)
+            temp_dict = {s: dict(ini_parser.items(s)) for s in ini_parser.sections()}
+
+            # move items under fake top-level sections to the root
+            raw_output = temp_dict.pop(my_uuid)
+
+            # get the rest of the sections
+            raw_output.update(temp_dict)
+
+    return raw_output if raw else _process(raw_output)
 
-    if raw:
-        return raw_output
-    else:
-        return _process(raw_output)

jc/parsers/ini_dup.py

@@ -0,0 +1,225 @@
+"""jc - JSON Convert INI with duplicate key file parser
+
+Parses standard INI files and preserves duplicate values. All values are
+contained in lists/arrays.
+
+- Delimiter can be `=` or `:`. Missing values are supported.
+- Comment prefix can be `#` or `;`. Comments must be on their own line.
+- If any section names have the same name as a top-level key, the top-level
+  key will be overwritten by the section data.
+- If multi-line values are used, each line will be a separate item in the
+  value list. Blank lines in multi-line values are not supported.
+
+> Note: Values starting and ending with double or single quotation marks
+> will have the marks removed. If you would like to keep the quotation
+> marks, use the `-r` command-line argument or the `raw=True` argument in
+> `parse()`.
+
+Usage (cli):
+
+    $ cat foo.ini | jc --ini
+
+Usage (module):
+
+    import jc
+    result = jc.parse('ini', ini_file_output)
+
+Schema:
+
+INI document converted to a dictionary - see the python configparser
+standard library documentation for more details.
+
+    {
+      "<key1>": [
+                            string
+      ],
+      "<key2>": [
+                            string
+      ],
+      "<section1>": {
+        "<key1>": [
+                            string
+        ],
+        "<key2>": [
+                            string
+        ]
+      }
+    }
+
+Examples:
+
+    $ cat example.ini
+    foo = fiz
+    bar = buz
+
+    [section1]
+    fruit = apple
+    color = blue
+    color = red
+
+    [section2]
+    fruit = pear
+    fruit = peach
+    color = green
+
+    $ cat example.ini | jc --ini -p
+    {
+      "foo": [
+        "fiz"
+      ],
+      "bar": [
+        "buz"
+      ],
+      "section1": {
+        "fruit": [
+          "apple"
+        ],
+        "color": [
+          "blue",
+          "red"
+        ]
+      },
+      "section2": {
+        "fruit": [
+          "pear",
+          "peach"
+        ],
+        "color": [
+          "green"
+        ]
+      }
+    }
+"""
+import jc.utils
+import configparser
+import uuid
+
+
+class info():
+    """Provides parser metadata (version, author, etc.)"""
+    version = '1.0'
+    description = 'INI with duplicate key file parser'
+    author = 'Kelly Brazil'
+    author_email = 'kellyjonbrazil@gmail.com'
+    details = 'Using configparser from the python standard library'
+    compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
+
+
+__version__ = info.version
+
+
+class MultiDict(dict):
+    # https://stackoverflow.com/a/38286559/12303989
+    def __setitem__(self, key, value):
+        if key in self:
+            if isinstance(value, list):
+                self[key].extend(value)
+
+            elif isinstance(value, str):
+                if len(self[key])>1:
+                    return
+
+        else:
+            super().__setitem__(key, value)
+
+
+def _remove_quotes(value):
+    if value is None:
+        value = ''
+
+    elif value.startswith('"') and value.endswith('"'):
+        value = value[1:-1]
+
+    elif value.startswith("'") and value.endswith("'"):
+        value = value[1:-1]
+
+    return value
+
+
+def _process(proc_data):
+    """
+    Final processing to conform to the schema.
+
+    Parameters:
+
+        proc_data:   (Dictionary) raw structured data to process
+
+    Returns:
+
+        Dictionary representing the INI file.
+    """
+    # remove quotation marks from beginning and end of values
+    for k, v in proc_data.items():
+        if isinstance(v, dict):
+            for key, value in v.items():
+                if isinstance(value, list):
+                    v[key] = [_remove_quotes(x) for x in value]
+                else:
+                    v[key] = _remove_quotes(value)
+            continue
+
+        elif isinstance(v, list):
+            proc_data[k] = [_remove_quotes(x) for x in v]
+
+        else:
+            proc_data[k] = _remove_quotes(v)
+
+    return proc_data
+
+
+def parse(data, raw=False, quiet=False):
+    """
+    Main text parsing function
+
+    Parameters:
+
+        data:        (string)  text data to parse
+        raw:         (boolean) unprocessed output if True
+        quiet:       (boolean) suppress warning messages if True
+
+    Returns:
+
+        Dictionary representing the INI file.
+    """
+    jc.utils.compatibility(__name__, info.compatible, quiet)
+    jc.utils.input_type_check(data)
+
+    raw_output = {}
+
+    if jc.utils.has_data(data):
+
+        ini_parser = configparser.ConfigParser(
+            dict_type = MultiDict,
+            allow_no_value=True,
+            interpolation=None,
+            default_section=None,
+            empty_lines_in_values=False,
+            strict=False
+        )
+
+        # don't convert keys to lower-case:
+        ini_parser.optionxform = lambda option: option
+
+        try:
+            ini_parser.read_string(data)
+            raw_output = {s: dict(ini_parser.items(s)) for s in ini_parser.sections()}
+
+        except configparser.MissingSectionHeaderError:
+            # find a top-level section name that will not collide with any existing ones
+            while True:
+                my_uuid = str(uuid.uuid4())
+                if my_uuid not in data:
+                    break
+
+            data = f'[{my_uuid}]\n' + data
+            ini_parser.read_string(data)
+            temp_dict = {s: dict(ini_parser.items(s)) for s in ini_parser.sections()}
+
+            # move items under fake top-level sections to the root
+            raw_output = temp_dict.pop(my_uuid)
+
+            # get the rest of the sections
+            raw_output.update(temp_dict)
+
+    return raw_output if raw else _process(raw_output)

jc/parsers/iostat.py

@@ -166,6 +166,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['iostat']
+    tags = ['command']
 
 
 __version__ = info.version