Merge pull request #343 from kellyjonbrazil/dev

Dev v1.22.4

CHANGELOG

@@ -1,5 +1,15 @@
 jc changelog
 
+20221230 v1.22.4
+- Add `iwconfig` command parser
+- Add NeXTSTEP format support to the PLIST file parser
+- Fix `proc` parser magic signature detection for `/proc/pid/stat` hacks
+- Fix `x509-cert` parser for string serial numbers
+- Add category tags to parser metadata: generic, standard, file, string, binary, command
+- Add "list parsers by category" view to help
+- Fix python 3.6-related issues
+- Add python 3.6 to automated tests
+
 20221216 v1.22.3
 - Add Common Log Format and Combined Log Format file parser (standard and streaming)
 - Add PostgreSQL password file parser

README.md

@@ -206,10 +206,11 @@ option.
 | `   --ip-address` | IPv4 and IPv6 Address string parser                     | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ip_address)     |
 | `     --iptables` | `iptables` command parser                               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iptables)       |
 | `      --iw-scan` | `iw dev [device] scan` command parser                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iw_scan)        |
+| `     --iwconfig` | `iwconfig` command parser                               | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iwconfig)       |
 | ` --jar-manifest` | Java MANIFEST.MF file parser                            | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jar_manifest)   |
 | `         --jobs` | `jobs` command parser                                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jobs)           |
 | `          --jwt` | JWT string parser                                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/jwt)            |
-| `           --kv` | Key/Value file parser                                   | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/kv)             |
+| `           --kv` | Key/Value file and string parser                        | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/kv)             |
 | `         --last` | `last` and `lastb` command parser                       | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/last)           |
 | `           --ls` | `ls` command parser                                     | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ls)             |
 | `         --ls-s` | `ls` command streaming parser                           | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ls_s)           |

completions/jc_bash_completion.sh

@@ -3,8 +3,8 @@ _jc()
     local cur prev words cword jc_commands jc_parsers jc_options \
           jc_about_options jc_about_mod_options jc_help_options jc_special_options
 
-    jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo)
+    jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo)
-    jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo)
+    jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo)
     jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y)
     jc_about_options=(--about -a)
     jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C)

completions/jc_zsh_completion.sh

@@ -9,7 +9,7 @@ _jc() {
              jc_help_options jc_help_options_describe \
              jc_special_options jc_special_options_describe
 
-    jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo)
+    jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo)
     jc_commands_describe=(
         'acpi:run "acpi" command with magic syntax.'
         'airport:run "airport" command with magic syntax.'
@@ -38,6 +38,7 @@ _jc() {
         'iostat:run "iostat" command with magic syntax.'
         'iptables:run "iptables" command with magic syntax.'
         'iw:run "iw" command with magic syntax.'
+        'iwconfig:run "iwconfig" command with magic syntax.'
         'jobs:run "jobs" command with magic syntax.'
         'last:run "last" command with magic syntax.'
         'lastb:run "lastb" command with magic syntax.'
@@ -101,7 +102,7 @@ _jc() {
         'xrandr:run "xrandr" command with magic syntax.'
         'zipinfo:run "zipinfo" command with magic syntax.'
     )
-    jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo)
+    jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo)
     jc_parsers_describe=(
         '--acpi:`acpi` command parser'
         '--airport:`airport -I` command parser'
@@ -155,10 +156,11 @@ _jc() {
         '--ip-address:IPv4 and IPv6 Address string parser'
         '--iptables:`iptables` command parser'
         '--iw-scan:`iw dev [device] scan` command parser'
+        '--iwconfig:`iwconfig` command parser'
         '--jar-manifest:Java MANIFEST.MF file parser'
         '--jobs:`jobs` command parser'
         '--jwt:JWT string parser'
-        '--kv:Key/Value file parser'
+        '--kv:Key/Value file and string parser'
         '--last:`last` and `lastb` command parser'
         '--ls:`ls` command parser'
         '--ls-s:`ls` command streaming parser'
@@ -289,8 +291,8 @@ _jc() {
     )
     jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y)
     jc_options_describe=(
-        '--force-color:force color output even when using pipes (overrides -m)'
+        '--force-color:force color output (overrides -m)'
-        '-C:force color output even when using pipes (overrides -m)'
+        '-C:force color output (overrides -m)'
         '--debug:debug (double for verbose debug)'
         '-d:debug (double for verbose debug)'
         '--monochrome:monochrome output'
@@ -321,8 +323,8 @@ _jc() {
         '-y:YAML output'
         '--monochrome:monochrome output'
         '-m:monochrome output'
-        '--force-color:force color output even when using pipes (overrides -m)'
+        '--force-color:force color output (overrides -m)'
-        '-C:force color output even when using pipes (overrides -m)'
+        '-C:force color output (overrides -m)'
     )
     jc_help_options=(--help -h)
     jc_help_options_describe=(

docs/parsers/findmnt.md

@@ -114,4 +114,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/ifconfig.md

@@ -240,4 +240,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, aix, freebsd, darwin
 
-Version 2.1 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 2.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/iwconfig.md

@@ -0,0 +1,112 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+<a id="jc.parsers.iwconfig"></a>
+
+# jc.parsers.iwconfig
+
+jc - JSON Convert `iwconfig` command output parser
+
+No `iwconfig` options are supported.
+
+Usage (cli):
+
+    $ iwconfig | jc --iwconfig
+
+or
+
+    $ jc iwconfig
+
+Usage (module):
+
+    import jc
+    result = jc.parse('iwconfig', iwconfig_command_output)
+
+Schema:
+
+    [
+        {
+            "name":                         string,
+            "protocol":                     string,
+            "essid":                        string,
+            "mode":                         string,
+            "frequency":                    float,
+            "frequency_unit":               string,
+            "access_point":                 string,
+            "bit_rate":                     float,
+            "bit_rate_unit":                string,
+            "tx_power":                     integer,
+            "tx_power_unit":                string,
+            "retry_short_limit":            integer,
+            "rts_threshold":                boolean,
+            "fragment_threshold":           boolean,
+            "power_management":             boolean,
+            "link_quality":                 string,
+            "signal_level":                 integer,
+            "signal_level_unit":            string,
+            "rx_invalid_nwid":              integer,
+            "rx_invalid_crypt":             integer,
+            "rx_invalid_frag":              integer,
+            "tx_excessive_retries":         integer,
+            "invalid_misc":                 integer,
+            "missed_beacon":                integer
+        }
+    ]
+
+
+Examples:
+
+    $  iwconfig | jc --iwconfig -p
+    [
+      {
+        "name": "wlp5s0",
+        "protocol": "IEEE 802.11",
+        "essid": "BLABLABLA",
+        "mode": "Managed",
+        "frequency": 5.18,
+        "frequency_unit": "GHz",
+        "access_point": "E6:64:DA:16:51:BF",
+        "bit_rate": 6.0,
+        "bit_rate_unit": "Mb/s",
+        "tx_power": 30,
+        "tx_power_unit": "dBm",
+        "retry_short_limit": 7,
+        "rts_threshold": false,
+        "fragment_threshold": false,
+        "power_management": true,
+        "link_quality": "61/70",
+        "signal_level": -49,
+        "signal_level_unit": "dBm",
+        "rx_invalid_nwid": 0,
+        "rx_invalid_crypt": 0,
+        "rx_invalid_frag": 0,
+        "tx_excessive_retries": 0,
+        "invalid_misc": 2095,
+        "missed_beacon": 0
+      }
+    ]
+
+<a id="jc.parsers.iwconfig.parse"></a>
+
+### parse
+
+```python
+def parse(data: str,
+          raw: bool = False,
+          quiet: bool = False) -> List[JSONDictType]
+```
+
+Main text parsing function
+
+Parameters:
+
+    data:        (string)  text data to parse
+    raw:         (boolean) unprocessed output if True
+    quiet:       (boolean) suppress warning messages if True
+
+Returns:
+
+    List of Dictionaries. Raw or processed structured data.
+
+### Parser Information
+Compatibility:  linux
+
+Version 1.0 by Thomas Vincent (vrince@gmail.com)

docs/parsers/kv.md

@@ -3,7 +3,7 @@
 
 # jc.parsers.kv
 
-jc - JSON Convert `Key/Value` file parser
+jc - JSON Convert `Key/Value` file and string parser
 
 Supports files containing simple key/value pairs.
 

docs/parsers/plist.md

@@ -5,7 +5,7 @@
 
 jc - JSON Convert PLIST file parser
 
-Converts binary and XML PLIST files.
+Converts binary, XML, and NeXTSTEP PLIST files.
 
 Binary values are converted into an ASCII hex representation.
 
@@ -69,9 +69,9 @@ Parameters:
 
 Returns:
 
-    List of Dictionaries. Raw or processed structured data.
+    Dictionary. Raw or processed structured data.
 
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/proc.md

@@ -139,4 +139,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/x509_cert.md

@@ -32,6 +32,7 @@ Schema:
         "tbs_certificate": {
           "version":                      string,
           "serial_number":                string,  # [0]
+          "serial_number_str":            string,
           "signature": {
             "algorithm":                  string,
             "parameters":                 string/null,
@@ -43,7 +44,9 @@ Schema:
             "organization_name":          array/string,
             "organizational_unit_name":   array/string,
             "common_name":                string,
-            "email_address":              string
+            "email_address":              string,
+            "serial_number":              string,   # [0]
+            "serial_number_str":          string
           },
           "validity": {
             "not_before":                 integer,  # [1]
@@ -58,7 +61,9 @@ Schema:
             "organization_name":          array/string,
             "organizational_unit_name":   array/string,
             "common_name":                string,
-            "email_address":              string
+            "email_address":              string,
+            "serial_number":              string,   # [0]
+            "serial_number_str":          string
           },
           "subject_public_key_info": {
             "algorithm": {
@@ -428,4 +433,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

jc/cli.py

@@ -9,7 +9,7 @@ from datetime import datetime, timezone
 import textwrap
 import shlex
 import subprocess
-from typing import List, Union, Optional, TextIO
+from typing import List, Dict, Union, Optional, TextIO
 from types import ModuleType
 from .lib import (
     __version__, parser_info, all_parser_info, parsers, _get_parser, _parser_is_streaming,
@@ -64,12 +64,12 @@ if PYGMENTS_INSTALLED:
 class JcCli():
     __slots__ = (
         'data_in', 'data_out', 'options', 'args', 'parser_module', 'parser_name', 'indent', 'pad',
-        'custom_colors', 'show_hidden', 'ascii_only', 'json_separators', 'json_indent',
+        'custom_colors', 'show_hidden', 'show_categories', 'ascii_only', 'json_separators',
-        'run_timestamp', 'about', 'debug', 'verbose_debug', 'force_color', 'mono', 'help_me',
+        'json_indent', 'run_timestamp', 'about', 'debug', 'verbose_debug', 'force_color', 'mono',
-        'pretty', 'quiet', 'ignore_exceptions', 'raw', 'meta_out', 'unbuffer', 'version_info',
+        'help_me', 'pretty', 'quiet', 'ignore_exceptions', 'raw', 'meta_out', 'unbuffer',
-        'yaml_output', 'bash_comp', 'zsh_comp', 'magic_found_parser', 'magic_options',
+        'version_info', 'yaml_output', 'bash_comp', 'zsh_comp', 'magic_found_parser',
-        'magic_run_command', 'magic_run_command_str', 'magic_stdout', 'magic_stderr',
+        'magic_options', 'magic_run_command', 'magic_run_command_str', 'magic_stdout',
-        'magic_returncode'
+        'magic_stderr', 'magic_returncode'
     )
 
     def __init__(self) -> None:
@@ -83,6 +83,7 @@ class JcCli():
         self.pad: int = 0
         self.custom_colors: CustomColorType = {}
         self.show_hidden: bool = False
+        self.show_categories: bool = False
         self.ascii_only: bool = False
         self.json_separators: Optional[tuple[str, str]] = (',', ':')
         self.json_indent: Optional[int] = None
@@ -198,6 +199,41 @@ class JcCli():
 
         return ptext
 
+    def parser_categories_text(self) -> str:
+        """Return lists of parsers by category"""
+        category_text: str = ''
+        padding_char: str = ' '
+        all_parsers = all_parser_info(show_hidden=True, show_deprecated=False)
+        generic = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'generic' in x['tags']]
+        standard = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'standard' in x['tags']]
+        command = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if 'command' in x['tags']]
+        file_str_bin = [
+            {'arg': x['argument'], 'desc': x['description']} for x in all_parsers
+                if 'file' in x['tags'] or
+                'string' in x['tags'] or
+                'binary' in x['tags']
+        ]
+        streaming = [{'arg': x['argument'], 'desc': x['description']} for x in all_parsers if x.get('streaming')]
+        categories: Dict = {
+            'Generic Parsers:': generic,
+            'Standard Spec Parsers:': standard,
+            'File/String/Binary Parsers:': file_str_bin,
+            'Streaming Parsers:': streaming,
+            'Command Parsers:': command
+        }
+
+        for cat, cat_objs in categories.items():
+            category_text += f'{cat}  ({len(cat_objs)})\n'
+            for p in cat_objs:
+                parser_arg: str = p.get('arg', 'UNKNOWN')
+                parser_desc: str = p.get('desc', 'No description available.')
+                padding: int = self.pad - len(parser_arg)
+                padding_text: str = padding_char * padding
+                category_text += f'{parser_arg}{padding_text}{parser_desc}\n'
+            category_text += '\n'
+
+        return category_text[:-1]
+
     def options_text(self) -> str:
         """Return the argument and description information from each option"""
         otext: str = ''
@@ -236,8 +272,6 @@ class JcCli():
 
     def helptext(self) -> str:
         """Return the help text with the list of parsers"""
-        self.indent = 4
-        self.pad = 20
         parsers_string: str = self.parsers_text()
         options_string: str = self.options_text()
         helptext_string: str = f'{helptext_preamble_string}{parsers_string}\nOptions:\n{options_string}\n{helptext_end_string}'
@@ -248,6 +282,13 @@ class JcCli():
         Pages the parser documentation if a parser is found in the arguments,
         otherwise the general help text is printed.
         """
+        self.indent = 4
+        self.pad = 22
+
+        if self.show_categories:
+            utils._safe_print(self.parser_categories_text())
+            return
+
         for arg in self.args:
             parser_name: str = self.parser_shortname(arg)
 
@@ -655,6 +696,7 @@ class JcCli():
         self.force_color = 'C' in self.options
         self.help_me = 'h' in self.options
         self.show_hidden = self.options.count('h') > 1   # verbose help
+        self.show_categories = self.options.count('h') > 2
         self.pretty = 'p' in self.options
         self.quiet = 'q' in self.options
         self.ignore_exceptions = self.options.count('q') > 1

jc/cli_data.py

@@ -3,7 +3,7 @@ from typing import List, Dict
 
 long_options_map: Dict[str, List[str]] = {
     '--about': ['a', 'about jc'],
-    '--force-color': ['C', 'force color output even when using pipes (overrides -m)'],
+    '--force-color': ['C', 'force color output (overrides -m)'],
     '--debug': ['d', 'debug (double for verbose debug)'],
     '--help': ['h', 'help (--help --parser_name for parser documentation)'],
     '--monochrome': ['m', 'monochrome output'],
@@ -91,6 +91,7 @@ Examples:
     Parser Documentation:
         $ jc --help --dig
 
-    Show Hidden Parsers:
+    More Help:
-        $ jc -hh
+        $ jc -hh          # show hidden parsers
+        $ jc -hhh         # list parsers by category tags
 '''

jc/jc_types.py

@@ -20,6 +20,7 @@ if sys.version_info >= (3, 8):
             "author_email": str,
             "compatible": List[str],
             "magic_commands": List[str],
+            "tags": List[str],
             "documentation": str,
             "streaming": bool,
             "plugin": bool,

jc/lib.py

@@ -9,7 +9,7 @@ from .jc_types import ParserInfoType, JSONDictType
 from jc import appdirs
 
 
-__version__ = '1.22.3'
+__version__ = '1.22.4'
 
 parsers: List[str] = [
     'acpi',
@@ -65,6 +65,7 @@ parsers: List[str] = [
     'iptables',
     'iso-datetime',
     'iw-scan',
+    'iwconfig',
     'jar-manifest',
     'jobs',
     'jwt',

jc/parsers/acpi.py

@@ -233,6 +233,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['acpi']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/airport.py

@@ -86,6 +86,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['darwin']
     magic_commands = ['airport -I']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/airport_s.py

@@ -115,6 +115,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['darwin']
     magic_commands = ['airport -s']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/arp.py

@@ -125,6 +125,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'aix', 'freebsd', 'darwin']
     magic_commands = ['arp']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/asciitable.py

@@ -130,6 +130,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['generic', 'string']
 
 
 __version__ = info.version

jc/parsers/asciitable_m.py

@@ -115,6 +115,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['generic', 'string']
 
 
 __version__ = info.version

jc/parsers/blkid.py

@@ -127,6 +127,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['blkid']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/cbt.py

@@ -106,6 +106,7 @@ class info():
     author_email = 'andreas.weiden@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
     magic_commands = ['cbt']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/cef.py

@@ -129,6 +129,8 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using the pycef library at https://github.com/DavidJBianco/pycef/releases/tag/v1.11-2'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
+
 
 __version__ = info.version
 

jc/parsers/cef_s.py

@@ -103,6 +103,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using the pycef library at https://github.com/DavidJBianco/pycef/releases/tag/v1.11-2'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
     streaming = True
 
 

jc/parsers/chage.py

@@ -63,6 +63,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['chage --list', 'chage -l']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/cksum.py

@@ -60,6 +60,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
     magic_commands = ['cksum', 'sum']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/clf.py

@@ -179,6 +179,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
 
 
 __version__ = info.version

jc/parsers/clf_s.py

@@ -95,6 +95,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
     streaming = True
 
 

jc/parsers/crontab.py

@@ -180,6 +180,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
     magic_commands = ['crontab']
+    tags = ['file', 'command']
 
 
 __version__ = info.version

jc/parsers/crontab_u.py

@@ -176,6 +176,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
+    tags = ['file', 'command']
 
 
 __version__ = info.version

jc/parsers/csv.py

@@ -86,6 +86,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using the python standard csv library'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
 
 
 __version__ = info.version

jc/parsers/csv_s.py

@@ -69,6 +69,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using the python standard csv library'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
     streaming = True
 
 

jc/parsers/date.py

@@ -84,6 +84,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'freebsd']
     magic_commands = ['date']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/datetime_iso.py

@@ -75,6 +75,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using the pyiso8601 library from https://github.com/micktwomey/pyiso8601/releases/tag/1.0.2'
     compatible = ['linux', 'aix', 'freebsd', 'darwin', 'win32', 'cygwin']
+    tags = ['standard', 'string']
 
 
 __version__ = info.version

jc/parsers/df.py

@@ -105,6 +105,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'freebsd']
     magic_commands = ['df']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/dig.py

@@ -328,6 +328,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'aix', 'freebsd', 'darwin', 'win32', 'cygwin']
     magic_commands = ['dig']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/dir.py

@@ -126,6 +126,7 @@ class info():
     author = 'Rasheed Elsaleh'
     author_email = 'rasheed@rebelliondefense.com'
     compatible = ['win32']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/dmidecode.py

@@ -131,6 +131,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['dmidecode']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/dpkg_l.py

@@ -138,6 +138,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['dpkg -l']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/du.py

@@ -98,6 +98,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
     magic_commands = ['du']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/email_address.py

@@ -47,6 +47,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'string']
 
 
 __version__ = info.version

jc/parsers/env.py

@@ -78,6 +78,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
     magic_commands = ['env', 'printenv']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/file.py

@@ -69,6 +69,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'aix', 'freebsd', 'darwin']
     magic_commands = ['file']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/findmnt.py

@@ -93,12 +93,13 @@ import jc.utils
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.0'
+    version = '1.1'
     description = '`findmnt` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['findmnt']
+    tags = ['command']
 
 
 __version__ = info.version
@@ -140,7 +141,7 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]:
     return proc_data
 
 
-def _replace(matchobj: re.Match) -> str:
+def _replace(matchobj):
     if matchobj:
         matchlen = len(matchobj.group(1))
         return ' ' * matchlen + '/'

jc/parsers/finger.py

@@ -98,6 +98,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'freebsd']
     magic_commands = ['finger']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/foo.py

@@ -48,6 +48,9 @@ class info():
 
     # compatible options: linux, darwin, cygwin, win32, aix, freebsd
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+
+    # tags options: generic, standard, file, string, binary, command
+    tags = ['command']
     magic_commands = ['foo']
 
 

jc/parsers/foo_s.py

@@ -58,6 +58,9 @@ class info():
 
     # compatible options: linux, darwin, cygwin, win32, aix, freebsd
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+
+    # tags options: generic, standard, file, string, binary, command
+    tags = ['command']
     streaming = True
 
 

jc/parsers/free.py

@@ -79,6 +79,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['free']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/fstab.py

@@ -90,6 +90,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/git_log.py

@@ -159,6 +159,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
     magic_commands = ['git log']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/git_log_s.py

@@ -93,6 +93,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['command']
     streaming = True
 
 

jc/parsers/git_ls_remote.py

@@ -72,6 +72,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
     magic_commands = ['git ls-remote']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/gpg.py

@@ -126,6 +126,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['gpg --with-colons']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/group.py

@@ -114,6 +114,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/gshadow.py

@@ -82,6 +82,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'aix', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/hash.py

@@ -43,6 +43,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/hashsum.py

@@ -76,6 +76,7 @@ class info():
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
     magic_commands = ['md5sum', 'md5', 'shasum', 'sha1sum', 'sha224sum',
                       'sha256sum', 'sha384sum', 'sha512sum']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/hciconfig.py

@@ -323,6 +323,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['hciconfig']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/history.py

@@ -69,6 +69,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Optimizations by https://github.com/philippeitis'
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/hosts.py

@@ -79,6 +79,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/id.py

@@ -112,6 +112,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
     magic_commands = ['id']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/ifconfig.py

@@ -219,12 +219,13 @@ import jc.utils
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '2.1'
+    version = '2.2'
     description = '`ifconfig` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'aix', 'freebsd', 'darwin']
     magic_commands = ['ifconfig']
+    tags = ['command']
 
 
 __version__ = info.version
@@ -326,7 +327,7 @@ def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]:
     return proc_data
 
 
-def _bundle_match(pattern_list: List[re.Pattern], string: str) -> Optional[re.Match]:
+def _bundle_match(pattern_list, string):
     """Returns a match object if a string matches one of a list of patterns.
     If no match is found, returns None"""
     for pattern in pattern_list:

jc/parsers/ini.py

@@ -76,6 +76,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Using configparser from the standard library'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'file', 'string']
 
 
 __version__ = info.version

jc/parsers/iostat.py

@@ -166,6 +166,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['iostat']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/iostat_s.py

@@ -113,6 +113,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
+    tags = ['command']
     streaming = True
 
 

jc/parsers/ip_address.py

@@ -538,6 +538,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'string']
 
 
 __version__ = info.version

jc/parsers/iptables.py

@@ -169,6 +169,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['iptables']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/iso_datetime.py

@@ -17,6 +17,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'Deprecated - please use datetime-iso'
     compatible = ['linux', 'aix', 'freebsd', 'darwin', 'win32', 'cygwin']
+    tags = ['standard', 'string']
     deprecated = True
 
 

jc/parsers/iw_scan.py

@@ -129,6 +129,7 @@ class info():
     details = 'Enhancements by Philipp Schmitt (https://pschmitt.dev/)'
     compatible = ['linux']
     magic_commands = ['iw dev']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/iwconfig.py

@@ -0,0 +1,204 @@
+"""jc - JSON Convert `iwconfig` command output parser
+
+No `iwconfig` options are supported.
+
+Usage (cli):
+
+    $ iwconfig | jc --iwconfig
+
+or
+
+    $ jc iwconfig
+
+Usage (module):
+
+    import jc
+    result = jc.parse('iwconfig', iwconfig_command_output)
+
+Schema:
+
+    [
+        {
+            "name":                         string,
+            "protocol":                     string,
+            "essid":                        string,
+            "mode":                         string,
+            "frequency":                    float,
+            "frequency_unit":               string,
+            "access_point":                 string,
+            "bit_rate":                     float,
+            "bit_rate_unit":                string,
+            "tx_power":                     integer,
+            "tx_power_unit":                string,
+            "retry_short_limit":            integer,
+            "rts_threshold":                boolean,
+            "fragment_threshold":           boolean,
+            "power_management":             boolean,
+            "link_quality":                 string,
+            "signal_level":                 integer,
+            "signal_level_unit":            string,
+            "rx_invalid_nwid":              integer,
+            "rx_invalid_crypt":             integer,
+            "rx_invalid_frag":              integer,
+            "tx_excessive_retries":         integer,
+            "invalid_misc":                 integer,
+            "missed_beacon":                integer
+        }
+    ]
+
+
+Examples:
+
+    $  iwconfig | jc --iwconfig -p
+    [
+      {
+        "name": "wlp5s0",
+        "protocol": "IEEE 802.11",
+        "essid": "BLABLABLA",
+        "mode": "Managed",
+        "frequency": 5.18,
+        "frequency_unit": "GHz",
+        "access_point": "E6:64:DA:16:51:BF",
+        "bit_rate": 6.0,
+        "bit_rate_unit": "Mb/s",
+        "tx_power": 30,
+        "tx_power_unit": "dBm",
+        "retry_short_limit": 7,
+        "rts_threshold": false,
+        "fragment_threshold": false,
+        "power_management": true,
+        "link_quality": "61/70",
+        "signal_level": -49,
+        "signal_level_unit": "dBm",
+        "rx_invalid_nwid": 0,
+        "rx_invalid_crypt": 0,
+        "rx_invalid_frag": 0,
+        "tx_excessive_retries": 0,
+        "invalid_misc": 2095,
+        "missed_beacon": 0
+      }
+    ]
+
+"""
+import re
+from typing import List, Dict
+from jc.jc_types import JSONDictType
+import jc.utils
+
+class info():
+    """Provides parser metadata (version, author, etc.)"""
+    version = '1.0'
+    description = '`iwconfig` command parser'
+    author = 'Thomas Vincent'
+    author_email = 'vrince@gmail.com'
+    compatible = ['linux']
+    magic_commands = ['iwconfig']
+    tags = ['command']
+
+
+__version__ = info.version
+
+
+def _process(proc_data: List[JSONDictType]) -> List[JSONDictType]:
+    """
+    Final processing to conform to the schema.
+
+    Parameters:
+
+        proc_data:   (List of Dictionaries) raw structured data to process
+
+    Returns:
+
+        List of Dictionaries. Structured to conform to the schema.
+    """
+    int_list = [
+        'signal_level', 'rx_invalid_nwid', 'rx_invalid_crypt', 'rx_invalid_frag',
+        'tx_excessive_retries', 'invalid_misc', 'missed_beacon', 'tx_power', 'retry_short_limit'
+    ]
+    float_list = ['frequency', 'bit_rate']
+    bool_list = ['rts_threshold', 'fragment_threshold', 'power_management']
+
+    proc_data = [ { key: int(value) if key in int_list else value for key, value in proc_data_item.items() } for proc_data_item in proc_data ]
+    proc_data = [ { key: float(value) if key in float_list else value for key, value in proc_data_item.items() } for proc_data_item in proc_data ]
+    proc_data = [ { key: value == 'on' if key in bool_list else value for key, value in proc_data_item .items() } for proc_data_item in proc_data ]
+
+    return proc_data
+
+
+def parse(
+    data: str,
+    raw: bool = False,
+    quiet: bool = False
+) -> List[JSONDictType]:
+    """
+    Main text parsing function
+
+    Parameters:
+
+        data:        (string)  text data to parse
+        raw:         (boolean) unprocessed output if True
+        quiet:       (boolean) suppress warning messages if True
+
+    Returns:
+
+        List of Dictionaries. Raw or processed structured data.
+    """
+    jc.utils.compatibility(__name__, info.compatible, quiet)
+    jc.utils.input_type_check(data)
+
+    raw_output: List[Dict] = []
+
+    re_interface = re.compile(r'^(?P<name>[a-zA-Z0-9:._-]+)\s+(?P<protocol>([a-zA-Z0-9]+\s)*[a-zA-Z0-9.]+)\s+ESSID:\"(?P<essid>[a-zA-Z0-9:._\s]+)\"')
+    re_mode = re.compile(r'Mode:(?P<mode>\w+)')
+    re_frequency = re.compile(r'Frequency:(?P<frequency>[0-9.]+)\s(?P<frequency_unit>\w+)')
+    re_access_point = re.compile(r'Access Point:\s*(?P<access_point>[0-9A-F:]+)')
+    re_bit_rate = re.compile(r'Bit Rate=(?P<bit_rate>[0-9.]+)\s(?P<bit_rate_unit>[\w\/]+)')
+    re_tx_power= re.compile(r'Tx-Power=(?P<tx_power>[-0-9]+)\s(?P<tx_power_unit>[\w]+)')
+    re_retry_short_limit = re.compile(r'Retry short limit:(?P<retry_short_limit>[0-9\/]+)')
+    re_rts_threshold = re.compile(r'RTS thr:(?P<rts_threshold>(off|on))')
+    re_fragment_threshold = re.compile(r'Fragment thr:(?P<fragment_threshold>(off|on))')
+    re_power_management = re.compile(r'Power Management:(?P<power_management>(off|on))')
+    re_link_quality = re.compile(r'Link Quality=(?P<link_quality>[0-9\/]+)')
+    re_signal_level = re.compile(r'Signal level=(?P<signal_level>[-0-9]+)\s(?P<signal_level_unit>[\w]+)')
+    re_rx_invalid_nwid = re.compile(r'Rx invalid nwid:(?P<rx_invalid_nwid>[-0-9]+)')
+    re_rx_invalid_crypt = re.compile(r'Rx invalid crypt:(?P<rx_invalid_crypt>[-0-9]+)')
+    re_rx_invalid_frag = re.compile(r'Rx invalid frag:(?P<rx_invalid_frag>[-0-9]+)')
+    re_tx_excessive_retries = re.compile(r'Tx excessive retries:(?P<tx_excessive_retries>[-0-9]+)')
+    re_invalid_misc = re.compile(r'Invalid misc:(?P<invalid_misc>[0-9]+)')
+    re_missed_beacon = re.compile(r'Missed beacon:(?P<missed_beacon>[0-9]+)')
+
+    re_all = [
+        re_mode, re_frequency, re_access_point, re_bit_rate, re_tx_power,
+        re_retry_short_limit, re_rts_threshold, re_fragment_threshold, re_power_management,
+        re_link_quality, re_signal_level, re_rx_invalid_nwid, re_rx_invalid_crypt,
+        re_rx_invalid_frag, re_tx_excessive_retries, re_invalid_misc, re_missed_beacon
+    ]
+
+    interface_item = None
+    if jc.utils.has_data(data):
+        for line in filter(None, data.splitlines()):
+
+            # Find new interface lines
+            interface_match = re.search(re_interface, line)
+            if interface_match:
+                if interface_item is not None:
+                    raw_output.append(interface_item)
+
+                interface_item = dict()
+                interface_item.update(interface_match.groupdict())
+                continue
+
+            # we do not have any interface yet continue to search for it --> next line
+            if interface_item is None:
+                continue
+
+            # Filling interface with whatever we can find
+            for re_entry in re_all:
+                match = re.search(re_entry, line)
+                if match:
+                    interface_item.update(match.groupdict())
+
+    if interface_item is not None:
+        raw_output.append(interface_item)
+
+    return raw_output if raw else _process(raw_output)

jc/parsers/jar_manifest.py

@@ -83,6 +83,7 @@ class info():
     author = 'Matt J'
     author_email = 'https://github.com/listuser'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/jobs.py

@@ -99,6 +99,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
     magic_commands = ['jobs']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/jwt.py

@@ -56,6 +56,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['standard', 'string']
 
 
 __version__ = info.version

jc/parsers/kv.py

@@ -1,4 +1,4 @@
-"""jc - JSON Convert `Key/Value` file parser
+"""jc - JSON Convert `Key/Value` file and string parser
 
 Supports files containing simple key/value pairs.
 
@@ -55,11 +55,12 @@ Examples:
 class info():
     """Provides parser metadata (version, author, etc.)"""
     version = '1.2'
-    description = 'Key/Value file parser'
+    description = 'Key/Value file and string parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     details = 'This is a wrapper for the INI parser'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['generic', 'file', 'string']
 
 
 __version__ = info.version

jc/parsers/last.py

@@ -113,6 +113,7 @@ class info():
     details = 'Enhancements by https://github.com/zerolagtime'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
     magic_commands = ['last', 'lastb']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/ls.py

@@ -124,6 +124,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
     magic_commands = ['ls', 'vdir']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/ls_s.py

@@ -82,6 +82,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
+    tags = ['command']
     streaming = True
 
 

jc/parsers/lsblk.py

@@ -281,6 +281,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['lsblk']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/lsmod.py

@@ -132,6 +132,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['lsmod']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/lsof.py

@@ -126,6 +126,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
     magic_commands = ['lsof']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/lspci.py

@@ -129,6 +129,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['lspci']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/lsusb.py

@@ -275,6 +275,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['lsusb']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/m3u.py

@@ -71,6 +71,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/mdadm.py

@@ -235,6 +235,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['mdadm']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/mount.py

@@ -81,6 +81,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'freebsd']
     magic_commands = ['mount']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/mpstat.py

@@ -122,6 +122,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['mpstat']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/mpstat_s.py

@@ -106,6 +106,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
+    tags = ['command']
     streaming = True
 
 

jc/parsers/netstat.py

@@ -361,6 +361,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'freebsd']
     magic_commands = ['netstat']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/nmcli.py

@@ -155,6 +155,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['nmcli']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/ntpq.py

@@ -213,6 +213,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'freebsd']
     magic_commands = ['ntpq']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/openvpn.py

@@ -161,6 +161,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/os_prober.py

@@ -48,6 +48,7 @@ class info():
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux']
     magic_commands = ['os-prober']
+    tags = ['command']
 
 
 __version__ = info.version

jc/parsers/passwd.py

@@ -99,6 +99,7 @@ class info():
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
     compatible = ['linux', 'darwin', 'aix', 'freebsd']
+    tags = ['file']
 
 
 __version__ = info.version

jc/parsers/pbPlist/StrParse.py

@@ -0,0 +1,366 @@
+# Copyright (c) 2016, Samantha Marshall (http://pewpewthespells.com)
+# All rights reserved.
+#
+# https://github.com/samdmarshall/pbPlist
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation and/or
+# other materials provided with the distribution.
+#
+# 3. Neither the name of Samantha Marshall nor the names of its contributors may
+# be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+import string
+
+if sys.version_info >= (3, 0):
+    def unichr(character): # pylint: disable=redefined-builtin
+        return chr(character)
+
+def ConvertNEXTSTEPToUnicode(hex_digits):
+    # taken from http://ftp.unicode.org/Public/MAPPINGS/VENDORS/NEXT/NEXTSTEP.TXT
+    conversion = {
+        "80":	"a0",	# NO-BREAK SPACE
+        "81":	"c0",	# LATIN CAPITAL LETTER A WITH GRAVE
+        "82":	"c1",	# LATIN CAPITAL LETTER A WITH ACUTE
+        "83":	"c2",	# LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+        "84":	"c3",	# LATIN CAPITAL LETTER A WITH TILDE
+        "85":	"c4",	# LATIN CAPITAL LETTER A WITH DIAERESIS
+        "86":	"c5",	# LATIN CAPITAL LETTER A WITH RING
+        "87":	"c7",	# LATIN CAPITAL LETTER C WITH CEDILLA
+        "88":	"c8",	# LATIN CAPITAL LETTER E WITH GRAVE
+        "89":	"c9",	# LATIN CAPITAL LETTER E WITH ACUTE
+        "8a":	"ca",	# LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+        "8b":	"cb",	# LATIN CAPITAL LETTER E WITH DIAERESIS
+        "8c":	"cc",	# LATIN CAPITAL LETTER I WITH GRAVE
+        "8d":	"cd",	# LATIN CAPITAL LETTER I WITH ACUTE
+        "8e":	"ce",	# LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+        "8f":	"cf",	# LATIN CAPITAL LETTER I WITH DIAERESIS
+        "90":	"d0",	# LATIN CAPITAL LETTER ETH
+        "91":	"d1",	# LATIN CAPITAL LETTER N WITH TILDE
+        "92":	"d2",	# LATIN CAPITAL LETTER O WITH GRAVE
+        "93":	"d3",	# LATIN CAPITAL LETTER O WITH ACUTE
+        "94":	"d4",	# LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+        "95":	"d5",	# LATIN CAPITAL LETTER O WITH TILDE
+        "96":	"d6",	# LATIN CAPITAL LETTER O WITH DIAERESIS
+        "97":	"d9",	# LATIN CAPITAL LETTER U WITH GRAVE
+        "98":	"da",	# LATIN CAPITAL LETTER U WITH ACUTE
+        "99":	"db",	# LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+        "9a":	"dc",	# LATIN CAPITAL LETTER U WITH DIAERESIS
+        "9b":	"dd",	# LATIN CAPITAL LETTER Y WITH ACUTE
+        "9c":	"de",	# LATIN CAPITAL LETTER THORN
+        "9d":	"b5",	# MICRO SIGN
+        "9e":	"d7",	# MULTIPLICATION SIGN
+        "9f":	"f7",	# DIVISION SIGN
+        "a0":	"a9",	# COPYRIGHT SIGN
+        "a1":	"a1",	# INVERTED EXCLAMATION MARK
+        "a2":	"a2",	# CENT SIGN
+        "a3":	"a3",	# POUND SIGN
+        "a4":	"44",	# FRACTION SLASH
+        "a5":	"a5",	# YEN SIGN
+        "a6":	"92",	# LATIN SMALL LETTER F WITH HOOK
+        "a7":	"a7",	# SECTION SIGN
+        "a8":	"a4",	# CURRENCY SIGN
+        "a9":	"19",	# RIGHT SINGLE QUOTATION MARK
+        "aa":	"1c",	# LEFT DOUBLE QUOTATION MARK
+        "ab":	"ab",	# LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
+        "ac":	"39",	# SINGLE LEFT-POINTING ANGLE QUOTATION MARK
+        "ad":	"3a",	# SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
+        "ae":	"01",	# LATIN SMALL LIGATURE FI
+        "af":	"02",	# LATIN SMALL LIGATURE FL
+        "b0":	"ae",	# REGISTERED SIGN
+        "b1":	"13",	# EN DASH
+        "b2":	"20",	# DAGGER
+        "b3":	"21",	# DOUBLE DAGGER
+        "b4":	"b7",	# MIDDLE DOT
+        "b5":	"a6",	# BROKEN BAR
+        "b6":	"b6",	# PILCROW SIGN
+        "b7":	"22",	# BULLET
+        "b8":	"1a",	# SINGLE LOW-9 QUOTATION MARK
+        "b9":	"1e",	# DOUBLE LOW-9 QUOTATION MARK
+        "ba":	"1d",	# RIGHT DOUBLE QUOTATION MARK
+        "bb":	"bb",	# RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
+        "bc":	"26",	# HORIZONTAL ELLIPSIS
+        "bd":	"30",	# PER MILLE SIGN
+        "be":	"ac",	# NOT SIGN
+        "bf":	"bf",	# INVERTED QUESTION MARK
+        "c0":	"b9",	# SUPERSCRIPT ONE
+        "c1":	"cb",	# MODIFIER LETTER GRAVE ACCENT
+        "c2":	"b4",	# ACUTE ACCENT
+        "c3":	"c6",	# MODIFIER LETTER CIRCUMFLEX ACCENT
+        "c4":	"dc",	# SMALL TILDE
+        "c5":	"af",	# MACRON
+        "c6":	"d8",	# BREVE
+        "c7":	"d9",	# DOT ABOVE
+        "c8":	"a8",	# DIAERESIS
+        "c9":	"b2",	# SUPERSCRIPT TWO
+        "ca":	"da",	# RING ABOVE
+        "cb":	"b8",	# CEDILLA
+        "cc":	"b3",	# SUPERSCRIPT THREE
+        "cd":	"dd",	# DOUBLE ACUTE ACCENT
+        "ce":	"db",	# OGONEK
+        "cf":	"c7",	# CARON
+        "d0":	"14",	# EM DASH
+        "d1":	"b1",	# PLUS-MINUS SIGN
+        "d2":	"bc",	# VULGAR FRACTION ONE QUARTER
+        "d3":	"bd",	# VULGAR FRACTION ONE HALF
+        "d4":	"be",	# VULGAR FRACTION THREE QUARTERS
+        "d5":	"e0",	# LATIN SMALL LETTER A WITH GRAVE
+        "d6":	"e1",	# LATIN SMALL LETTER A WITH ACUTE
+        "d7":	"e2",	# LATIN SMALL LETTER A WITH CIRCUMFLEX
+        "d8":	"e3",	# LATIN SMALL LETTER A WITH TILDE
+        "d9":	"e4",	# LATIN SMALL LETTER A WITH DIAERESIS
+        "da":	"e5",	# LATIN SMALL LETTER A WITH RING ABOVE
+        "db":	"e7",	# LATIN SMALL LETTER C WITH CEDILLA
+        "dc":	"e8",	# LATIN SMALL LETTER E WITH GRAVE
+        "dd":	"e9",	# LATIN SMALL LETTER E WITH ACUTE
+        "de":	"ea",	# LATIN SMALL LETTER E WITH CIRCUMFLEX
+        "df":	"eb",	# LATIN SMALL LETTER E WITH DIAERESIS
+        "e0":	"ec",	# LATIN SMALL LETTER I WITH GRAVE
+        "e1":	"c6",	# LATIN CAPITAL LETTER AE
+        "e2":	"ed",	# LATIN SMALL LETTER I WITH ACUTE
+        "e3":	"aa",	# FEMININE ORDINAL INDICATOR
+        "e4":	"ee",	# LATIN SMALL LETTER I WITH CIRCUMFLEX
+        "e5":	"ef",	# LATIN SMALL LETTER I WITH DIAERESIS
+        "e6":	"f0",	# LATIN SMALL LETTER ETH
+        "e7":	"f1",	# LATIN SMALL LETTER N WITH TILDE
+        "e8":	"41",	# LATIN CAPITAL LETTER L WITH STROKE
+        "e9":	"d8",	# LATIN CAPITAL LETTER O WITH STROKE
+        "ea":	"52",	# LATIN CAPITAL LIGATURE OE
+        "eb":	"ba",	# MASCULINE ORDINAL INDICATOR
+        "ec":	"f2",	# LATIN SMALL LETTER O WITH GRAVE
+        "ed":	"f3",	# LATIN SMALL LETTER O WITH ACUTE
+        "ee":	"f4",	# LATIN SMALL LETTER O WITH CIRCUMFLEX
+        "ef":	"f5",	# LATIN SMALL LETTER O WITH TILDE
+        "f0":	"f6",	# LATIN SMALL LETTER O WITH DIAERESIS
+        "f1":	"e6",	# LATIN SMALL LETTER AE
+        "f2":	"f9",	# LATIN SMALL LETTER U WITH GRAVE
+        "f3":	"fa",	# LATIN SMALL LETTER U WITH ACUTE
+        "f4":	"fb",	# LATIN SMALL LETTER U WITH CIRCUMFLEX
+        "f5":	"31",	# LATIN SMALL LETTER DOTLESS I
+        "f6":	"fc",	# LATIN SMALL LETTER U WITH DIAERESIS
+        "f7":	"fd",	# LATIN SMALL LETTER Y WITH ACUTE
+        "f8":	"42",	# LATIN SMALL LETTER L WITH STROKE
+        "f9":	"f8",	# LATIN SMALL LETTER O WITH STROKE
+        "fa":	"53",	# LATIN SMALL LIGATURE OE
+        "fb":	"df",	# LATIN SMALL LETTER SHARP S
+        "fc":	"fe",	# LATIN SMALL LETTER THORN
+        "fd":	"ff",	# LATIN SMALL LETTER Y WITH DIAERESIS
+        "fe":	"fd",	# .notdef, REPLACEMENT CHARACTER
+        "ff":	"fd",	# .notdef, REPLACEMENT CHARACTER
+    }
+    return conversion[hex_digits]
+
+def IsOctalNumber(character):
+    oct_digits = set(string.octdigits)
+    return set(character).issubset(oct_digits)
+
+def IsHexNumber(character):
+    hex_digits = set(string.hexdigits)
+    return set(character).issubset(hex_digits)
+
+def SanitizeCharacter(character):
+    char = character
+    escaped_characters = {
+        '\a': '\\a',
+        '\b': '\\b',
+        '\f': '\\f',
+        '\n': '\\n',
+        '\r': '\\r',
+        '\t': '\\t',
+        '\v': '\\v',
+        '\"': '\\"',
+    }
+    if character in escaped_characters.keys():
+        char = escaped_characters[character]
+    return char
+
+# http://www.opensource.apple.com/source/CF/CF-744.19/CFOldStylePList.c See `getSlashedChar()`
+def UnQuotifyString(string_data, start_index, end_index): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
+    formatted_string = ''
+    extracted_string = string_data[start_index:end_index]
+    string_length = len(extracted_string)
+    all_cases = ['0', '1', '2', '3', '4', '5', '6', '7', 'a', 'b', 'f', 'n', 'r', 't', 'v', '\"', '\n', 'U']
+    index = 0
+    while index < string_length: # pylint: disable=too-many-nested-blocks
+        current_char = extracted_string[index]
+        if current_char == '\\':
+            next_char = extracted_string[index+1]
+            if next_char in all_cases:
+                index += 1
+                if next_char == 'a':
+                    formatted_string += '\a'
+                if next_char == 'b':
+                    formatted_string += '\b'
+                if next_char == 'f':
+                    formatted_string += '\f'
+                if next_char == 'n':
+                    formatted_string += '\n'
+                if next_char == 'r':
+                    formatted_string += '\r'
+                if next_char == 't':
+                    formatted_string += '\t'
+                if next_char == 'v':
+                    formatted_string += '\v'
+                if next_char == '"':
+                    formatted_string += '\"'
+                if next_char == '\n':
+                    formatted_string += '\n'
+                if next_char == 'U':
+                    starting_index = index + 1
+                    ending_index = starting_index + 4
+                    unicode_numbers = extracted_string[starting_index:ending_index]
+                    for number in unicode_numbers:
+                        index += 1
+                        if IsHexNumber(number) is False: # pragma: no cover
+                            message = 'Invalid unicode sequence on line '+str(LineNumberForIndex(string_data, start_index+index))
+                            raise Exception(message)
+                    formatted_string += unichr(int(unicode_numbers, 16))
+                if IsOctalNumber(next_char) is True: # https://twitter.com/Catfish_Man/status/658014170055507968
+                    starting_index = index
+                    ending_index = starting_index + 1
+                    for oct_index in range(3):
+                        test_index = starting_index + oct_index
+                        test_oct = extracted_string[test_index]
+                        if IsOctalNumber(test_oct) is True:
+                            ending_index += 1
+                    octal_numbers = extracted_string[starting_index:ending_index]
+                    hex_number = int(octal_numbers, 8)
+                    hex_str = format(hex_number, 'x')
+                    if hex_number >= 0x80:
+                        hex_str = ConvertNEXTSTEPToUnicode(hex_str)
+                    formatted_string += unichr(int('00'+hex_str, 16))
+            else:
+                formatted_string += current_char
+                index += 1
+                formatted_string += next_char
+        else:
+            formatted_string += current_char
+        index += 1
+    return formatted_string
+
+def LineNumberForIndex(string_data, current_index):
+    line_number = 1
+    index = 0
+    string_length = len(string_data)
+    while (index < current_index) and (index < string_length):
+        current_char = string_data[index]
+        if IsNewline(current_char) is True:
+            line_number += 1
+        index += 1
+    return line_number
+
+def IsValidUnquotedStringCharacter(character):
+    if len(character) == 1:
+        valid_characters = set(string.ascii_letters+string.digits+'_$/:.-')
+        return set(character).issubset(valid_characters)
+    else: # pragma: no cover
+        message = 'The function "IsValidUnquotedStringCharacter()" can only take single characters!'
+        raise ValueError(message)
+
+def IsSpecialWhitespace(character):
+    value = ord(character)
+    result = (value >= 9 and value <= 13) # tab, newline, vt, form feed, carriage return
+    return result
+
+def IsUnicodeSeparator(character):
+    value = ord(character)
+    result = (value == 8232 or value == 8233)
+    return result
+
+def IsRegularWhitespace(character):
+    value = ord(character)
+    result = (value == 32 or IsUnicodeSeparator(character)) # space and Unicode line sep, para sep
+    return result
+
+def IsDataFormattingWhitespace(character):
+    value = ord(character)
+    result = (IsNewline(character) or IsRegularWhitespace(character) or value == 9)
+    return result
+
+def IsNewline(character):
+    value = ord(character)
+    result = (value == 13 or value == 10)
+    return result
+
+def IsEndOfLine(character):
+    result = (IsNewline(character) or IsUnicodeSeparator(character))
+    return result
+
+def IndexOfNextNonSpace(string_data, current_index): # pylint: disable=too-many-branches,too-many-statements
+    successful = False
+    found_index = current_index
+    string_length = len(string_data)
+    annotation_string = ''
+    while found_index < string_length: # pylint: disable=too-many-nested-blocks
+        current_char = string_data[found_index]
+        if IsSpecialWhitespace(current_char) is True:
+            found_index += 1
+            continue
+        if IsRegularWhitespace(current_char) is True:
+            found_index += 1
+            continue
+        if current_char == '/':
+            next_index = found_index + 1
+            if next_index >= string_length:
+                successful = True
+                break
+            else:
+                next_character = string_data[next_index]
+                if next_character == '/': # found a line comment "//"
+                    found_index += 1
+                    next_index = found_index
+                    first_pass = True
+                    while next_index < string_length:
+                        test_char = string_data[next_index]
+                        if IsEndOfLine(test_char) is True:
+                            break
+                        else:
+                            if first_pass is False:
+                                annotation_string += test_char
+                            else:
+                                first_pass = False
+                        next_index += 1
+                    found_index = next_index
+                elif next_character == '*': # found a block comment "/* ... */"
+                    found_index += 1
+                    next_index = found_index
+                    first_pass = True
+                    while next_index < string_length:
+                        test_char = string_data[next_index]
+                        if test_char == '*' and (next_index+1 < string_length) and string_data[next_index+1] == '/':
+                            next_index += 2
+                            break
+                        else:
+                            if first_pass != True:
+                                annotation_string += test_char
+                            else:
+                                first_pass = False
+                        next_index += 1
+                    found_index = next_index
+                else:
+                    successful = True
+                    break
+        else:
+            successful = True
+            break
+    result = (successful, found_index, annotation_string)
+    return result

jc/parsers/pbPlist/Switch.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2016, Samantha Marshall (http://pewpewthespells.com)
+# All rights reserved.
+#
+# https://github.com/samdmarshall/pylocalizer
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation and/or
+# other materials provided with the distribution.
+#
+# 3. Neither the name of Samantha Marshall nor the names of its contributors may
+# be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Original code taken from http://code.activestate.com/recipes/410692/
+
+class Switch(object):
+    def __init__(self, value):
+        self.value = value
+        self.fall = False
+
+    def __iter__(self):
+        """Return the match method once, then stop"""
+        yield self.match
+
+    def match(self, *args):
+        """Indicate whether or not to enter a case suite"""
+        result = False
+        if self.fall or not args:
+            result = True
+        elif self.value in args: # changed for v1.5, see below
+            self.fall = True
+            result = True
+        return result

jc/parsers/pbPlist/__init__.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2016, Samantha Marshall (http://pewpewthespells.com)
+# All rights reserved.
+#
+# https://github.com/samdmarshall/pbPlist
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation and/or
+# other materials provided with the distribution.
+#
+# 3. Neither the name of Samantha Marshall nor the names of its contributors may
+# be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from . import pbPlist
+__version__ = '1.0.4'

jc/parsers/pbPlist/pbItem.py

@@ -0,0 +1,262 @@
+# Copyright (c) 2016, Samantha Marshall (http://pewpewthespells.com)
+# All rights reserved.
+#
+# https://github.com/samdmarshall/pbPlist
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation and/or
+# other materials provided with the distribution.
+#
+# 3. Neither the name of Samantha Marshall nor the names of its contributors may
+# be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from . import StrParse
+
+def PushIndent(indent_level):
+    return indent_level + 1
+
+def PopIndent(indent_level):
+    return indent_level - 1
+
+def WriteIndent(level=0):
+    output = ''
+    for _index in range(level):
+        output += '\t'
+    return output
+
+def WriteNewline(level=0, indent=True):
+    output = '\n'
+    if indent:
+        output += WriteIndent(level)
+    return output
+
+class pbItem(object):
+    def __init__(self, value=None, type_name=None, annotation=None):
+        if value != None and type_name != None:
+            self.value = value
+            if type_name not in KnownTypes.keys(): # pragma: no cover
+                message = 'Unknown type "'+type_name+'" passed to '+self.__class__.__name__+' initializer!'
+                raise TypeError(message)
+            self.type_name = type_name
+            self.annotation = annotation
+        else: # pragma: no cover
+            message = 'The class "'+self.__class__.__name__+'" must be initialized with a non-None value'
+            raise ValueError(message)
+
+    def __eq__(self, other):
+        is_equal = False
+        if isinstance(other, pbItem):
+            other = other.value
+        if type(other) is type(self.value):
+            is_equal = self.value.__eq__(other)
+        return is_equal
+
+    def __hash__(self):
+        return self.value.__hash__()
+
+    def __repr__(self):
+        return self.value.__repr__()
+
+    def __iter__(self):
+        return self.value.__iter__()
+
+    def __getattr__(self, attrib):
+        return self.value.__getattr__(attrib)
+
+    def __str__(self):
+        return self.writeStringRep(0, False)[0]
+
+    def __getitem__(self, key):
+        return self.value.__getitem__(key)
+
+    def __setitem__(self, key, value):
+        self.value.__setitem__(key, value)
+
+    def __len__(self):
+        return self.value.__len__()
+
+    def __contains__(self, item):
+        return self.value.__contains__(item)
+
+    def __get__(self, obj, objtype):
+        return self.value.__get__(obj, objtype)
+
+    def writeStringRep(self, indent_level=0, pretty=True):
+        return self.writeString(indent_level, pretty)
+
+    def writeString(self, indent_level=0, pretty=True): # pylint: disable=no-self-use,unused-variable,unused-argument ; # pragma: no cover
+        message = 'This is a base class, it cannot write!'
+        raise Exception(message)
+
+    def nativeType(self):
+        return self.value
+
+    def writeAnnotation(self):
+        output_string = ''
+        if self.annotation != None and len(self.annotation) > 0:
+            output_string += ' '
+            output_string += '/*'
+            output_string += self.annotation
+            output_string += '*/'
+        return output_string
+
+class pbString(pbItem):
+    def writeString(self, indent_level=0, pretty=True):
+        string_string = ''
+        string_string += self.value
+        if pretty is True:
+            string_string += self.writeAnnotation()
+        return (string_string, indent_level)
+
+class pbQString(pbItem):
+    def writeStringRep(self, indent_level=0, pretty=True):
+        qstring_string = ''
+        for character in self.value:
+            qstring_string += StrParse.SanitizeCharacter(character)
+        return (qstring_string, indent_level)
+
+    def writeString(self, indent_level=0, pretty=True):
+        qstring_string = ''
+        qstring_string += '"'
+        string_rep, indent_level = self.writeStringRep(indent_level, pretty)
+        qstring_string += string_rep
+        qstring_string += '"'
+        if pretty is True:
+            qstring_string += self.writeAnnotation()
+        return (qstring_string, indent_level)
+
+class pbData(pbItem):
+    def writeString(self, indent_level=0, pretty=True):
+        data_string = ''
+        indent_level = PushIndent(indent_level)
+        data_string += '<'
+        grouping_byte_counter = 0
+        grouping_line_counter = 0
+        for hex_byte in map(ord, self.value.decode()):
+            data_string += format(hex_byte, 'x')
+            grouping_byte_counter += 1
+            # write a space every 4th byte
+            if grouping_byte_counter == 4:
+                data_string += ' '
+                grouping_byte_counter = 0
+                grouping_line_counter += 1
+            # write a newline every 4th grouping of bytes
+            if grouping_line_counter == 4:
+                data_string += WriteNewline(indent_level)
+                data_string += ' ' # indent an additional space to make the byte groupings line up
+                grouping_line_counter = 0
+        data_string += '>'
+        if pretty is True:
+            data_string += self.writeAnnotation()
+        indent_level = PopIndent(indent_level)
+        return (data_string, indent_level)
+
+class pbDictionary(pbItem):
+    def nativeType(self):
+        new_value = dict()
+        for key in self.keys():
+            value = self[key]
+            new_value[str(key)] = value.nativeType()
+        return new_value
+    def writeString(self, indent_level=0, pretty=True):
+        dictionary_string = ''
+        dictionary_string += '{'
+        has_sorted_keys, keys_array = self.value.sortedKeys()
+        dictionary_string += WriteNewline(indent_level, not has_sorted_keys)
+        indent_level = PushIndent(indent_level)
+        previous_value_type = None
+        if len(keys_array) == 0:
+            indent_level = PopIndent(indent_level)
+        else:
+            if not has_sorted_keys:
+                dictionary_string += '\t'
+            for key in keys_array:
+                if has_sorted_keys:
+                    current_value_type = str(self.value[key]['isa'])
+                    if previous_value_type != current_value_type:
+                        if previous_value_type != None:
+                            dictionary_string += '/* End '+previous_value_type+' section */'
+                            dictionary_string += WriteNewline(indent_level, False)
+                        previous_value_type = current_value_type
+                        dictionary_string += '\n/* Begin '+current_value_type+' section */'
+                        dictionary_string += WriteNewline(indent_level)
+                    else:
+                        dictionary_string += WriteIndent(indent_level)
+                write_string, indent_level = key.writeString(indent_level, pretty)
+                dictionary_string += write_string
+                dictionary_string += ' = '
+                write_string, indent_level = self.value[key].writeString(indent_level, pretty)
+                dictionary_string += write_string
+                dictionary_string += ';'
+                should_indent = True
+                is_last_key = (key == keys_array[-1])
+                if is_last_key:
+                    if has_sorted_keys:
+                        dictionary_string += WriteNewline(indent_level, False)
+                        dictionary_string += '/* End '+previous_value_type+' section */'
+                    indent_level = PopIndent(indent_level)
+                else:
+                    if has_sorted_keys:
+                        should_indent = False
+                dictionary_string += WriteNewline(indent_level, should_indent)
+        dictionary_string += '}'
+        return (dictionary_string, indent_level)
+
+class pbArray(pbItem):
+    def nativeType(self):
+        new_value = [item.nativeType() for item in self.value]
+        return new_value
+    def writeString(self, indent_level=0, pretty=True):
+        array_string = ''
+        array_string += '('
+        array_string += WriteNewline(indent_level)
+        indent_level = PushIndent(indent_level)
+        values_array = list(self.value)
+        if len(values_array) == 0:
+            indent_level = PopIndent(indent_level)
+        else:
+            array_string += '\t'
+            for value in values_array:
+                write_string, indent_level = value.writeString(indent_level, pretty)
+                array_string += write_string
+                if value != values_array[-1]:
+                    array_string += ','
+                else:
+                    indent_level = PopIndent(indent_level)
+                array_string += WriteNewline(indent_level)
+        array_string += ')'
+        return (array_string, indent_level)
+
+KnownTypes = {
+    'string': pbString,
+    'qstring': pbQString,
+    'data': pbData,
+    'dictionary': pbDictionary,
+    'array': pbArray,
+}
+
+def pbItemResolver(obj, type_name):
+    initializer = KnownTypes[type_name]
+    if initializer:
+        return initializer(obj, type_name)
+    else: # pragma: no cover
+        message = 'Unknown type "'+type_name+'" passed to pbItemResolver!'
+        raise TypeError(message)

jc/parsers/pbPlist/pbParser.py

@@ -0,0 +1,291 @@
+# Copyright (c) 2016, Samantha Marshall (http://pewpewthespells.com)
+# All rights reserved.
+#
+# https://github.com/samdmarshall/pbPlist
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation and/or
+# other materials provided with the distribution.
+#
+# 3. Neither the name of Samantha Marshall nor the names of its contributors may
+# be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__            import print_function
+import os
+import sys
+import codecs
+from .                     import StrParse
+from .                     import pbRoot
+from .                     import pbItem
+from .Switch               import Switch
+
+def GetFileEncoding(path):
+    encoding = 'utf-8-sig'
+
+    size = os.path.getsize(path)
+    if size > 2:
+        file_descriptor = OpenFile(path)
+        first_two_bytes = file_descriptor.read(2)
+        file_descriptor.close()
+
+        for case in Switch(first_two_bytes):
+            if case(codecs.BOM_UTF16):
+                encoding = 'utf-16'
+                break
+            if case(codecs.BOM_UTF16_LE):
+                encoding = 'utf-16-le'
+                break
+            if case(codecs.BOM_UTF16_BE):
+                encoding = 'utf-16-be'
+                break
+            if case():
+                break # pragma: no cover
+
+    return encoding
+
+def OpenFileWithEncoding(file_path, encoding):
+    return codecs.open(file_path, 'r', encoding=encoding, errors='ignore')
+
+if sys.version_info < (3, 0):
+    def OpenFile(file_path):
+        return open(file_path, 'rb')
+else:
+    def OpenFile(file_path):
+        return open(file_path, 'br')
+
+class PBParser(object):
+
+    def __init__(self, file_path=None):
+        self.index = 0
+        self.string_encoding = None
+        self.file_path = file_path
+        self.file_type = None
+        try:
+            encoding = GetFileEncoding(self.file_path)
+            file_descriptor = OpenFileWithEncoding(self.file_path, encoding)
+            self.data = file_descriptor.read()
+            if self.file_path.endswith('.strings'):
+                self.data = '{'+self.data+'}'
+            file_descriptor.close()
+        except IOError as exception: # pragma: no cover
+            print('I/O error({0}): {1}'.format(exception.errno, exception.strerror))
+        except: # pragma: no cover
+            print('Unexpected error:'+str(sys.exc_info()[0]))
+            raise
+
+    def read(self):
+        parsed_plist = None
+        prefix = self.data[0:6]
+        for case in Switch(prefix):
+            if case('bplist'):
+                self.file_type = 'binary'
+                import biplist
+                parsed_plist = biplist.readPlist(self.file_path)
+                break
+            if case('<?xml '):
+                self.file_type = 'xml'
+                import plistlib
+                parsed_plist = plistlib.readPlist(self.file_path)
+                break
+            if case():
+                self.file_type = 'ascii'
+                # test for encoding hint
+                if self.data[0:2] == '//':
+                    # this is to try to see if we can locate the desired string encoding of the file
+                    import re
+                    result = re.search('^// !\$\*(.+?)\*\$!', self.data) # pylint: disable=anomalous-backslash-in-string
+                    if result:
+                        self.string_encoding = result.group(1)
+                #now return the parse
+                parsed_plist = self.__readTest(True)
+                break
+        return parsed_plist
+
+    def __readTest(self, requires_object=True):
+        read_result = None
+        # can we parse this?
+        can_parse, self.index, _annotation = StrParse.IndexOfNextNonSpace(self.data, self.index)
+        # we can ignore the annotation value here
+        if not can_parse:
+            if self.index != len(self.data):
+                if requires_object is True: # pragma: no cover
+                    message = 'Invalid plist file!'
+                    raise Exception(message)
+        else:
+            read_result = self.__parse(requires_object)
+        return read_result
+
+    def __parse(self, requires_object=True):
+        parsed_item = None
+        starting_character = self.data[self.index]
+        for case in Switch(starting_character):
+            if case('{'):
+                # parse dictionary
+                parsed_item = pbItem.pbItemResolver(self.__parseDict(), 'dictionary') # pylint: disable=redefined-variable-type
+                break
+            if case('('):
+                # parse array
+                parsed_item = pbItem.pbItemResolver(self.__parseArray(), 'array') # pylint: disable=redefined-variable-type
+                break
+            if case('<'):
+                # parse data
+                parsed_item = pbItem.pbItemResolver(self.__parseData(), 'data') # pylint: disable=redefined-variable-type
+                break
+            if case('\''):
+                pass
+            if case('\"'):
+                # parse quoted string
+                parsed_item = pbItem.pbItemResolver(self.__parseQuotedString(), 'qstring') # pylint: disable=redefined-variable-type
+                break
+            if case():
+                if StrParse.IsValidUnquotedStringCharacter(starting_character) is True:
+                    # parse unquoted string
+                    parsed_item = pbItem.pbItemResolver(self.__parseUnquotedString(), 'string') # pylint: disable=redefined-variable-type
+                else:
+                    if requires_object is True: # pragma: no cover
+                        message = 'Unexpected character "0x%s" at line %i of file %s' % (str(format(ord(starting_character), 'x')), StrParse.LineNumberForIndex(self.data, self.index), self.file_path)
+                        raise Exception(message)
+        return parsed_item
+
+    def __parseUnquotedString(self):
+        string_length = len(self.data)
+        start_index = self.index
+        while self.index < string_length:
+            current_char = self.data[self.index]
+            if StrParse.IsValidUnquotedStringCharacter(current_char) is True:
+                self.index += 1
+            else:
+                break
+        if start_index != self.index:
+            return self.data[start_index:self.index]
+        else: # pragma: no cover
+            message = 'Unexpected EOF in file %s' % self.file_path
+            raise Exception(message)
+
+    def __parseQuotedString(self):
+        quote = self.data[self.index]
+        string_length = len(self.data)
+        self.index += 1 # skip over the first quote
+        start_index = self.index
+        while self.index < string_length:
+            current_char = self.data[self.index]
+            if current_char == quote:
+                break
+            if current_char == '\\':
+                self.index += 2
+            else:
+                self.index += 1
+        if self.index >= string_length: # pragma: no cover
+            message = 'Unterminated quoted string starting on line %s in file %s' % (str(StrParse.LineNumberForIndex(self.data, start_index)), self.file_path)
+            raise Exception(message)
+        else:
+            string_without_quotes = StrParse.UnQuotifyString(self.data, start_index, self.index)
+            self.index += 1 # advance past quote character
+            return string_without_quotes
+
+    def __parseData(self):
+        string_length = len(self.data)
+        self.index += 1 # skip over "<"
+        start_index = self.index
+        end_index = 0
+        byte_stream = ''
+        while self.index < string_length:
+            current_char = self.data[self.index]
+            if current_char == '>':
+                self.index += 1 # move past the ">"
+                end_index = self.index
+                break
+            if StrParse.IsHexNumber(current_char) is True:
+                byte_stream += current_char
+            else:
+                if not StrParse.IsDataFormattingWhitespace(current_char): # pragma: no cover
+                    message = 'Malformed data byte group (invalid hex) at line %s in file %s' % (str(StrParse.LineNumberForIndex(self.data, start_index)), self.file_path)
+                    raise Exception(message)
+            self.index += 1
+        if (len(byte_stream) % 2) == 1: # pragma: no cover
+            message = 'Malformed data byte group (uneven length) at line %s in file %s' % (str(StrParse.LineNumberForIndex(self.data, start_index)), self.file_path)
+            raise Exception(message)
+        if end_index == 0: # pragma: no cover
+            message = 'Expected terminating >" for data at line %s in file %s' % (str(StrParse.LineNumberForIndex(self.data, start_index)), self.file_path)
+            raise Exception(message)
+        data_object = bytearray.fromhex(byte_stream)
+        return data_object
+
+    def __parseArray(self):
+        array_objects = list()
+        self.index += 1  # move past the "("
+        start_index = self.index
+        new_object = self.__readTest(False)
+        while new_object is not None:
+            can_parse, self.index, new_object.annotation = StrParse.IndexOfNextNonSpace(self.data, self.index)
+            _can_parse = can_parse # pylint: disable=unused-variable
+            array_objects.append(new_object)
+            current_char = self.data[self.index]
+            if current_char == ',':
+                self.index += 1
+            new_object = self.__readTest(False)
+        current_char = self.data[self.index]
+        if current_char != ')': # pragma: no cover
+            message = 'Expected terminating ")" for array at line %s in file %s' % (str(StrParse.LineNumberForIndex(self.data, start_index)), self.file_path)
+            raise Exception(message)
+        self.index += 1 # skip over ending ")"
+        return array_objects
+
+    def __parseDict(self):
+        dictionary = pbRoot.pbRoot()
+        self.index += 1 # move past the "{"
+        start_index = self.index
+        new_object = self.__readTest(False)
+        while new_object is not None:
+            can_parse, self.index, new_object.annotation = StrParse.IndexOfNextNonSpace(self.data, self.index)
+            _can_parse = can_parse # pylint: disable=unused-variable
+            key_object = new_object
+            current_char = self.data[self.index]
+            value_object = None
+            for case in Switch(current_char):
+                if case('='):
+                    self.index += 1
+                    value_object = self.__readTest(True)
+                    break
+                if case(';'):
+                    # this is for strings files where the key and the value may be the same thing
+                    self.index += 1
+                    value_object = pbItem.pbItemResolver(new_object.value, new_object.type_name)
+                    value_object.annotation = new_object.annotation
+                    break
+                if case(): # pragma: no cover
+                    message = 'Missing ";" or "=" on line %s in file %s' % (str(StrParse.LineNumberForIndex(self.data, start_index)), self.file_path)
+                    raise Exception(message)
+            can_parse, self.index, annotation = StrParse.IndexOfNextNonSpace(self.data, self.index)
+            _can_parse = can_parse # pylint: disable=unused-variable
+            if value_object.annotation is None: # this is to prevent losing the annotation of the key when parsing strings dicts
+                value_object.annotation = annotation
+            dictionary[key_object] = value_object
+            current_char = self.data[self.index]
+            if current_char == ';':
+                self.index += 1 # advancing to the next key
+            new_object = self.__readTest(False)
+        current_char = self.data[self.index]
+        if current_char != '}': # pragma: no cover
+            message = 'Expected terminating "}" for dictionary at line %s in file %s' % (str(StrParse.LineNumberForIndex(self.data, start_index)), self.file_path)
+            raise Exception(message)
+        self.index += 1 # skip over ending "}"
+        return dictionary

jc/parsers/pbPlist/pbPlist.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2016, Samantha Marshall (http://pewpewthespells.com)
+# All rights reserved.
+#
+# https://github.com/samdmarshall/pbPlist
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation and/or
+# other materials provided with the distribution.
+#
+# 3. Neither the name of Samantha Marshall nor the names of its contributors may
+# be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+from .pbParser     import PBParser
+from .pbSerializer import PBSerializer
+
+class PBPlist(object):
+
+    def __init__(self, file_path):
+        self.root = None
+        if self.__checkFile(file_path) is True:
+            self.file_path = file_path
+            parser = PBParser(self.file_path)
+            self.root = parser.read()
+            self.string_encoding = parser.string_encoding
+            self.file_type = parser.file_type
+
+    def write(self, file_path=None):
+        if file_path is None:
+            file_path = self.file_path
+        serializer = PBSerializer(file_path, self.string_encoding, self.file_type)
+        serializer.write(self.root)
+
+    def __checkFile(self, file_path):
+        can_access_file = os.path.exists(file_path)
+        if can_access_file is True:
+            self.file_path = file_path
+        return can_access_file