From 029b5abcac1f2253773d9eaf52eb2d23f035d7de Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 8 Jan 2023 19:46:13 -0800 Subject: [PATCH] add ini-dup parser --- README.md | 39 +++--- completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 3 +- docs/parsers/ini.md | 40 +++--- jc/lib.py | 1 + jc/parsers/ini.py | 40 +++--- jc/parsers/ini_dup.py | 224 ++++++++++++++++++++++++++++++ man/jc.1 | 5 + templates/readme_template | 38 +++-- 9 files changed, 314 insertions(+), 78 deletions(-) create mode 100644 jc/parsers/ini_dup.py diff --git a/README.md b/README.md index 9d76e358..e397cb46 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,7 @@ option. | ` --id` | `id` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/id) | | ` --ifconfig` | `ifconfig` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ifconfig) | | ` --ini` | INI file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini) | +| ` --ini-dup` | INI with duplicate key file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ini_dup) | | ` --iostat` | `iostat` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat) | | ` --iostat-s` | `iostat` command streaming parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/iostat_s) | | ` --ip-address` | IPv4 and IPv6 Address string parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ip_address) | @@ -757,39 +758,33 @@ ifconfig | jc -p --ifconfig # or: jc -p ifconfig cat example.ini ``` ``` -[DEFAULT] -ServerAliveInterval = 45 -Compression = yes -CompressionLevel = 9 -ForwardX11 = yes +foo = fiz +bar = buz -[bitbucket.org] -User = hg +[section1] +fruit = apple +color = blue -[topsecret.server.com] -Port = 50022 -ForwardX11 = no +[section2] +fruit = pear +color = green ``` ```bash cat example.ini | jc -p --ini ``` ```json { - "DEFAULT": { - "ServerAliveInterval": "45", - "Compression": "yes", - "CompressionLevel": "9", - "ForwardX11": "yes" + "foo": "fiz", + "bar": "buz", + "section1": { + "fruit": "apple", + "color": "blue" }, - "bitbucket.org": { - "User": "hg" - }, - "topsecret.server.com": { - "Port": "50022", - "ForwardX11": "no" + "section2": { + "fruit": "pear", + "color": "green" } } - ``` ### ls ```bash diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index abd9ee23..f1060a83 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -4,7 +4,7 @@ _jc() jc_about_options jc_about_mod_options jc_help_options jc_special_options jc_commands=(acpi airport arp blkid cbt chage cksum crontab date df dig dmidecode dpkg du env file findmnt finger free git gpg hciconfig id ifconfig iostat iptables iw iwconfig jobs last lastb ls lsblk lsmod lsof lspci lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq os-prober pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss sshd stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 udevadm ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index acde6b0f..d9d1addc 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -102,7 +102,7 @@ _jc() { 'xrandr:run "xrandr" command with magic syntax.' 'zipinfo:run "zipinfo" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cbt --cef --cef-s --chage --cksum --clf --clf-s --crontab --crontab-u --csv --csv-s --date --datetime-iso --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --findmnt --finger --free --fstab --git-log --git-log-s --git-ls-remote --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --ini-dup --iostat --iostat-s --ip-address --iptables --iw-scan --iwconfig --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lspci --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --openvpn --os-prober --passwd --pci-ids --pgpass --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --proc-buddyinfo --proc-consoles --proc-cpuinfo --proc-crypto --proc-devices --proc-diskstats --proc-filesystems --proc-interrupts --proc-iomem --proc-ioports --proc-loadavg --proc-locks --proc-meminfo --proc-modules --proc-mtrr --proc-pagetypeinfo --proc-partitions --proc-slabinfo --proc-softirqs --proc-stat --proc-swaps --proc-uptime --proc-version --proc-vmallocinfo --proc-vmstat --proc-zoneinfo --proc-driver-rtc --proc-net-arp --proc-net-dev --proc-net-dev-mcast --proc-net-if-inet6 --proc-net-igmp --proc-net-igmp6 --proc-net-ipv6-route --proc-net-netlink --proc-net-netstat --proc-net-packet --proc-net-protocols --proc-net-route --proc-net-unix --proc-pid-fdinfo --proc-pid-io --proc-pid-maps --proc-pid-mountinfo --proc-pid-numa-maps --proc-pid-smaps --proc-pid-stat --proc-pid-statm --proc-pid-status --ps --route --rpm-qi --rsync --rsync-s --semver --sfdisk --shadow --ss --sshd-conf --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --toml --top --top-s --tracepath --traceroute --udevadm --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -151,6 +151,7 @@ _jc() { '--id:`id` command parser' '--ifconfig:`ifconfig` command parser' '--ini:INI file parser' + '--ini-dup:INI with duplicate key file parser' '--iostat:`iostat` command parser' '--iostat-s:`iostat` command streaming parser' '--ip-address:IPv4 and IPv6 Address string parser' diff --git a/docs/parsers/ini.md b/docs/parsers/ini.md index e8b64677..52b65d73 100644 --- a/docs/parsers/ini.md +++ b/docs/parsers/ini.md @@ -3,9 +3,9 @@ # jc.parsers.ini -jc - JSON Convert `INI` file parser +jc - JSON Convert INI file parser -Parses standard `INI` files. +Parses standard INI files. - Delimiter can be `=` or `:`. Missing values are supported. - Comment prefix can be `#` or `;`. Comments must be on their own line. @@ -31,35 +31,43 @@ INI document converted to a dictionary - see the python configparser standard library documentation for more details. { - "key1": string, - "key2": string + "": string, + "": string, + "": { + "": string, + "": string + }, + "": { + "": string, + "": string + } } Examples: $ cat example.ini - foo = bar - baz = buz + foo = fiz + bar = buz [section1] - key1 = value1 - key2 = value2 + fruit = apple + color = blue [section2] - key1 = value1 - key2 = value2 + fruit = pear + color = green $ cat example.ini | jc --ini -p { - "foo": "bar", - "baz": "buz", + "foo": "fiz", + "bar": "buz", "section1": { - "key1": "value1", - "key2": "value2" + "fruit": "apple", + "color": "blue" }, "section2": { - "key1": "value1", - "key2": "value2" + "fruit": "pear", + "color": "green" } } diff --git a/jc/lib.py b/jc/lib.py index 28f8e476..55978c14 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -59,6 +59,7 @@ parsers: List[str] = [ 'id', 'ifconfig', 'ini', + 'ini-dup', 'iostat', 'iostat-s', 'ip-address', diff --git a/jc/parsers/ini.py b/jc/parsers/ini.py index 5c19d7e4..0990fec6 100644 --- a/jc/parsers/ini.py +++ b/jc/parsers/ini.py @@ -1,6 +1,6 @@ -"""jc - JSON Convert `INI` file parser +"""jc - JSON Convert INI file parser -Parses standard `INI` files. +Parses standard INI files. - Delimiter can be `=` or `:`. Missing values are supported. - Comment prefix can be `#` or `;`. Comments must be on their own line. @@ -26,35 +26,43 @@ INI document converted to a dictionary - see the python configparser standard library documentation for more details. { - "key1": string, - "key2": string + "": string, + "": string, + "": { + "": string, + "": string + }, + "": { + "": string, + "": string + } } Examples: $ cat example.ini - foo = bar - baz = buz + foo = fiz + bar = buz [section1] - key1 = value1 - key2 = value2 + fruit = apple + color = blue [section2] - key1 = value1 - key2 = value2 + fruit = pear + color = green $ cat example.ini | jc --ini -p { - "foo": "bar", - "baz": "buz", + "foo": "fiz", + "bar": "buz", "section1": { - "key1": "value1", - "key2": "value2" + "fruit": "apple", + "color": "blue" }, "section2": { - "key1": "value1", - "key2": "value2" + "fruit": "pear", + "color": "green" } } """ diff --git a/jc/parsers/ini_dup.py b/jc/parsers/ini_dup.py new file mode 100644 index 00000000..b01b9d48 --- /dev/null +++ b/jc/parsers/ini_dup.py @@ -0,0 +1,224 @@ +"""jc - JSON Convert INI with duplicate key file parser + +Parses standard INI files and preserves duplicate values. All values are +contained in lists/arrays. Multi-line values are not supported. + +- Delimiter can be `=` or `:`. Missing values are supported. +- Comment prefix can be `#` or `;`. Comments must be on their own line. +- If duplicate keys are found, only the last value will be used. + +> Note: Values starting and ending with double or single quotation marks +> will have the marks removed. If you would like to keep the quotation +> marks, use the `-r` command-line argument or the `raw=True` argument in +> `parse()`. + +Usage (cli): + + $ cat foo.ini | jc --ini + +Usage (module): + + import jc + result = jc.parse('ini', ini_file_output) + +Schema: + +INI document converted to a dictionary - see the python configparser +standard library documentation for more details. + + { + "": [ + string + ], + "": [ + string + ], + "": { + "": [ + string + ], + "": [ + string + ] + } + } + +Examples: + + $ cat example.ini + foo = fiz + bar = buz + + [section1] + fruit = apple + color = blue + color = red + + [section2] + fruit = pear + fruit = peach + color = green + + $ cat example.ini | jc --ini -p + { + "foo": [ + "fiz" + ], + "bar": [ + "buz" + ], + "section1": { + "fruit": [ + "apple" + ], + "color": [ + "blue", + "red" + ] + }, + "section2": { + "fruit": [ + "pear", + "peach" + ], + "color": [ + "green" + ] + } + } +""" +import jc.utils +import configparser +import uuid + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'INI with duplicate key file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + details = 'Using configparser from the python standard library' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + tags = ['standard', 'file', 'string'] + + +__version__ = info.version + + +class MultiDict(dict): + def __setitem__(self, key, value): + if key in self: + if isinstance(value, list): + self[key].extend(value) + + elif isinstance(value, str): + if len(self[key])>1: + return + + else: + super().__setitem__(key, value) + + +def _remove_quotes(value): + if value is None: + value = '' + + elif value.startswith('"') and value.endswith('"'): + value = value[1:-1] + + elif value.startswith("'") and value.endswith("'"): + value = value[1:-1] + + return value + + +def _process(proc_data): + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary representing the INI file. + """ + # remove quotation marks from beginning and end of values + for k, v in proc_data.items(): + if isinstance(v, dict): + for key, value in v.items(): + if isinstance(value, list): + v[key] = [_remove_quotes(x) for x in value] + else: + v[key] = _remove_quotes(value) + continue + + elif isinstance(v, list): + proc_data[k] = [_remove_quotes(x) for x in v] + + else: + proc_data[k] = _remove_quotes(v) + + return proc_data + + +def parse(data, raw=False, quiet=False): + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary representing the INI file. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output = {} + + if jc.utils.has_data(data): + + # clean the data by removing blank lines and stripping leading whitespace + data = '\n'.join([x.lstrip() for x in data.splitlines() if x]) + + ini_parser = configparser.ConfigParser( + dict_type = MultiDict, + allow_no_value=True, + interpolation=None, + default_section=None, + empty_lines_in_values=False, + strict=False + ) + + # don't convert keys to lower-case: + ini_parser.optionxform = lambda option: option + + try: + ini_parser.read_string(data) + raw_output = {s: dict(ini_parser.items(s)) for s in ini_parser.sections()} + + except configparser.MissingSectionHeaderError: + # find a top-level section name that will not collide with any existing ones + while True: + my_uuid = str(uuid.uuid4()) + if my_uuid not in data: + break + + data = f'[{my_uuid}]\n' + data + ini_parser.read_string(data) + temp_dict = {s: dict(ini_parser.items(s)) for s in ini_parser.sections()} + + # move items under fake top-level sections to the root + raw_output = temp_dict.pop(my_uuid) + + # get the rest of the sections + raw_output.update(temp_dict) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index df6bb5aa..77ae70a9 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -265,6 +265,11 @@ hashsum command parser (`md5sum`, `shasum`, etc.) \fB--ini\fP INI file parser +.TP +.B +\fB--ini-dup\fP +INI with duplicate key file parser + .TP .B \fB--iostat\fP diff --git a/templates/readme_template b/templates/readme_template index f609db2e..8a07465f 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -622,39 +622,33 @@ ifconfig | jc -p --ifconfig # or: jc -p ifconfig cat example.ini ``` ``` -[DEFAULT] -ServerAliveInterval = 45 -Compression = yes -CompressionLevel = 9 -ForwardX11 = yes +foo = fiz +bar = buz -[bitbucket.org] -User = hg +[section1] +fruit = apple +color = blue -[topsecret.server.com] -Port = 50022 -ForwardX11 = no +[section2] +fruit = pear +color = green ``` ```bash cat example.ini | jc -p --ini ``` ```json { - "DEFAULT": { - "ServerAliveInterval": "45", - "Compression": "yes", - "CompressionLevel": "9", - "ForwardX11": "yes" + "foo": "fiz", + "bar": "buz", + "section1": { + "fruit": "apple", + "color": "blue" }, - "bitbucket.org": { - "User": "hg" - }, - "topsecret.server.com": { - "Port": "50022", - "ForwardX11": "no" + "section2": { + "fruit": "pear", + "color": "green" } } - ``` ### ls ```bash