From a249ca1da3454a8ebdac5067354fd7312c637230 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 2 Sep 2022 18:29:16 -0700 Subject: [PATCH 01/71] add initial procfile parser --- jc/lib.py | 1 + jc/parsers/procfile.py | 197 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 jc/parsers/procfile.py diff --git a/jc/lib.py b/jc/lib.py index 98c5084e..5d8e9815 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -84,6 +84,7 @@ parsers = [ 'pip-show', 'plist', 'postconf', + 'procfile', 'ps', 'route', 'rpm-qi', diff --git a/jc/parsers/procfile.py b/jc/parsers/procfile.py new file mode 100644 index 00000000..54fa5578 --- /dev/null +++ b/jc/parsers/procfile.py @@ -0,0 +1,197 @@ +"""jc - JSON Convert Proc file output parser + +<> + +Usage (cli): + + $ cat /proc/ | jc --procfile + +Usage (module): + + import jc + result = jc.parse('procfile', proc_file) + +Schema: + + [ + { + "procfile": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ procfile | jc --procfile -p + [] + + $ procfile | jc --procfile -p -r + [] +""" +import re +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = 'Proc file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + + # process the data here + # rebuild output for added semantic information + # use helper functions in jc.utils for int, float, bool + # conversions and timestamps + + return proc_data + + +def _parse_uptime(): + print('uptime') + + +def _parse_loadavg(): + print('loadavg') + + +def _parse_cpuinfo(): + print('cpuinfo') + + +def _parse_meminfo(): + print('meminfo') + + +def _parse_version(): + print('version') + + +def _parse_crypto(): + print('crypto') + + +def _parse_diskstats(): + print('diskstats') + + +def _parse_filesystems(): + print('filesystems') + + +def _parse_pid_status(): + print('pid status') + + +def _parse_pid_statm(): + print('pid statm') + + +def _parse_pid_stat(): + print('pid stat') + + +def _parse_pid_smaps(): + print('pid smaps') + + +def _parse_pid_maps(): + print('pid maps') + + +def _parse_pid_numa_maps(): + print('pid numa_maps') + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + # signatures + uptime_p = re.compile(r'^\d+.\d\d \d+.\d\d$') + loadavg_p = re.compile(r'^\d+.\d\d \d+.\d\d \d+.\d\d \d+/\d+ \d+$') + cpuinfo_p = re.compile(r'^processor\s+:.*\nvendor_id\s+:.*\ncpu family\s+:.*\n') + meminfo_p = re.compile(r'^MemTotal:.*\nMemFree:.*\nMemAvailable:.*\n') + version_p = re.compile(r'^.+\sversion\s[^\n]+$') + crypto_p = re.compile(r'^name\s+:.*\ndriver\s+:.*\nmodule\s+:.*\n') + diskstats_p = re.compile(r'^\s*\d+\s+\d\s\w+\s(?:\d+\s){16}\d\n') + filesystems_p = re.compile(r'^(?:(?:nodev\t|\t)\w+\n){3}') + pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n') + pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$') + pid_stat_p = re.compile(r'^\d+ \(.{1,16}\) \w \d+ \d+ \d+ \d+ -?\d+ (?:\d+ ){43}\d+$') + pid_smaps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ [^\n]+\nSize:\s+\d+ \S\S\n') + pid_maps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ ') + pid_numa_maps_p = re.compile(r'^[a-f0-9]{12} default [^\n]+\n') + + procmap = { + uptime_p: _parse_uptime, + loadavg_p: _parse_loadavg, + cpuinfo_p: _parse_cpuinfo, + meminfo_p: _parse_meminfo, + version_p: _parse_version, + crypto_p: _parse_crypto, + diskstats_p: _parse_diskstats, + filesystems_p: _parse_filesystems, + pid_status_p: _parse_pid_status, + pid_statm_p: _parse_pid_statm, + pid_stat_p: _parse_pid_stat, + pid_smaps_p: _parse_pid_smaps, # before pid_maps + pid_maps_p: _parse_pid_maps, # after pid_smaps + pid_numa_maps_p: _parse_pid_numa_maps + } + + for reg_pattern, parse_func in procmap.items(): + if reg_pattern.search(data): + parse_func() + break + + # for line in filter(None, data.splitlines()): + + # # parse the content here + # # check out helper functions in jc.utils + # # and jc.parsers.universal + + # pass + + return raw_output if raw else _process(raw_output) From edcb2280cc6433dc0a78c85bfacfba0b0700a37b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 2 Sep 2022 19:20:36 -0700 Subject: [PATCH 02/71] add more filetypes --- jc/parsers/procfile.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/jc/parsers/procfile.py b/jc/parsers/procfile.py index 54fa5578..0fa624c1 100644 --- a/jc/parsers/procfile.py +++ b/jc/parsers/procfile.py @@ -99,6 +99,22 @@ def _parse_filesystems(): print('filesystems') +def _parse_interrupts(): + print('interrupts') + + +def _parse_buddyinfo(): + print('buddyinfo') + + +def _parse_pagetypeinfo(): + print('pagetypinfo') + + +def _parse_vmallocinfo(): + print('vmallocinfo') + + def _parse_pid_status(): print('pid status') @@ -157,6 +173,10 @@ def parse( crypto_p = re.compile(r'^name\s+:.*\ndriver\s+:.*\nmodule\s+:.*\n') diskstats_p = re.compile(r'^\s*\d+\s+\d\s\w+\s(?:\d+\s){16}\d\n') filesystems_p = re.compile(r'^(?:(?:nodev\t|\t)\w+\n){3}') + interrupts_p = re.compile(r'^\s+(?:CPU\d+ +)+\n\s*\d+:\s+\d+') + buddyinfo_p = re.compile(r'^Node \d+, zone\s+\w+\s+(?:\d+\s+){11}\n') + pagetypeinfo_p = re.compile(r'^Page block order:\s+\d+\nPages per block:\s+\d+\n\n') + vmallocinfo_p = re.compile(r'^0x[0-9a-f]{16}-0x[0-9a-f]{16}\s+\d+ \w+\+\w+/\w+ ') pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n') pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$') pid_stat_p = re.compile(r'^\d+ \(.{1,16}\) \w \d+ \d+ \d+ \d+ -?\d+ (?:\d+ ){43}\d+$') @@ -173,6 +193,10 @@ def parse( crypto_p: _parse_crypto, diskstats_p: _parse_diskstats, filesystems_p: _parse_filesystems, + interrupts_p: _parse_interrupts, + buddyinfo_p: _parse_buddyinfo, + pagetypeinfo_p: _parse_pagetypeinfo, + vmallocinfo_p: _parse_vmallocinfo, pid_status_p: _parse_pid_status, pid_statm_p: _parse_pid_statm, pid_stat_p: _parse_pid_stat, From e771b36a18b733106a1efb6c1c568c9279865cae Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 3 Sep 2022 10:52:35 -0700 Subject: [PATCH 03/71] add more signatures --- jc/parsers/procfile.py | 156 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) diff --git a/jc/parsers/procfile.py b/jc/parsers/procfile.py index 0fa624c1..9ade1aaa 100644 --- a/jc/parsers/procfile.py +++ b/jc/parsers/procfile.py @@ -103,6 +103,26 @@ def _parse_interrupts(): print('interrupts') +def _parse_iomem(): + print('iomem') + + +def _parse_ioports(): + print('ioports') + + +def _parse_locks(): + print('locks') + + +def _parse_modules(): + print('modules') + + +def _parse_partitions(): + print('partitions') + + def _parse_buddyinfo(): print('buddyinfo') @@ -115,6 +135,72 @@ def _parse_vmallocinfo(): print('vmallocinfo') +def _parse_softirqs(): + print('softirqs') + + +def _parse_scsi(): + print('scsi') + + +def _parse_stat(): + print('stat') + + +def _parse_swaps(): + print('swaps') + + +def _parse_slabinfo(): + print('slabinfo') + + +def _parse_consoles(): + print('consoles') + + +def _parse_devices(): + print('devices') + + +def _parse_vmstat(): + print('vmstat') + + +def _parse_zoneinfo(): + print('zoneinfo') + +#### + +def _parse_driver_rtc(): + print('driver rtc') + +#### + +def _parse_net_arp(): + print('net arp') + +def _parse_net_dev(): + print('net dev') + +def _parse_net_dev_mcast(): + print('net dev_mcast') + +def _parse_net_igmp(): + print('net igmp') + +def _parse_net_igmp6(): + print('net igmp6') + +def _parse_net_if_inet6(): + print('net if_inet6') + +def _parse_net_ipv6_route(): + print('net ipv6_route') + +#### + + def _parse_pid_status(): print('pid status') @@ -139,6 +225,18 @@ def _parse_pid_numa_maps(): print('pid numa_maps') +def _parse_pid_io(): + print('pid io') + + +def _parse_pid_mountinfo(): + print('pid mountinfo') + + +def _parse_pid_fdinfo(): + print('pid fdinfo') + + def parse( data: str, raw: bool = False, @@ -174,15 +272,43 @@ def parse( diskstats_p = re.compile(r'^\s*\d+\s+\d\s\w+\s(?:\d+\s){16}\d\n') filesystems_p = re.compile(r'^(?:(?:nodev\t|\t)\w+\n){3}') interrupts_p = re.compile(r'^\s+(?:CPU\d+ +)+\n\s*\d+:\s+\d+') + iomem_p = re.compile(r'^00000000-[0-9a-f]{8} : .*\n[0-9a-f]{8}-[0-9a-f]{8} : ') + ioports_p = re.compile(r'^0000-[0-9a-f]{4} : .*\n\s*0000-[0-9a-f]{4} : ') + locks_p = re.compile(r'^\d+: (?:POSIX|FLOCK|OFDLCK)\s+(?:ADVISORY|MANDATORY)\s+(?:READ|WRITE) ') + modules_p = re.compile(r'^\w+ \d+ \d+ (?:-|\w+,).*0x[0-9a-f]{16}\n') buddyinfo_p = re.compile(r'^Node \d+, zone\s+\w+\s+(?:\d+\s+){11}\n') pagetypeinfo_p = re.compile(r'^Page block order:\s+\d+\nPages per block:\s+\d+\n\n') + partitions_p = re.compile(r'^major minor #blocks name\n\n\s*\d+\s+\d+\s+\d+ \w+\n') vmallocinfo_p = re.compile(r'^0x[0-9a-f]{16}-0x[0-9a-f]{16}\s+\d+ \w+\+\w+/\w+ ') + softirqs_p = re.compile(r'^\s+(CPU\d+\s+)+\n\s+HI:\s+\d') + scsi_p = re.compile(r'^Attached devices:\nHost: \w+ ') + stat_p = re.compile(r'^cpu\s+(?: \d+){10}.*intr ', re.DOTALL) + consoles_p = re.compile(r'^\w+\s+[\-WUR]{3} \([ECBpba ]+\)\s+\d+:\d+\n') + devices_p = re.compile(r'^Character devices:\n\s+\d+ .*\n') + slabinfo_p = re.compile(r'^slabinfo - version: \d+.\d+\n') + swaps_p = re.compile(r'^Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n') + vmstat_p = re.compile(r'nr_free_pages \d+\n.* \d$', re.DOTALL) + zoneinfo_p = re.compile(r'^Node \d+, zone\s+\w+\n') + + driver_rtc_p = re.compile(r'^rtc_time\t: .*\nrtc_date\t: .*\nalrm_time\t: .*\n') + + net_arp_p = re.compile(r'^IP address\s+HW type\s+Flags\s+HW address\s+Mask\s+Device\n') + net_dev_p = re.compile(r'^Inter-\|\s+Receive\s+\|\s+Transmit\n') + net_dev_mcast_p = re.compile(r'^\d+\s+\w+\s+\d+\s+\d+\s+[0-9a-f]{12}') + net_igmp_p = re.compile(r'^Idx\tDevice\s+:\s+Count\s+Querier\tGroup\s+Users\s+Timer\tReporter\n') + net_igmp6_p = re.compile(r'^\d+\s+\w+\s+[0-9a-f]{32}\s+\d+\s+[0-9A-F]{8}\s+\d+') + net_if_inet6_p = re.compile(r'^[0-9a-f]{32} \d\d \d\d \d\d \d\d\s+\w+') + net_ipv6_route_p = re.compile(r'^[0-9a-f]{32} \d\d [0-9a-f]{32} \d\d [0-9a-f]{32} (?:[0-9a-f]{8} ){4}\s+\w+') + pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n') pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$') pid_stat_p = re.compile(r'^\d+ \(.{1,16}\) \w \d+ \d+ \d+ \d+ -?\d+ (?:\d+ ){43}\d+$') pid_smaps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ [^\n]+\nSize:\s+\d+ \S\S\n') pid_maps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ ') pid_numa_maps_p = re.compile(r'^[a-f0-9]{12} default [^\n]+\n') + pid_io_p = re.compile(r'^rchar: \d+\nwchar: \d+\nsyscr: \d+\n') + pid_mountinfo_p = re.compile(r'^\d+ \d+ \d+:\d+ /.+\n') + pid_fdinfo_p = re.compile(r'^pos:\t\d+\nflags:\t\d+\nmnt_id:\t\d+\n') procmap = { uptime_p: _parse_uptime, @@ -194,15 +320,43 @@ def parse( diskstats_p: _parse_diskstats, filesystems_p: _parse_filesystems, interrupts_p: _parse_interrupts, + iomem_p: _parse_iomem, + ioports_p: _parse_ioports, + locks_p: _parse_locks, + modules_p: _parse_modules, buddyinfo_p: _parse_buddyinfo, pagetypeinfo_p: _parse_pagetypeinfo, + partitions_p: _parse_partitions, vmallocinfo_p: _parse_vmallocinfo, + slabinfo_p: _parse_slabinfo, + softirqs_p: _parse_softirqs, + scsi_p: _parse_scsi, + stat_p: _parse_stat, + swaps_p: _parse_swaps, + consoles_p: _parse_consoles, + devices_p: _parse_devices, + vmstat_p: _parse_vmstat, + zoneinfo_p: _parse_zoneinfo, + + driver_rtc_p: _parse_driver_rtc, + + net_arp_p: _parse_net_arp, + net_dev_p: _parse_net_dev, + net_ipv6_route_p: _parse_net_ipv6_route, # before net_dev_mcast + net_dev_mcast_p: _parse_net_dev_mcast, # after net_ipv6_route + net_igmp_p: _parse_net_igmp, + net_igmp6_p: _parse_net_igmp6, + net_if_inet6_p: _parse_net_if_inet6, + pid_status_p: _parse_pid_status, pid_statm_p: _parse_pid_statm, pid_stat_p: _parse_pid_stat, pid_smaps_p: _parse_pid_smaps, # before pid_maps pid_maps_p: _parse_pid_maps, # after pid_smaps - pid_numa_maps_p: _parse_pid_numa_maps + pid_numa_maps_p: _parse_pid_numa_maps, + pid_io_p: _parse_pid_io, + pid_mountinfo_p: _parse_pid_mountinfo, + pid_fdinfo_p: _parse_pid_fdinfo } for reg_pattern, parse_func in procmap.items(): From c1b2bae3334dbe96d8b673f21304cdbbe22cea57 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 3 Sep 2022 11:34:28 -0700 Subject: [PATCH 04/71] add more net files --- jc/parsers/procfile.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/jc/parsers/procfile.py b/jc/parsers/procfile.py index 9ade1aaa..a2d0532f 100644 --- a/jc/parsers/procfile.py +++ b/jc/parsers/procfile.py @@ -198,6 +198,21 @@ def _parse_net_if_inet6(): def _parse_net_ipv6_route(): print('net ipv6_route') +def _parse_net_netlink(): + print('net netlink') + +def _parse_net_netstat(): + print('net netstat') + +def _parse_net_packet(): + print('net packet') + +def _parse_net_protocols(): + print('net protocols') + +def _parse_net_route(): + print('net route') + #### @@ -299,6 +314,11 @@ def parse( net_igmp6_p = re.compile(r'^\d+\s+\w+\s+[0-9a-f]{32}\s+\d+\s+[0-9A-F]{8}\s+\d+') net_if_inet6_p = re.compile(r'^[0-9a-f]{32} \d\d \d\d \d\d \d\d\s+\w+') net_ipv6_route_p = re.compile(r'^[0-9a-f]{32} \d\d [0-9a-f]{32} \d\d [0-9a-f]{32} (?:[0-9a-f]{8} ){4}\s+\w+') + net_netlink_p = re.compile(r'^sk\s+Eth Pid\s+Groups\s+Rmem\s+Wmem') + net_netstat_p = re.compile(r'^TcpExt: SyncookiesSent SyncookiesRecv SyncookiesFailed') + net_packet_p = re.compile(r'^sk RefCnt Type Proto Iface R Rmem User Inode\n') + net_protocols_p = re.compile(r'^protocol size sockets memory press maxhdr slab module cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n') + net_route_p = re.compile(r'^Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\s+\n') pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n') pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$') @@ -347,6 +367,11 @@ def parse( net_igmp_p: _parse_net_igmp, net_igmp6_p: _parse_net_igmp6, net_if_inet6_p: _parse_net_if_inet6, + net_netlink_p: _parse_net_netlink, + net_netstat_p: _parse_net_netstat, + net_packet_p: _parse_net_packet, + net_protocols_p: _parse_net_protocols, + net_route_p: _parse_net_route, pid_status_p: _parse_pid_status, pid_statm_p: _parse_pid_statm, From 146dc070ea761502b814e40bfd873ea859a78e44 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 3 Sep 2022 14:05:57 -0700 Subject: [PATCH 05/71] reorder patterns --- jc/parsers/procfile.py | 89 +++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/jc/parsers/procfile.py b/jc/parsers/procfile.py index a2d0532f..8e5cf2a0 100644 --- a/jc/parsers/procfile.py +++ b/jc/parsers/procfile.py @@ -119,6 +119,10 @@ def _parse_modules(): print('modules') +def _parse_mtrr(): + print('mtrr') + + def _parse_partitions(): print('partitions') @@ -139,10 +143,6 @@ def _parse_softirqs(): print('softirqs') -def _parse_scsi(): - print('scsi') - - def _parse_stat(): print('stat') @@ -213,6 +213,9 @@ def _parse_net_protocols(): def _parse_net_route(): print('net route') +def _parse_net_unix(): + print('net unix') + #### @@ -251,6 +254,14 @@ def _parse_pid_mountinfo(): def _parse_pid_fdinfo(): print('pid fdinfo') +#### + +def _parse_scsi_scsi(): + print('scsi scsi') + +def _parse_scsi_device_info(): + print('scsi device_info') + def parse( data: str, @@ -278,30 +289,30 @@ def parse( if jc.utils.has_data(data): # signatures - uptime_p = re.compile(r'^\d+.\d\d \d+.\d\d$') - loadavg_p = re.compile(r'^\d+.\d\d \d+.\d\d \d+.\d\d \d+/\d+ \d+$') + buddyinfo_p = re.compile(r'^Node \d+, zone\s+\w+\s+(?:\d+\s+){11}\n') + consoles_p = re.compile(r'^\w+\s+[\-WUR]{3} \([ECBpba ]+\)\s+\d+:\d+\n') cpuinfo_p = re.compile(r'^processor\s+:.*\nvendor_id\s+:.*\ncpu family\s+:.*\n') - meminfo_p = re.compile(r'^MemTotal:.*\nMemFree:.*\nMemAvailable:.*\n') - version_p = re.compile(r'^.+\sversion\s[^\n]+$') crypto_p = re.compile(r'^name\s+:.*\ndriver\s+:.*\nmodule\s+:.*\n') + devices_p = re.compile(r'^Character devices:\n\s+\d+ .*\n') diskstats_p = re.compile(r'^\s*\d+\s+\d\s\w+\s(?:\d+\s){16}\d\n') filesystems_p = re.compile(r'^(?:(?:nodev\t|\t)\w+\n){3}') interrupts_p = re.compile(r'^\s+(?:CPU\d+ +)+\n\s*\d+:\s+\d+') iomem_p = re.compile(r'^00000000-[0-9a-f]{8} : .*\n[0-9a-f]{8}-[0-9a-f]{8} : ') ioports_p = re.compile(r'^0000-[0-9a-f]{4} : .*\n\s*0000-[0-9a-f]{4} : ') + loadavg_p = re.compile(r'^\d+.\d\d \d+.\d\d \d+.\d\d \d+/\d+ \d+$') locks_p = re.compile(r'^\d+: (?:POSIX|FLOCK|OFDLCK)\s+(?:ADVISORY|MANDATORY)\s+(?:READ|WRITE) ') + meminfo_p = re.compile(r'^MemTotal:.*\nMemFree:.*\nMemAvailable:.*\n') modules_p = re.compile(r'^\w+ \d+ \d+ (?:-|\w+,).*0x[0-9a-f]{16}\n') - buddyinfo_p = re.compile(r'^Node \d+, zone\s+\w+\s+(?:\d+\s+){11}\n') + mtrr_p = re.compile(r'^reg\d+: base=0x[0-9a-f]+ \(') pagetypeinfo_p = re.compile(r'^Page block order:\s+\d+\nPages per block:\s+\d+\n\n') partitions_p = re.compile(r'^major minor #blocks name\n\n\s*\d+\s+\d+\s+\d+ \w+\n') - vmallocinfo_p = re.compile(r'^0x[0-9a-f]{16}-0x[0-9a-f]{16}\s+\d+ \w+\+\w+/\w+ ') - softirqs_p = re.compile(r'^\s+(CPU\d+\s+)+\n\s+HI:\s+\d') - scsi_p = re.compile(r'^Attached devices:\nHost: \w+ ') - stat_p = re.compile(r'^cpu\s+(?: \d+){10}.*intr ', re.DOTALL) - consoles_p = re.compile(r'^\w+\s+[\-WUR]{3} \([ECBpba ]+\)\s+\d+:\d+\n') - devices_p = re.compile(r'^Character devices:\n\s+\d+ .*\n') slabinfo_p = re.compile(r'^slabinfo - version: \d+.\d+\n') + softirqs_p = re.compile(r'^\s+(CPU\d+\s+)+\n\s+HI:\s+\d') + stat_p = re.compile(r'^cpu\s+(?: \d+){10}.*intr ', re.DOTALL) swaps_p = re.compile(r'^Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n') + uptime_p = re.compile(r'^\d+.\d\d \d+.\d\d$') + version_p = re.compile(r'^.+\sversion\s[^\n]+$') + vmallocinfo_p = re.compile(r'^0x[0-9a-f]{16}-0x[0-9a-f]{16}\s+\d+ \w+\+\w+/\w+ ') vmstat_p = re.compile(r'nr_free_pages \d+\n.* \d$', re.DOTALL) zoneinfo_p = re.compile(r'^Node \d+, zone\s+\w+\n') @@ -310,15 +321,16 @@ def parse( net_arp_p = re.compile(r'^IP address\s+HW type\s+Flags\s+HW address\s+Mask\s+Device\n') net_dev_p = re.compile(r'^Inter-\|\s+Receive\s+\|\s+Transmit\n') net_dev_mcast_p = re.compile(r'^\d+\s+\w+\s+\d+\s+\d+\s+[0-9a-f]{12}') + net_if_inet6_p = re.compile(r'^[0-9a-f]{32} \d\d \d\d \d\d \d\d\s+\w+') net_igmp_p = re.compile(r'^Idx\tDevice\s+:\s+Count\s+Querier\tGroup\s+Users\s+Timer\tReporter\n') net_igmp6_p = re.compile(r'^\d+\s+\w+\s+[0-9a-f]{32}\s+\d+\s+[0-9A-F]{8}\s+\d+') - net_if_inet6_p = re.compile(r'^[0-9a-f]{32} \d\d \d\d \d\d \d\d\s+\w+') net_ipv6_route_p = re.compile(r'^[0-9a-f]{32} \d\d [0-9a-f]{32} \d\d [0-9a-f]{32} (?:[0-9a-f]{8} ){4}\s+\w+') net_netlink_p = re.compile(r'^sk\s+Eth Pid\s+Groups\s+Rmem\s+Wmem') net_netstat_p = re.compile(r'^TcpExt: SyncookiesSent SyncookiesRecv SyncookiesFailed') net_packet_p = re.compile(r'^sk RefCnt Type Proto Iface R Rmem User Inode\n') net_protocols_p = re.compile(r'^protocol size sockets memory press maxhdr slab module cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n') net_route_p = re.compile(r'^Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\s+\n') + net_unix_p = re.compile(r'^Num RefCount Protocol Flags Type St Inode Path\n') pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n') pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$') @@ -330,31 +342,34 @@ def parse( pid_mountinfo_p = re.compile(r'^\d+ \d+ \d+:\d+ /.+\n') pid_fdinfo_p = re.compile(r'^pos:\t\d+\nflags:\t\d+\nmnt_id:\t\d+\n') + scsi_device_info = re.compile(r"^'\w+' '.+' 0x\d+") + scsi_scsi_p = re.compile(r'^Attached devices:\nHost: \w+ ') + procmap = { - uptime_p: _parse_uptime, - loadavg_p: _parse_loadavg, + buddyinfo_p: _parse_buddyinfo, + consoles_p: _parse_consoles, cpuinfo_p: _parse_cpuinfo, - meminfo_p: _parse_meminfo, - version_p: _parse_version, crypto_p: _parse_crypto, + devices_p: _parse_devices, diskstats_p: _parse_diskstats, filesystems_p: _parse_filesystems, interrupts_p: _parse_interrupts, iomem_p: _parse_iomem, ioports_p: _parse_ioports, + loadavg_p: _parse_loadavg, locks_p: _parse_locks, + meminfo_p: _parse_meminfo, modules_p: _parse_modules, - buddyinfo_p: _parse_buddyinfo, + mtrr_p: _parse_mtrr, pagetypeinfo_p: _parse_pagetypeinfo, partitions_p: _parse_partitions, - vmallocinfo_p: _parse_vmallocinfo, slabinfo_p: _parse_slabinfo, softirqs_p: _parse_softirqs, - scsi_p: _parse_scsi, stat_p: _parse_stat, swaps_p: _parse_swaps, - consoles_p: _parse_consoles, - devices_p: _parse_devices, + uptime_p: _parse_uptime, + version_p: _parse_version, + vmallocinfo_p: _parse_vmallocinfo, vmstat_p: _parse_vmstat, zoneinfo_p: _parse_zoneinfo, @@ -362,26 +377,30 @@ def parse( net_arp_p: _parse_net_arp, net_dev_p: _parse_net_dev, - net_ipv6_route_p: _parse_net_ipv6_route, # before net_dev_mcast - net_dev_mcast_p: _parse_net_dev_mcast, # after net_ipv6_route + net_if_inet6_p: _parse_net_if_inet6, net_igmp_p: _parse_net_igmp, net_igmp6_p: _parse_net_igmp6, - net_if_inet6_p: _parse_net_if_inet6, net_netlink_p: _parse_net_netlink, net_netstat_p: _parse_net_netstat, net_packet_p: _parse_net_packet, net_protocols_p: _parse_net_protocols, net_route_p: _parse_net_route, + net_unix_p: _parse_net_unix, + net_ipv6_route_p: _parse_net_ipv6_route, # before net_dev_mcast + net_dev_mcast_p: _parse_net_dev_mcast, # after net_ipv6_route - pid_status_p: _parse_pid_status, - pid_statm_p: _parse_pid_statm, - pid_stat_p: _parse_pid_stat, - pid_smaps_p: _parse_pid_smaps, # before pid_maps - pid_maps_p: _parse_pid_maps, # after pid_smaps - pid_numa_maps_p: _parse_pid_numa_maps, + pid_fdinfo_p: _parse_pid_fdinfo, pid_io_p: _parse_pid_io, pid_mountinfo_p: _parse_pid_mountinfo, - pid_fdinfo_p: _parse_pid_fdinfo + pid_numa_maps_p: _parse_pid_numa_maps, + pid_stat_p: _parse_pid_stat, + pid_statm_p: _parse_pid_statm, + pid_status_p: _parse_pid_status, + pid_smaps_p: _parse_pid_smaps, # before pid_maps + pid_maps_p: _parse_pid_maps, # after pid_smaps + + scsi_device_info: _parse_scsi_device_info, + scsi_scsi_p: _parse_scsi_scsi } for reg_pattern, parse_func in procmap.items(): From 79ade2c1825f1ca81e2c8381a4473ad130d948ee Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sun, 4 Sep 2022 10:42:16 -0700 Subject: [PATCH 06/71] move to module architecture. first two proc parsers --- jc/lib.py | 2 +- jc/parsers/proc.py | 195 +++++++++++++++++ jc/parsers/proc_meminfo.py | 92 ++++++++ jc/parsers/proc_modules.py | 172 +++++++++++++++ jc/parsers/procfile.py | 419 ------------------------------------- 5 files changed, 460 insertions(+), 420 deletions(-) create mode 100644 jc/parsers/proc.py create mode 100644 jc/parsers/proc_meminfo.py create mode 100644 jc/parsers/proc_modules.py delete mode 100644 jc/parsers/procfile.py diff --git a/jc/lib.py b/jc/lib.py index 5d8e9815..0b554691 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -84,7 +84,7 @@ parsers = [ 'pip-show', 'plist', 'postconf', - 'procfile', + 'proc', 'ps', 'route', 'rpm-qi', diff --git a/jc/parsers/proc.py b/jc/parsers/proc.py new file mode 100644 index 00000000..4793bf9d --- /dev/null +++ b/jc/parsers/proc.py @@ -0,0 +1,195 @@ +"""jc - JSON Convert Proc file output parser + +<> + +Usage (cli): + + $ cat /proc/ | jc --procfile + +Usage (module): + + import jc + result = jc.parse('procfile', proc_file) + +Schema: + + [ + { + "procfile": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ procfile | jc --procfile -p + [] + + $ procfile | jc --procfile -p -r + [] +""" +import re +import importlib +from typing import List, Dict +import jc.utils +from jc.exceptions import ParseError + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + + +__version__ = info.version + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.input_type_check(data) + + if jc.utils.has_data(data): + # signatures + buddyinfo_p = re.compile(r'^Node \d+, zone\s+\w+\s+(?:\d+\s+){11}\n') + consoles_p = re.compile(r'^\w+\s+[\-WUR]{3} \([ECBpba ]+\)\s+\d+:\d+\n') + cpuinfo_p = re.compile(r'^processor\s+:.*\nvendor_id\s+:.*\ncpu family\s+:.*\n') + crypto_p = re.compile(r'^name\s+:.*\ndriver\s+:.*\nmodule\s+:.*\n') + devices_p = re.compile(r'^Character devices:\n\s+\d+ .*\n') + diskstats_p = re.compile(r'^\s*\d+\s+\d\s\w+\s(?:\d+\s){16}\d\n') + filesystems_p = re.compile(r'^(?:(?:nodev\t|\t)\w+\n){3}') + interrupts_p = re.compile(r'^\s+(?:CPU\d+ +)+\n\s*\d+:\s+\d+') + iomem_p = re.compile(r'^00000000-[0-9a-f]{8} : .*\n[0-9a-f]{8}-[0-9a-f]{8} : ') + ioports_p = re.compile(r'^0000-[0-9a-f]{4} : .*\n\s*0000-[0-9a-f]{4} : ') + loadavg_p = re.compile(r'^\d+.\d\d \d+.\d\d \d+.\d\d \d+/\d+ \d+$') + locks_p = re.compile(r'^\d+: (?:POSIX|FLOCK|OFDLCK)\s+(?:ADVISORY|MANDATORY)\s+(?:READ|WRITE) ') + meminfo_p = re.compile(r'^MemTotal:.*\nMemFree:.*\nMemAvailable:.*\n') + modules_p = re.compile(r'^\w+ \d+ \d+ (?:-|\w+,).*0x[0-9a-f]{16}\n') + mtrr_p = re.compile(r'^reg\d+: base=0x[0-9a-f]+ \(') + pagetypeinfo_p = re.compile(r'^Page block order:\s+\d+\nPages per block:\s+\d+\n\n') + partitions_p = re.compile(r'^major minor #blocks name\n\n\s*\d+\s+\d+\s+\d+ \w+\n') + slabinfo_p = re.compile(r'^slabinfo - version: \d+.\d+\n') + softirqs_p = re.compile(r'^\s+(CPU\d+\s+)+\n\s+HI:\s+\d') + stat_p = re.compile(r'^cpu\s+(?: \d+){10}.*intr ', re.DOTALL) + swaps_p = re.compile(r'^Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n') + uptime_p = re.compile(r'^\d+.\d\d \d+.\d\d$') + version_p = re.compile(r'^.+\sversion\s[^\n]+$') + vmallocinfo_p = re.compile(r'^0x[0-9a-f]{16}-0x[0-9a-f]{16}\s+\d+ \w+\+\w+/\w+ ') + vmstat_p = re.compile(r'nr_free_pages \d+\n.* \d$', re.DOTALL) + zoneinfo_p = re.compile(r'^Node \d+, zone\s+\w+\n') + + driver_rtc_p = re.compile(r'^rtc_time\t: .*\nrtc_date\t: .*\nalrm_time\t: .*\n') + + net_arp_p = re.compile(r'^IP address\s+HW type\s+Flags\s+HW address\s+Mask\s+Device\n') + net_dev_p = re.compile(r'^Inter-\|\s+Receive\s+\|\s+Transmit\n') + net_dev_mcast_p = re.compile(r'^\d+\s+\w+\s+\d+\s+\d+\s+[0-9a-f]{12}') + net_if_inet6_p = re.compile(r'^[0-9a-f]{32} \d\d \d\d \d\d \d\d\s+\w+') + net_igmp_p = re.compile(r'^Idx\tDevice\s+:\s+Count\s+Querier\tGroup\s+Users\s+Timer\tReporter\n') + net_igmp6_p = re.compile(r'^\d+\s+\w+\s+[0-9a-f]{32}\s+\d+\s+[0-9A-F]{8}\s+\d+') + net_ipv6_route_p = re.compile(r'^[0-9a-f]{32} \d\d [0-9a-f]{32} \d\d [0-9a-f]{32} (?:[0-9a-f]{8} ){4}\s+\w+') + net_netlink_p = re.compile(r'^sk\s+Eth Pid\s+Groups\s+Rmem\s+Wmem') + net_netstat_p = re.compile(r'^TcpExt: SyncookiesSent SyncookiesRecv SyncookiesFailed') + net_packet_p = re.compile(r'^sk RefCnt Type Proto Iface R Rmem User Inode\n') + net_protocols_p = re.compile(r'^protocol size sockets memory press maxhdr slab module cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n') + net_route_p = re.compile(r'^Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\s+\n') + net_unix_p = re.compile(r'^Num RefCount Protocol Flags Type St Inode Path\n') + + pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n') + pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$') + pid_stat_p = re.compile(r'^\d+ \(.{1,16}\) \w \d+ \d+ \d+ \d+ -?\d+ (?:\d+ ){43}\d+$') + pid_smaps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ [^\n]+\nSize:\s+\d+ \S\S\n') + pid_maps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ ') + pid_numa_maps_p = re.compile(r'^[a-f0-9]{12} default [^\n]+\n') + pid_io_p = re.compile(r'^rchar: \d+\nwchar: \d+\nsyscr: \d+\n') + pid_mountinfo_p = re.compile(r'^\d+ \d+ \d+:\d+ /.+\n') + pid_fdinfo_p = re.compile(r'^pos:\t\d+\nflags:\t\d+\nmnt_id:\t\d+\n') + + scsi_device_info = re.compile(r"^'\w+' '.+' 0x\d+") + scsi_scsi_p = re.compile(r'^Attached devices:\nHost: \w+ ') + + procmap = { + buddyinfo_p: 'proc_buddyinfo', + consoles_p: 'proc_consoles', + cpuinfo_p: 'proc_cpuinfo', + crypto_p: 'proc_crypto', + devices_p: 'proc_devices', + diskstats_p: 'proc_diskstats', + filesystems_p: 'proc_filesystems', + interrupts_p: 'proc_interrupts', + iomem_p: 'proc_iomem', + ioports_p: 'proc_ioports', + loadavg_p: 'proc_loadavg', + locks_p: 'proc_locks', + meminfo_p: 'proc_meminfo', ####### + modules_p: 'proc_modules', + mtrr_p: 'proc_mtrr', + pagetypeinfo_p: 'proc_pagetypeinfo', + partitions_p: 'proc_partitions', + slabinfo_p: 'proc_slabinfo', + softirqs_p: 'proc_softirqs', + stat_p: 'proc_stat', + swaps_p: 'proc_swaps', + uptime_p: 'proc_uptime', + version_p: 'proc_version', + vmallocinfo_p: 'proc_vmallocinfo', + vmstat_p: 'proc_vmstat', + zoneinfo_p: 'proc_zoneinfo', + + driver_rtc_p: 'proc_driver_rtc', + + net_arp_p: 'proc_net_arp', + net_dev_p: 'proc_net_dev', + net_if_inet6_p: 'proc_net_if_inet6', + net_igmp_p: 'proc_net_igmp', + net_igmp6_p: 'proc_net_igmp6', + net_netlink_p: 'proc_net_netlink', + net_netstat_p: 'proc_net_netstat', + net_packet_p: 'proc_net_packet', + net_protocols_p: 'proc_net_protocols', + net_route_p: 'proc_net_route', + net_unix_p: 'proc_net_unix', + net_ipv6_route_p: 'proc_net_ipv6_route', # before net_dev_mcast + net_dev_mcast_p: 'proc_net_dev_mcast', # after net_ipv6_route + + pid_fdinfo_p: 'proc_pid_fdinfo', + pid_io_p: 'proc_pid_io', + pid_mountinfo_p: 'proc_pid_mountinfo', + pid_numa_maps_p: 'proc_pid_numa_maps', + pid_stat_p: 'proc_pid_stat', + pid_statm_p: 'proc_pid_statm', + pid_status_p: 'proc_pid_status', + pid_smaps_p: 'proc_pid_smaps', # before pid_maps + pid_maps_p: 'proc_pid_maps', # after pid_smaps + + scsi_device_info: 'proc_scsi_device_info', + scsi_scsi_p: 'proc_scsi_scsi' + } + + for reg_pattern, parse_mod in procmap.items(): + if reg_pattern.search(data): + try: + procparser = importlib.import_module('jc.parsers.' + parse_mod) + return procparser.parse(data, quiet=quiet, raw=raw) + except ModuleNotFoundError: + raise ParseError('Proc file type not yet implemented.') + + raise ParseError('Proc file could not be identified.') diff --git a/jc/parsers/proc_meminfo.py b/jc/parsers/proc_meminfo.py new file mode 100644 index 00000000..4f1a12ba --- /dev/null +++ b/jc/parsers/proc_meminfo.py @@ -0,0 +1,92 @@ +"""jc - JSON Convert `/proc/meminfo` file parser + +Usage (cli): + + $ cat /proc/meminfo | jc --proc + +Usage (module): + + import jc + result = jc.parse('proc_meminfo', proc_meminfo_file) + +Schema: + + [ + { + "foo": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ foo | jc --foo -p + [] + + $ foo | jc --foo -p -r + [] +""" +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/meminfo` command parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + split_line = line.split(':', maxsplit=1) + key = split_line[0] + val = int(split_line[1].rsplit(maxsplit=1)[0]) + raw_output[key] = val + + return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/proc_modules.py b/jc/parsers/proc_modules.py new file mode 100644 index 00000000..366d8250 --- /dev/null +++ b/jc/parsers/proc_modules.py @@ -0,0 +1,172 @@ +"""jc - JSON Convert `/proc/modules` command output parser + +<> + +Usage (cli): + + $ cat /proc/modules | jc --proc + +Usage (module): + + import jc + result = jc.parse('proc_modules', proc_modules_file) + +Schema: + + [ + { + "module": string, + "size": integer, + "used": integer, + "used_by": [ + string + ], + "status": string, + "location": string + } + ] + +Examples: + + $ cat /proc/modules | jc --proc -p + [ + { + "module": "binfmt_misc", + "size": 24576, + "used": 1, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": 16384, + "used": 0, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": 36864, + "used": 1, + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] + + $ proc_modules | jc --proc_modules -p -r + [ + { + "module": "binfmt_misc", + "size": "24576", + "used": "1", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": "16384", + "used": "0", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": "36864", + "used": "1", + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/modules` command parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'size', 'used'} + + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + module, size, used, used_by, status, location = line.split() + used_by_list = used_by.split(',')[:-1] + + raw_output.append( + { + 'module': module, + 'size': size, + 'used': used, + 'used_by': used_by_list, + 'status': status, + 'location': location + } + ) + + return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/procfile.py b/jc/parsers/procfile.py deleted file mode 100644 index 8e5cf2a0..00000000 --- a/jc/parsers/procfile.py +++ /dev/null @@ -1,419 +0,0 @@ -"""jc - JSON Convert Proc file output parser - -<> - -Usage (cli): - - $ cat /proc/ | jc --procfile - -Usage (module): - - import jc - result = jc.parse('procfile', proc_file) - -Schema: - - [ - { - "procfile": string, - "bar": boolean, - "baz": integer - } - ] - -Examples: - - $ procfile | jc --procfile -p - [] - - $ procfile | jc --procfile -p -r - [] -""" -import re -from typing import List, Dict -import jc.utils - - -class info(): - """Provides parser metadata (version, author, etc.)""" - version = '1.0' - description = 'Proc file parser' - author = 'Kelly Brazil' - author_email = 'kellyjonbrazil@gmail.com' - compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] - - -__version__ = info.version - - -def _process(proc_data: List[Dict]) -> List[Dict]: - """ - Final processing to conform to the schema. - - Parameters: - - proc_data: (List of Dictionaries) raw structured data to process - - Returns: - - List of Dictionaries. Structured to conform to the schema. - """ - - # process the data here - # rebuild output for added semantic information - # use helper functions in jc.utils for int, float, bool - # conversions and timestamps - - return proc_data - - -def _parse_uptime(): - print('uptime') - - -def _parse_loadavg(): - print('loadavg') - - -def _parse_cpuinfo(): - print('cpuinfo') - - -def _parse_meminfo(): - print('meminfo') - - -def _parse_version(): - print('version') - - -def _parse_crypto(): - print('crypto') - - -def _parse_diskstats(): - print('diskstats') - - -def _parse_filesystems(): - print('filesystems') - - -def _parse_interrupts(): - print('interrupts') - - -def _parse_iomem(): - print('iomem') - - -def _parse_ioports(): - print('ioports') - - -def _parse_locks(): - print('locks') - - -def _parse_modules(): - print('modules') - - -def _parse_mtrr(): - print('mtrr') - - -def _parse_partitions(): - print('partitions') - - -def _parse_buddyinfo(): - print('buddyinfo') - - -def _parse_pagetypeinfo(): - print('pagetypinfo') - - -def _parse_vmallocinfo(): - print('vmallocinfo') - - -def _parse_softirqs(): - print('softirqs') - - -def _parse_stat(): - print('stat') - - -def _parse_swaps(): - print('swaps') - - -def _parse_slabinfo(): - print('slabinfo') - - -def _parse_consoles(): - print('consoles') - - -def _parse_devices(): - print('devices') - - -def _parse_vmstat(): - print('vmstat') - - -def _parse_zoneinfo(): - print('zoneinfo') - -#### - -def _parse_driver_rtc(): - print('driver rtc') - -#### - -def _parse_net_arp(): - print('net arp') - -def _parse_net_dev(): - print('net dev') - -def _parse_net_dev_mcast(): - print('net dev_mcast') - -def _parse_net_igmp(): - print('net igmp') - -def _parse_net_igmp6(): - print('net igmp6') - -def _parse_net_if_inet6(): - print('net if_inet6') - -def _parse_net_ipv6_route(): - print('net ipv6_route') - -def _parse_net_netlink(): - print('net netlink') - -def _parse_net_netstat(): - print('net netstat') - -def _parse_net_packet(): - print('net packet') - -def _parse_net_protocols(): - print('net protocols') - -def _parse_net_route(): - print('net route') - -def _parse_net_unix(): - print('net unix') - -#### - - -def _parse_pid_status(): - print('pid status') - - -def _parse_pid_statm(): - print('pid statm') - - -def _parse_pid_stat(): - print('pid stat') - - -def _parse_pid_smaps(): - print('pid smaps') - - -def _parse_pid_maps(): - print('pid maps') - - -def _parse_pid_numa_maps(): - print('pid numa_maps') - - -def _parse_pid_io(): - print('pid io') - - -def _parse_pid_mountinfo(): - print('pid mountinfo') - - -def _parse_pid_fdinfo(): - print('pid fdinfo') - -#### - -def _parse_scsi_scsi(): - print('scsi scsi') - -def _parse_scsi_device_info(): - print('scsi device_info') - - -def parse( - data: str, - raw: bool = False, - quiet: bool = False -) -> List[Dict]: - """ - Main text parsing function - - Parameters: - - data: (string) text data to parse - raw: (boolean) unprocessed output if True - quiet: (boolean) suppress warning messages if True - - Returns: - - List of Dictionaries. Raw or processed structured data. - """ - jc.utils.compatibility(__name__, info.compatible, quiet) - jc.utils.input_type_check(data) - - raw_output: List = [] - - if jc.utils.has_data(data): - - # signatures - buddyinfo_p = re.compile(r'^Node \d+, zone\s+\w+\s+(?:\d+\s+){11}\n') - consoles_p = re.compile(r'^\w+\s+[\-WUR]{3} \([ECBpba ]+\)\s+\d+:\d+\n') - cpuinfo_p = re.compile(r'^processor\s+:.*\nvendor_id\s+:.*\ncpu family\s+:.*\n') - crypto_p = re.compile(r'^name\s+:.*\ndriver\s+:.*\nmodule\s+:.*\n') - devices_p = re.compile(r'^Character devices:\n\s+\d+ .*\n') - diskstats_p = re.compile(r'^\s*\d+\s+\d\s\w+\s(?:\d+\s){16}\d\n') - filesystems_p = re.compile(r'^(?:(?:nodev\t|\t)\w+\n){3}') - interrupts_p = re.compile(r'^\s+(?:CPU\d+ +)+\n\s*\d+:\s+\d+') - iomem_p = re.compile(r'^00000000-[0-9a-f]{8} : .*\n[0-9a-f]{8}-[0-9a-f]{8} : ') - ioports_p = re.compile(r'^0000-[0-9a-f]{4} : .*\n\s*0000-[0-9a-f]{4} : ') - loadavg_p = re.compile(r'^\d+.\d\d \d+.\d\d \d+.\d\d \d+/\d+ \d+$') - locks_p = re.compile(r'^\d+: (?:POSIX|FLOCK|OFDLCK)\s+(?:ADVISORY|MANDATORY)\s+(?:READ|WRITE) ') - meminfo_p = re.compile(r'^MemTotal:.*\nMemFree:.*\nMemAvailable:.*\n') - modules_p = re.compile(r'^\w+ \d+ \d+ (?:-|\w+,).*0x[0-9a-f]{16}\n') - mtrr_p = re.compile(r'^reg\d+: base=0x[0-9a-f]+ \(') - pagetypeinfo_p = re.compile(r'^Page block order:\s+\d+\nPages per block:\s+\d+\n\n') - partitions_p = re.compile(r'^major minor #blocks name\n\n\s*\d+\s+\d+\s+\d+ \w+\n') - slabinfo_p = re.compile(r'^slabinfo - version: \d+.\d+\n') - softirqs_p = re.compile(r'^\s+(CPU\d+\s+)+\n\s+HI:\s+\d') - stat_p = re.compile(r'^cpu\s+(?: \d+){10}.*intr ', re.DOTALL) - swaps_p = re.compile(r'^Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n') - uptime_p = re.compile(r'^\d+.\d\d \d+.\d\d$') - version_p = re.compile(r'^.+\sversion\s[^\n]+$') - vmallocinfo_p = re.compile(r'^0x[0-9a-f]{16}-0x[0-9a-f]{16}\s+\d+ \w+\+\w+/\w+ ') - vmstat_p = re.compile(r'nr_free_pages \d+\n.* \d$', re.DOTALL) - zoneinfo_p = re.compile(r'^Node \d+, zone\s+\w+\n') - - driver_rtc_p = re.compile(r'^rtc_time\t: .*\nrtc_date\t: .*\nalrm_time\t: .*\n') - - net_arp_p = re.compile(r'^IP address\s+HW type\s+Flags\s+HW address\s+Mask\s+Device\n') - net_dev_p = re.compile(r'^Inter-\|\s+Receive\s+\|\s+Transmit\n') - net_dev_mcast_p = re.compile(r'^\d+\s+\w+\s+\d+\s+\d+\s+[0-9a-f]{12}') - net_if_inet6_p = re.compile(r'^[0-9a-f]{32} \d\d \d\d \d\d \d\d\s+\w+') - net_igmp_p = re.compile(r'^Idx\tDevice\s+:\s+Count\s+Querier\tGroup\s+Users\s+Timer\tReporter\n') - net_igmp6_p = re.compile(r'^\d+\s+\w+\s+[0-9a-f]{32}\s+\d+\s+[0-9A-F]{8}\s+\d+') - net_ipv6_route_p = re.compile(r'^[0-9a-f]{32} \d\d [0-9a-f]{32} \d\d [0-9a-f]{32} (?:[0-9a-f]{8} ){4}\s+\w+') - net_netlink_p = re.compile(r'^sk\s+Eth Pid\s+Groups\s+Rmem\s+Wmem') - net_netstat_p = re.compile(r'^TcpExt: SyncookiesSent SyncookiesRecv SyncookiesFailed') - net_packet_p = re.compile(r'^sk RefCnt Type Proto Iface R Rmem User Inode\n') - net_protocols_p = re.compile(r'^protocol size sockets memory press maxhdr slab module cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n') - net_route_p = re.compile(r'^Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\s+\n') - net_unix_p = re.compile(r'^Num RefCount Protocol Flags Type St Inode Path\n') - - pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n') - pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$') - pid_stat_p = re.compile(r'^\d+ \(.{1,16}\) \w \d+ \d+ \d+ \d+ -?\d+ (?:\d+ ){43}\d+$') - pid_smaps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ [^\n]+\nSize:\s+\d+ \S\S\n') - pid_maps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ ') - pid_numa_maps_p = re.compile(r'^[a-f0-9]{12} default [^\n]+\n') - pid_io_p = re.compile(r'^rchar: \d+\nwchar: \d+\nsyscr: \d+\n') - pid_mountinfo_p = re.compile(r'^\d+ \d+ \d+:\d+ /.+\n') - pid_fdinfo_p = re.compile(r'^pos:\t\d+\nflags:\t\d+\nmnt_id:\t\d+\n') - - scsi_device_info = re.compile(r"^'\w+' '.+' 0x\d+") - scsi_scsi_p = re.compile(r'^Attached devices:\nHost: \w+ ') - - procmap = { - buddyinfo_p: _parse_buddyinfo, - consoles_p: _parse_consoles, - cpuinfo_p: _parse_cpuinfo, - crypto_p: _parse_crypto, - devices_p: _parse_devices, - diskstats_p: _parse_diskstats, - filesystems_p: _parse_filesystems, - interrupts_p: _parse_interrupts, - iomem_p: _parse_iomem, - ioports_p: _parse_ioports, - loadavg_p: _parse_loadavg, - locks_p: _parse_locks, - meminfo_p: _parse_meminfo, - modules_p: _parse_modules, - mtrr_p: _parse_mtrr, - pagetypeinfo_p: _parse_pagetypeinfo, - partitions_p: _parse_partitions, - slabinfo_p: _parse_slabinfo, - softirqs_p: _parse_softirqs, - stat_p: _parse_stat, - swaps_p: _parse_swaps, - uptime_p: _parse_uptime, - version_p: _parse_version, - vmallocinfo_p: _parse_vmallocinfo, - vmstat_p: _parse_vmstat, - zoneinfo_p: _parse_zoneinfo, - - driver_rtc_p: _parse_driver_rtc, - - net_arp_p: _parse_net_arp, - net_dev_p: _parse_net_dev, - net_if_inet6_p: _parse_net_if_inet6, - net_igmp_p: _parse_net_igmp, - net_igmp6_p: _parse_net_igmp6, - net_netlink_p: _parse_net_netlink, - net_netstat_p: _parse_net_netstat, - net_packet_p: _parse_net_packet, - net_protocols_p: _parse_net_protocols, - net_route_p: _parse_net_route, - net_unix_p: _parse_net_unix, - net_ipv6_route_p: _parse_net_ipv6_route, # before net_dev_mcast - net_dev_mcast_p: _parse_net_dev_mcast, # after net_ipv6_route - - pid_fdinfo_p: _parse_pid_fdinfo, - pid_io_p: _parse_pid_io, - pid_mountinfo_p: _parse_pid_mountinfo, - pid_numa_maps_p: _parse_pid_numa_maps, - pid_stat_p: _parse_pid_stat, - pid_statm_p: _parse_pid_statm, - pid_status_p: _parse_pid_status, - pid_smaps_p: _parse_pid_smaps, # before pid_maps - pid_maps_p: _parse_pid_maps, # after pid_smaps - - scsi_device_info: _parse_scsi_device_info, - scsi_scsi_p: _parse_scsi_scsi - } - - for reg_pattern, parse_func in procmap.items(): - if reg_pattern.search(data): - parse_func() - break - - # for line in filter(None, data.splitlines()): - - # # parse the content here - # # check out helper functions in jc.utils - # # and jc.parsers.universal - - # pass - - return raw_output if raw else _process(raw_output) From a9b0fe6728d0e2eebc192cdb9cd27bfe2d1b9098 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 6 Sep 2022 09:21:40 -0700 Subject: [PATCH 07/71] add 'hidden' attribute to parsers and wire up to jc.lib.all_parser_info() --- jc/lib.py | 22 ++++++++++++++++++++-- jc/parsers/proc_meminfo.py | 1 + jc/parsers/proc_modules.py | 1 + tests/test_jc_lib.py | 5 +++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/jc/lib.py b/jc/lib.py index 0b554691..49a343b7 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -85,6 +85,8 @@ parsers = [ 'plist', 'postconf', 'proc', + 'proc-meminfo', + 'proc-modules', 'ps', 'route', 'rpm-qi', @@ -340,7 +342,9 @@ def parser_info(parser_mod_name: str, documentation: bool = False) -> Dict: return info_dict -def all_parser_info(documentation: bool = False) -> List[Dict]: +def all_parser_info(documentation: bool = False, + show_hidden: bool = False +) -> List[Dict]: """ Returns a list of dictionaries that includes metadata for all parser modules. @@ -348,8 +352,22 @@ def all_parser_info(documentation: bool = False) -> List[Dict]: Parameters: documentation: (boolean) include parser docstrings if True + show_hidden: (boolean) also show parsers marked as hidden + in their info metadata. """ - return [parser_info(p, documentation=documentation) for p in parsers] + temp_list = [parser_info(p, documentation=documentation) for p in parsers] + + p_list = [] + if show_hidden: + p_list = temp_list + + else: + for item in temp_list: + if not item.get('hidden', None): + p_list.append(item) + + return p_list + def get_help(parser_mod_name: str) -> None: """ diff --git a/jc/parsers/proc_meminfo.py b/jc/parsers/proc_meminfo.py index 4f1a12ba..e9030084 100644 --- a/jc/parsers/proc_meminfo.py +++ b/jc/parsers/proc_meminfo.py @@ -38,6 +38,7 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux'] + hidden = True __version__ = info.version diff --git a/jc/parsers/proc_modules.py b/jc/parsers/proc_modules.py index 366d8250..4cc8593e 100644 --- a/jc/parsers/proc_modules.py +++ b/jc/parsers/proc_modules.py @@ -101,6 +101,7 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux'] + hidden = True __version__ = info.version diff --git a/tests/test_jc_lib.py b/tests/test_jc_lib.py index a33a5ae9..609f1c3c 100644 --- a/tests/test_jc_lib.py +++ b/tests/test_jc_lib.py @@ -53,6 +53,11 @@ class MyTests(unittest.TestCase): def test_lib_all_parser_info_length(self): self.assertGreaterEqual(len(jc.lib.all_parser_info()), 80) + def test_lib_all_parser_hidden_length(self): + reg_length = len(jc.lib.all_parser_info()) + hidden_length = len(jc.lib.all_parser_info(show_hidden=True)) + self.assertGreater(hidden_length, reg_length) + def test_lib_plugin_parser_mod_list_is_list(self): self.assertIsInstance(jc.lib.plugin_parser_mod_list(), list) From a764642a8595fa6d8453b5682acefe553f329cf6 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 6 Sep 2022 10:07:19 -0700 Subject: [PATCH 08/71] fixup help, man, readme, about docs --- README.md | 3 +- completions/jc_bash_completion.sh | 2 +- completions/jc_zsh_completion.sh | 3 +- docs/lib.md | 5 +- docs/parsers/proc.md | 60 +++++++++++++++ docs/parsers/proc_meminfo.md | 58 ++++++++++++++ docs/parsers/proc_modules.md | 121 ++++++++++++++++++++++++++++++ jc/cli.py | 24 +++--- man/jc.1 | 19 ++++- readmegen.py | 5 +- templates/manpage_template | 2 +- templates/readme_template | 6 +- 12 files changed, 286 insertions(+), 22 deletions(-) create mode 100644 docs/parsers/proc.md create mode 100644 docs/parsers/proc_meminfo.md create mode 100644 docs/parsers/proc_modules.md diff --git a/README.md b/README.md index c6fa2030..6092740b 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,7 @@ option. | ` --pip-show` | `pip show` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/pip_show) | | ` --plist` | PLIST file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/plist) | | ` --postconf` | `postconf -M` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/postconf) | +| ` --proc` | `/proc/` file parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/proc) | | ` --ps` | `ps` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/ps) | | ` --route` | `route` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/route) | | ` --rpm-qi` | `rpm -qi` command parser | [details](https://kellyjonbrazil.github.io/jc/docs/parsers/rpm_qi) | @@ -278,7 +279,7 @@ option. | `-a` | `--about` | About `jc`. Prints information about `jc` and the parsers (in JSON or YAML, of course!) | | `-C` | `--force-color` | Force color output even when using pipes (overrides `-m` and the `NO_COLOR` env variable) | | `-d` | `--debug` | Debug mode. Prints trace messages if parsing issues are encountered (use`-dd` for verbose debugging) | -| `-h` | `--help` | Help. Use `jc -h --parser_name` for parser documentation | +| `-h` | `--help` | Help. Use `jc -h --parser_name` for parser documentation. Use twice to show hidden parsers (e.g. `-hh`) | | `-m` | `--monochrome` | Monochrome output | | `-M` | `--meta-out` | Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. | | | `-p` | `--pretty` | Pretty format the JSON output | diff --git a/completions/jc_bash_completion.sh b/completions/jc_bash_completion.sh index 3fa432e0..941ffc92 100644 --- a/completions/jc_bash_completion.sh +++ b/completions/jc_bash_completion.sh @@ -4,7 +4,7 @@ _jc() jc_about_options jc_about_mod_options jc_help_options jc_special_options jc_commands=(acpi airport arp blkid chage cksum crontab date df dig dmidecode dpkg du env file finger free git gpg hciconfig id ifconfig iostat iptables iw jobs last lastb ls lsblk lsmod lsof lsusb md5 md5sum mdadm mount mpstat netstat nmcli ntpq pidstat ping ping6 pip pip3 postconf printenv ps route rpm rsync sfdisk sha1sum sha224sum sha256sum sha384sum sha512sum shasum ss stat sum sysctl systemctl systeminfo timedatectl top tracepath tracepath6 traceroute traceroute6 ufw uname update-alternatives upower uptime vdir vmstat w wc who xrandr zipinfo) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --cef-s --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --cef-s --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_options=(--force-color -C --debug -d --monochrome -m --meta-out -M --pretty -p --quiet -q --raw -r --unbuffer -u --yaml-out -y) jc_about_options=(--about -a) jc_about_mod_options=(--pretty -p --yaml-out -y --monochrome -m --force-color -C) diff --git a/completions/jc_zsh_completion.sh b/completions/jc_zsh_completion.sh index ec87a73b..13cddabf 100644 --- a/completions/jc_zsh_completion.sh +++ b/completions/jc_zsh_completion.sh @@ -95,7 +95,7 @@ _jc() { 'xrandr:run "xrandr" command with magic syntax.' 'zipinfo:run "zipinfo" command with magic syntax.' ) - jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --cef-s --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) + jc_parsers=(--acpi --airport --airport-s --arp --asciitable --asciitable-m --blkid --cef --cef-s --chage --cksum --crontab --crontab-u --csv --csv-s --date --df --dig --dir --dmidecode --dpkg-l --du --email-address --env --file --finger --free --fstab --git-log --git-log-s --gpg --group --gshadow --hash --hashsum --hciconfig --history --hosts --id --ifconfig --ini --iostat --iostat-s --ip-address --iptables --iso-datetime --iw-scan --jar-manifest --jobs --jwt --kv --last --ls --ls-s --lsblk --lsmod --lsof --lsusb --m3u --mdadm --mount --mpstat --mpstat-s --netstat --nmcli --ntpq --passwd --pidstat --pidstat-s --ping --ping-s --pip-list --pip-show --plist --postconf --proc --ps --route --rpm-qi --rsync --rsync-s --sfdisk --shadow --ss --stat --stat-s --sysctl --syslog --syslog-s --syslog-bsd --syslog-bsd-s --systemctl --systemctl-lj --systemctl-ls --systemctl-luf --systeminfo --time --timedatectl --timestamp --top --top-s --tracepath --traceroute --ufw --ufw-appinfo --uname --update-alt-gs --update-alt-q --upower --uptime --url --vmstat --vmstat-s --w --wc --who --x509-cert --xml --xrandr --yaml --zipinfo) jc_parsers_describe=( '--acpi:`acpi` command parser' '--airport:`airport -I` command parser' @@ -172,6 +172,7 @@ _jc() { '--pip-show:`pip show` command parser' '--plist:PLIST file parser' '--postconf:`postconf -M` command parser' + '--proc:`/proc/` file parser' '--ps:`ps` command parser' '--route:`route` command parser' '--rpm-qi:`rpm -qi` command parser' diff --git a/docs/lib.md b/docs/lib.md index 25c571bc..a264df2f 100644 --- a/docs/lib.md +++ b/docs/lib.md @@ -162,7 +162,8 @@ Parameters: ### all\_parser\_info ```python -def all_parser_info(documentation: bool = False) -> List[Dict] +def all_parser_info(documentation: bool = False, + show_hidden: bool = False) -> List[Dict] ``` Returns a list of dictionaries that includes metadata for all parser @@ -171,6 +172,8 @@ modules. Parameters: documentation: (boolean) include parser docstrings if True + show_hidden: (boolean) also show parsers marked as hidden + in their info metadata. diff --git a/docs/parsers/proc.md b/docs/parsers/proc.md new file mode 100644 index 00000000..3c3aa392 --- /dev/null +++ b/docs/parsers/proc.md @@ -0,0 +1,60 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc + +jc - JSON Convert Proc file output parser + +<> + +Usage (cli): + + $ cat /proc/ | jc --procfile + +Usage (module): + + import jc + result = jc.parse('procfile', proc_file) + +Schema: + + [ + { + "procfile": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ procfile | jc --procfile -p + [] + + $ procfile | jc --procfile -p -r + [] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/proc_meminfo.md b/docs/parsers/proc_meminfo.md new file mode 100644 index 00000000..95891fe4 --- /dev/null +++ b/docs/parsers/proc_meminfo.md @@ -0,0 +1,58 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_meminfo + +jc - JSON Convert `/proc/meminfo` file parser + +Usage (cli): + + $ cat /proc/meminfo | jc --proc + +Usage (module): + + import jc + result = jc.parse('proc_meminfo', proc_meminfo_file) + +Schema: + + [ + { + "foo": string, + "bar": boolean, + "baz": integer + } + ] + +Examples: + + $ foo | jc --foo -p + [] + + $ foo | jc --foo -p -r + [] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/proc_modules.md b/docs/parsers/proc_modules.md new file mode 100644 index 00000000..658fa12f --- /dev/null +++ b/docs/parsers/proc_modules.md @@ -0,0 +1,121 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_modules + +jc - JSON Convert `/proc/modules` command output parser + +<> + +Usage (cli): + + $ cat /proc/modules | jc --proc + +Usage (module): + + import jc + result = jc.parse('proc_modules', proc_modules_file) + +Schema: + + [ + { + "module": string, + "size": integer, + "used": integer, + "used_by": [ + string + ], + "status": string, + "location": string + } + ] + +Examples: + + $ cat /proc/modules | jc --proc -p + [ + { + "module": "binfmt_misc", + "size": 24576, + "used": 1, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": 16384, + "used": 0, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": 36864, + "used": 1, + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] + + $ proc_modules | jc --proc_modules -p -r + [ + { + "module": "binfmt_misc", + "size": "24576", + "used": "1", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": "16384", + "used": "0", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": "36864", + "used": "1", + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/cli.py b/jc/cli.py index a53ad99d..b272f277 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -11,8 +11,9 @@ import signal import shlex import subprocess from .lib import (__version__, parser_info, all_parser_info, parsers, - _get_parser, _parser_is_streaming, standard_parser_mod_list, - plugin_parser_mod_list, streaming_parser_mod_list) + _get_parser, _parser_is_streaming, parser_mod_list, + standard_parser_mod_list, plugin_parser_mod_list, + streaming_parser_mod_list) from . import utils from .cli_data import long_options_map, new_pygments_colors, old_pygments_colors from .shell_completions import bash_completion, zsh_completion @@ -120,11 +121,11 @@ def parser_shortname(parser_arg): return parser_arg[2:] -def parsers_text(indent=0, pad=0): +def parsers_text(indent=0, pad=0, show_hidden=False): """Return the argument and description information from each parser""" ptext = '' padding_char = ' ' - for p in all_parser_info(): + for p in all_parser_info(show_hidden=show_hidden): parser_arg = p.get('argument', 'UNKNOWN') padding = pad - len(parser_arg) parser_desc = p.get('description', 'No description available.') @@ -164,17 +165,17 @@ def about_jc(): 'license': info.license, 'python_version': '.'.join((str(sys.version_info.major), str(sys.version_info.minor), str(sys.version_info.micro))), 'python_path': sys.executable, - 'parser_count': len(all_parser_info()), + 'parser_count': len(parser_mod_list()), 'standard_parser_count': len(standard_parser_mod_list()), 'streaming_parser_count': len(streaming_parser_mod_list()), 'plugin_parser_count': len(plugin_parser_mod_list()), - 'parsers': all_parser_info() + 'parsers': all_parser_info(show_hidden=True) } -def helptext(): +def helptext(show_hidden=False): """Return the help text with the list of parsers""" - parsers_string = parsers_text(indent=4, pad=20) + parsers_string = parsers_text(indent=4, pad=20, show_hidden=show_hidden) options_string = options_text(indent=4, pad=20) helptext_string = f'''\ @@ -205,7 +206,7 @@ Examples: return helptext_string -def help_doc(options): +def help_doc(options, show_hidden=False): """ Returns the parser documentation if a parser is found in the arguments, otherwise the general help text is returned. @@ -228,7 +229,7 @@ def help_doc(options): utils._safe_pager(doc_text) return - utils._safe_print(helptext()) + utils._safe_print(helptext(show_hidden=show_hidden)) return @@ -525,6 +526,7 @@ def main(): force_color = 'C' in options mono = ('m' in options or bool(os.getenv('NO_COLOR'))) and not force_color help_me = 'h' in options + verbose_help = options.count('h') > 1 pretty = 'p' in options quiet = 'q' in options ignore_exceptions = options.count('q') > 1 @@ -552,7 +554,7 @@ def main(): sys.exit(0) if help_me: - help_doc(sys.argv) + help_doc(sys.argv, show_hidden=verbose_help) sys.exit(0) if version_info: diff --git a/man/jc.1 b/man/jc.1 index 8fb1459d..92497f32 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-08-29 1.21.2 "JSON Convert" +.TH jc 1 2022-09-06 1.21.2 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -392,6 +392,21 @@ PLIST file parser \fB--postconf\fP `postconf -M` command parser +.TP +.B +\fB--proc\fP +`/proc/` file parser + +.TP +.B +\fB--proc-meminfo\fP +`/proc/meminfo` command parser + +.TP +.B +\fB--proc-modules\fP +`/proc/modules` command parser + .TP .B \fB--ps\fP @@ -639,7 +654,7 @@ Debug - show traceback (use \fB-dd\fP for verbose traceback) .TP .B \fB-h\fP, \fB--help\fP -Help (\fB--help --parser_name\fP for parser documentation) +Help (\fB--help --parser_name\fP for parser documentation). Use twice to show hidden parsers (e.g. \fB-hh\fP) .TP .B \fB-m\fP, \fB--monochrome\fP diff --git a/readmegen.py b/readmegen.py index 5982d836..b408b93f 100755 --- a/readmegen.py +++ b/readmegen.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 # Genereate README.md from jc metadata using jinja2 templates import jc.cli +import jc.lib from jinja2 import Environment, FileSystemLoader file_loader = FileSystemLoader('templates') env = Environment(loader=file_loader) template = env.get_template('readme_template') -output = template.render(jc=jc.cli.about_jc()) +# output = template.render(jc=jc.cli.about_jc()) +output = template.render(parsers=jc.lib.all_parser_info(), + info=jc.cli.about_jc()) with open('README.md', 'w') as f: f.write(output) diff --git a/templates/manpage_template b/templates/manpage_template index c1bfacf6..03ebcd57 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -44,7 +44,7 @@ Debug - show traceback (use \fB-dd\fP for verbose traceback) .TP .B \fB-h\fP, \fB--help\fP -Help (\fB--help --parser_name\fP for parser documentation) +Help (\fB--help --parser_name\fP for parser documentation). Use twice to show hidden parsers (e.g. \fB-hh\fP) .TP .B \fB-m\fP, \fB--monochrome\fP diff --git a/templates/readme_template b/templates/readme_template index 963c8176..6e98f351 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -149,7 +149,7 @@ option. ### Parsers | Argument | Command or Filetype | Documentation | -|-------------------|---------------------------------------------------------|----------------------------------------------------------------------------|{% for parser in jc.parsers %} +|-------------------|---------------------------------------------------------|----------------------------------------------------------------------------|{% for parser in parsers %} | `{{ "{:>15}".format(parser.argument) }}` | {{ "{:<55}".format(parser.description) }} | {{ "{:<74}".format("[details](https://kellyjonbrazil.github.io/jc/docs/parsers/" + parser.name + ")") }} |{% endfor %} ### Options @@ -159,7 +159,7 @@ option. | `-a` | `--about` | About `jc`. Prints information about `jc` and the parsers (in JSON or YAML, of course!) | | `-C` | `--force-color` | Force color output even when using pipes (overrides `-m` and the `NO_COLOR` env variable) | | `-d` | `--debug` | Debug mode. Prints trace messages if parsing issues are encountered (use`-dd` for verbose debugging) | -| `-h` | `--help` | Help. Use `jc -h --parser_name` for parser documentation | +| `-h` | `--help` | Help. Use `jc -h --parser_name` for parser documentation. Use twice to show hidden parsers (e.g. `-hh`) | | `-m` | `--monochrome` | Monochrome output | | `-M` | `--meta-out` | Add metadata to output including timestamp, parser name, magic command, magic command exit code, etc. | | | `-p` | `--pretty` | Pretty format the JSON output | @@ -1104,4 +1104,4 @@ cat istio.yaml | jc --yaml -p ] ``` -{{ jc.copyright }} +{{ info.copyright }} From 5c354b02ea103e4a166f28c98ea8728c50ef5ed4 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 6 Sep 2022 11:42:24 -0700 Subject: [PATCH 09/71] doc update --- docs/parsers/proc.md | 37 ++++++++++++----- docs/parsers/proc_meminfo.md | 80 ++++++++++++++++++++++++++++++------ docs/parsers/proc_modules.md | 11 ++++- jc/parsers/proc.py | 39 +++++++++++++----- jc/parsers/proc_meminfo.py | 80 ++++++++++++++++++++++++++++++------ jc/parsers/proc_modules.py | 11 ++++- 6 files changed, 209 insertions(+), 49 deletions(-) diff --git a/docs/parsers/proc.md b/docs/parsers/proc.md index 3c3aa392..38f977a1 100644 --- a/docs/parsers/proc.md +++ b/docs/parsers/proc.md @@ -5,26 +5,43 @@ jc - JSON Convert Proc file output parser -<> +This parser automatically identifies the Proc file and calls the +corresponding parser to peform the parsing. The specific parsers can also +be called directly, if desired and have a naming convention of +`proc-` (cli) or `proc_` (module). Usage (cli): - $ cat /proc/ | jc --procfile + $ cat /proc/meminfo | jc --proc + +or + + $ cat /proc/meminfo | jc --proc-memifno Usage (module): import jc - result = jc.parse('procfile', proc_file) + result = jc.parse('proc', proc_file) Schema: - [ - { - "procfile": string, - "bar": boolean, - "baz": integer - } - ] +See the specific Proc parser for the schema: + + $ jc --help --proc- + +For example: + + $ jc --help --proc-meminfo + +Specific Proc file parser names can be found with `jc -hh` or `jc -a`. + +Schemas can also be found online at: + + https://kellyjonbrazil.github.io/jc/docs/parsers/proc_ + +For example: + + https://kellyjonbrazil.github.io/jc/docs/parsers/proc_meminfo Examples: diff --git a/docs/parsers/proc_meminfo.md b/docs/parsers/proc_meminfo.md index 95891fe4..edf76d2e 100644 --- a/docs/parsers/proc_meminfo.md +++ b/docs/parsers/proc_meminfo.md @@ -9,28 +9,84 @@ Usage (cli): $ cat /proc/meminfo | jc --proc +or + + $ cat /proc/meminfo | jc --proc-meminfo + Usage (module): + import jc + result = jc.parse('proc', proc_meminfo_file) + +or + import jc result = jc.parse('proc_meminfo', proc_meminfo_file) Schema: - [ - { - "foo": string, - "bar": boolean, - "baz": integer - } - ] +All values are integers. + + { + integer + } Examples: - $ foo | jc --foo -p - [] - - $ foo | jc --foo -p -r - [] + $ cat /proc/meminfo | jc --proc -p + { + "MemTotal": 3997272, + "MemFree": 2760316, + "MemAvailable": 3386876, + "Buffers": 40452, + "Cached": 684856, + "SwapCached": 0, + "Active": 475816, + "Inactive": 322064, + "Active(anon)": 70216, + "Inactive(anon)": 148, + "Active(file)": 405600, + "Inactive(file)": 321916, + "Unevictable": 19476, + "Mlocked": 19476, + "SwapTotal": 3996668, + "SwapFree": 3996668, + "Dirty": 152, + "Writeback": 0, + "AnonPages": 92064, + "Mapped": 79464, + "Shmem": 1568, + "KReclaimable": 188216, + "Slab": 288096, + "SReclaimable": 188216, + "SUnreclaim": 99880, + "KernelStack": 5872, + "PageTables": 1812, + "NFS_Unstable": 0, + "Bounce": 0, + "WritebackTmp": 0, + "CommitLimit": 5995304, + "Committed_AS": 445240, + "VmallocTotal": 34359738367, + "VmallocUsed": 21932, + "VmallocChunk": 0, + "Percpu": 107520, + "HardwareCorrupted": 0, + "AnonHugePages": 0, + "ShmemHugePages": 0, + "ShmemPmdMapped": 0, + "FileHugePages": 0, + "FilePmdMapped": 0, + "HugePages_Total": 0, + "HugePages_Free": 0, + "HugePages_Rsvd": 0, + "HugePages_Surp": 0, + "Hugepagesize": 2048, + "Hugetlb": 0, + "DirectMap4k": 192320, + "DirectMap2M": 4001792, + "DirectMap1G": 2097152 + } diff --git a/docs/parsers/proc_modules.md b/docs/parsers/proc_modules.md index 658fa12f..7497d141 100644 --- a/docs/parsers/proc_modules.md +++ b/docs/parsers/proc_modules.md @@ -5,14 +5,21 @@ jc - JSON Convert `/proc/modules` command output parser -<> - Usage (cli): $ cat /proc/modules | jc --proc +or + + $ cat /proc/modules | jc --proc-modules + Usage (module): + import jc + result = jc.parse('proc', proc_modules_file) + +or + import jc result = jc.parse('proc_modules', proc_modules_file) diff --git a/jc/parsers/proc.py b/jc/parsers/proc.py index 4793bf9d..46735933 100644 --- a/jc/parsers/proc.py +++ b/jc/parsers/proc.py @@ -1,25 +1,42 @@ """jc - JSON Convert Proc file output parser -<> +This parser automatically identifies the Proc file and calls the +corresponding parser to peform the parsing. The specific parsers can also +be called directly, if desired and have a naming convention of +`proc-` (cli) or `proc_` (module). Usage (cli): - $ cat /proc/ | jc --procfile + $ cat /proc/meminfo | jc --proc + +or + + $ cat /proc/meminfo | jc --proc-memifno Usage (module): import jc - result = jc.parse('procfile', proc_file) + result = jc.parse('proc', proc_file) Schema: - [ - { - "procfile": string, - "bar": boolean, - "baz": integer - } - ] +See the specific Proc parser for the schema: + + $ jc --help --proc- + +For example: + + $ jc --help --proc-meminfo + +Specific Proc file parser names can be found with `jc -hh` or `jc -a`. + +Schemas can also be found online at: + + https://kellyjonbrazil.github.io/jc/docs/parsers/proc_ + +For example: + + https://kellyjonbrazil.github.io/jc/docs/parsers/proc_meminfo Examples: @@ -139,7 +156,7 @@ def parse( ioports_p: 'proc_ioports', loadavg_p: 'proc_loadavg', locks_p: 'proc_locks', - meminfo_p: 'proc_meminfo', ####### + meminfo_p: 'proc_meminfo', modules_p: 'proc_modules', mtrr_p: 'proc_mtrr', pagetypeinfo_p: 'proc_pagetypeinfo', diff --git a/jc/parsers/proc_meminfo.py b/jc/parsers/proc_meminfo.py index e9030084..f1571441 100644 --- a/jc/parsers/proc_meminfo.py +++ b/jc/parsers/proc_meminfo.py @@ -4,28 +4,84 @@ Usage (cli): $ cat /proc/meminfo | jc --proc +or + + $ cat /proc/meminfo | jc --proc-meminfo + Usage (module): + import jc + result = jc.parse('proc', proc_meminfo_file) + +or + import jc result = jc.parse('proc_meminfo', proc_meminfo_file) Schema: - [ - { - "foo": string, - "bar": boolean, - "baz": integer - } - ] +All values are integers. + + { + integer + } Examples: - $ foo | jc --foo -p - [] - - $ foo | jc --foo -p -r - [] + $ cat /proc/meminfo | jc --proc -p + { + "MemTotal": 3997272, + "MemFree": 2760316, + "MemAvailable": 3386876, + "Buffers": 40452, + "Cached": 684856, + "SwapCached": 0, + "Active": 475816, + "Inactive": 322064, + "Active(anon)": 70216, + "Inactive(anon)": 148, + "Active(file)": 405600, + "Inactive(file)": 321916, + "Unevictable": 19476, + "Mlocked": 19476, + "SwapTotal": 3996668, + "SwapFree": 3996668, + "Dirty": 152, + "Writeback": 0, + "AnonPages": 92064, + "Mapped": 79464, + "Shmem": 1568, + "KReclaimable": 188216, + "Slab": 288096, + "SReclaimable": 188216, + "SUnreclaim": 99880, + "KernelStack": 5872, + "PageTables": 1812, + "NFS_Unstable": 0, + "Bounce": 0, + "WritebackTmp": 0, + "CommitLimit": 5995304, + "Committed_AS": 445240, + "VmallocTotal": 34359738367, + "VmallocUsed": 21932, + "VmallocChunk": 0, + "Percpu": 107520, + "HardwareCorrupted": 0, + "AnonHugePages": 0, + "ShmemHugePages": 0, + "ShmemPmdMapped": 0, + "FileHugePages": 0, + "FilePmdMapped": 0, + "HugePages_Total": 0, + "HugePages_Free": 0, + "HugePages_Rsvd": 0, + "HugePages_Surp": 0, + "Hugepagesize": 2048, + "Hugetlb": 0, + "DirectMap4k": 192320, + "DirectMap2M": 4001792, + "DirectMap1G": 2097152 + } """ from typing import Dict import jc.utils diff --git a/jc/parsers/proc_modules.py b/jc/parsers/proc_modules.py index 4cc8593e..92bb82c4 100644 --- a/jc/parsers/proc_modules.py +++ b/jc/parsers/proc_modules.py @@ -1,13 +1,20 @@ """jc - JSON Convert `/proc/modules` command output parser -<> - Usage (cli): $ cat /proc/modules | jc --proc +or + + $ cat /proc/modules | jc --proc-modules + Usage (module): + import jc + result = jc.parse('proc', proc_modules_file) + +or + import jc result = jc.parse('proc_modules', proc_modules_file) From d2895928bd515884eac02888e6aecdaa1045bee9 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 6 Sep 2022 11:46:35 -0700 Subject: [PATCH 10/71] add examples to doc --- docs/parsers/proc.md | 64 +++++++++++++++++++++++++++++++++++++++++--- jc/parsers/proc.py | 64 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 120 insertions(+), 8 deletions(-) diff --git a/docs/parsers/proc.md b/docs/parsers/proc.md index 38f977a1..531d601d 100644 --- a/docs/parsers/proc.md +++ b/docs/parsers/proc.md @@ -45,11 +45,67 @@ For example: Examples: - $ procfile | jc --procfile -p - [] + $ cat /proc/modules | jc --proc -p + [ + { + "module": "binfmt_misc", + "size": 24576, + "used": 1, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": 16384, + "used": 0, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": 36864, + "used": 1, + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] - $ procfile | jc --procfile -p -r - [] + $ proc_modules | jc --proc_modules -p -r + [ + { + "module": "binfmt_misc", + "size": "24576", + "used": "1", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": "16384", + "used": "0", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": "36864", + "used": "1", + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] diff --git a/jc/parsers/proc.py b/jc/parsers/proc.py index 46735933..d0a8495a 100644 --- a/jc/parsers/proc.py +++ b/jc/parsers/proc.py @@ -40,11 +40,67 @@ For example: Examples: - $ procfile | jc --procfile -p - [] + $ cat /proc/modules | jc --proc -p + [ + { + "module": "binfmt_misc", + "size": 24576, + "used": 1, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": 16384, + "used": 0, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": 36864, + "used": 1, + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] - $ procfile | jc --procfile -p -r - [] + $ proc_modules | jc --proc_modules -p -r + [ + { + "module": "binfmt_misc", + "size": "24576", + "used": "1", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": "16384", + "used": "0", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": "36864", + "used": "1", + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] """ import re import importlib From 61cd9acaa2a618872c4c88dfbc3711fed2813622 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 6 Sep 2022 15:19:59 -0700 Subject: [PATCH 11/71] add buddyinfo parser and doc update --- docs/parsers/proc_buddyinfo.md | 124 +++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_buddyinfo.py | 175 +++++++++++++++++++++++++++++++++ jc/parsers/proc_meminfo.py | 2 +- jc/parsers/proc_modules.py | 2 +- man/jc.1 | 9 +- 6 files changed, 309 insertions(+), 4 deletions(-) create mode 100644 docs/parsers/proc_buddyinfo.md create mode 100644 jc/parsers/proc_buddyinfo.py diff --git a/docs/parsers/proc_buddyinfo.md b/docs/parsers/proc_buddyinfo.md new file mode 100644 index 00000000..df66ef22 --- /dev/null +++ b/docs/parsers/proc_buddyinfo.md @@ -0,0 +1,124 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_buddyinfo + +jc - JSON Convert `/proc/buddyinfo` file parser + +Usage (cli): + + $ cat /proc/buddyinfo | jc --proc + +or + + $ cat /proc/buddyinfo | jc --proc-buddyinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_buddyinfo_file) + +or + + import jc + result = jc.parse('proc_buddyinfo', proc_buddyinfo_file) + +Schema: + +All values are integers. + + [ + { + "node": integer, + "zone": string, + "free_chunks": [ + integer # [0] + ] + } + ] + + [0] array index correlates to the Order number. + E.g. free_chunks[0] is the value for Order 0 + + +Examples: + + $ cat /proc/buddyinfo | jc --proc -p + [ + { + "node": 0, + "zone": "DMA", + "free_chunks": [ + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 3 + ] + }, + { + "node": 0, + "zone": "DMA32", + "free_chunks": [ + 78, + 114, + 82, + 52, + 38, + 25, + 13, + 9, + 3, + 4, + 629 + ] + }, + { + "node": 0, + "zone": "Normal", + "free_chunks": [ + 0, + 22, + 8, + 10, + 1, + 1, + 2, + 11, + 13, + 0, + 0 + ] + } + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 49a343b7..51261484 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -85,6 +85,7 @@ parsers = [ 'plist', 'postconf', 'proc', + 'proc-buddyinfo', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_buddyinfo.py b/jc/parsers/proc_buddyinfo.py new file mode 100644 index 00000000..3c2bd44e --- /dev/null +++ b/jc/parsers/proc_buddyinfo.py @@ -0,0 +1,175 @@ +"""jc - JSON Convert `/proc/buddyinfo` file parser + +Usage (cli): + + $ cat /proc/buddyinfo | jc --proc + +or + + $ cat /proc/buddyinfo | jc --proc-buddyinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_buddyinfo_file) + +or + + import jc + result = jc.parse('proc_buddyinfo', proc_buddyinfo_file) + +Schema: + +All values are integers. + + [ + { + "node": integer, + "zone": string, + "free_chunks": [ + integer # [0] + ] + } + ] + + [0] array index correlates to the Order number. + E.g. free_chunks[0] is the value for Order 0 + + +Examples: + + $ cat /proc/buddyinfo | jc --proc -p + [ + { + "node": 0, + "zone": "DMA", + "free_chunks": [ + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 3 + ] + }, + { + "node": 0, + "zone": "DMA32", + "free_chunks": [ + 78, + 114, + 82, + 52, + 38, + 25, + 13, + 9, + 3, + 4, + 629 + ] + }, + { + "node": 0, + "zone": "Normal", + "free_chunks": [ + 0, + 22, + 8, + 10, + 1, + 1, + 2, + 11, + 13, + 0, + 0 + ] + } + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/buddyinfo` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'node'} + + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + + if 'free_chunks' in entry: + entry['free_chunks'] = [int(x) for x in entry['free_chunks']] + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + buddy_list = line.split() + + raw_output.append( + { + 'node': buddy_list[1][:-1], + 'zone': buddy_list[3], + 'free_chunks': buddy_list[4:] + } + ) + + return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/proc_meminfo.py b/jc/parsers/proc_meminfo.py index f1571441..daed6188 100644 --- a/jc/parsers/proc_meminfo.py +++ b/jc/parsers/proc_meminfo.py @@ -90,7 +90,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" version = '1.0' - description = '`/proc/meminfo` command parser' + description = '`/proc/meminfo` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux'] diff --git a/jc/parsers/proc_modules.py b/jc/parsers/proc_modules.py index 92bb82c4..59715dff 100644 --- a/jc/parsers/proc_modules.py +++ b/jc/parsers/proc_modules.py @@ -104,7 +104,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" version = '1.0' - description = '`/proc/modules` command parser' + description = '`/proc/modules` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux'] diff --git a/man/jc.1 b/man/jc.1 index 92497f32..37af3839 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -397,15 +397,20 @@ PLIST file parser \fB--proc\fP `/proc/` file parser +.TP +.B +\fB--proc-buddyinfo\fP +`/proc/buddyinfo` file parser + .TP .B \fB--proc-meminfo\fP -`/proc/meminfo` command parser +`/proc/meminfo` file parser .TP .B \fB--proc-modules\fP -`/proc/modules` command parser +`/proc/modules` file parser .TP .B From 5b4e4fd9432ddb55a2e23d1b35089a12a81559ab Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 6 Sep 2022 15:52:26 -0700 Subject: [PATCH 12/71] add proc-consoles parser and doc update --- docs/parsers/proc_consoles.md | 107 ++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_consoles.py | 184 ++++++++++++++++++++++++++++++++++ man/jc.1 | 5 + 4 files changed, 297 insertions(+) create mode 100644 docs/parsers/proc_consoles.md create mode 100644 jc/parsers/proc_consoles.py diff --git a/docs/parsers/proc_consoles.md b/docs/parsers/proc_consoles.md new file mode 100644 index 00000000..72093124 --- /dev/null +++ b/docs/parsers/proc_consoles.md @@ -0,0 +1,107 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_consoles + +jc - JSON Convert `/proc/consoles` command output parser + +Usage (cli): + + $ cat /proc/consoles | jc --proc + +or + + $ cat /proc/consoles | jc --proc-consoles + +Usage (module): + + import jc + result = jc.parse('proc', proc_consoles_file) + +or + + import jc + result = jc.parse('proc_consoles', proc_consoles_file) + +Schema: + + [ + { + "device": string, + "operations": string, + "operations_list": [ + string # [0] + ], + "flags": string, + "flags_list": [ + string # [1] + ], + "major": integer, + "minor": integer + } + ] + + [0] Values: read, write, unblank + [1] Values: enabled, preferred, primary boot, prink buffer, + braille device, safe when CPU offline + +Examples: + + $ cat /proc/consoles | jc --proc -p + [ + { + "device": "tty0", + "operations": "-WU", + "operations_list": [ + "write", + "unblank" + ], + "flags": "ECp", + "flags_list": [ + "enabled", + "preferred", + "printk buffer" + ], + "major": 4, + "minor": 7 + }, + { + "device": "ttyS0", + "operations": "-W-", + "operations_list": [ + "write" + ], + "flags": "Ep", + "flags_list": [ + "enabled", + "printk buffer" + ], + "major": 4, + "minor": 64 + } + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 51261484..76bd1000 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -86,6 +86,7 @@ parsers = [ 'postconf', 'proc', 'proc-buddyinfo', + 'proc-consoles', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_consoles.py b/jc/parsers/proc_consoles.py new file mode 100644 index 00000000..5724a9b5 --- /dev/null +++ b/jc/parsers/proc_consoles.py @@ -0,0 +1,184 @@ +"""jc - JSON Convert `/proc/consoles` command output parser + +Usage (cli): + + $ cat /proc/consoles | jc --proc + +or + + $ cat /proc/consoles | jc --proc-consoles + +Usage (module): + + import jc + result = jc.parse('proc', proc_consoles_file) + +or + + import jc + result = jc.parse('proc_consoles', proc_consoles_file) + +Schema: + + [ + { + "device": string, + "operations": string, + "operations_list": [ + string # [0] + ], + "flags": string, + "flags_list": [ + string # [1] + ], + "major": integer, + "minor": integer + } + ] + + [0] Values: read, write, unblank + [1] Values: enabled, preferred, primary boot, prink buffer, + braille device, safe when CPU offline + +Examples: + + $ cat /proc/consoles | jc --proc -p + [ + { + "device": "tty0", + "operations": "-WU", + "operations_list": [ + "write", + "unblank" + ], + "flags": "ECp", + "flags_list": [ + "enabled", + "preferred", + "printk buffer" + ], + "major": 4, + "minor": 7 + }, + { + "device": "ttyS0", + "operations": "-W-", + "operations_list": [ + "write" + ], + "flags": "Ep", + "flags_list": [ + "enabled", + "printk buffer" + ], + "major": 4, + "minor": 64 + } + ] +""" +import shlex +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/consoles` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'major', 'minor'} + + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + operations_map = { + 'R': 'read', + 'W': 'write', + 'U': 'unblank' + } + + flags_map = { + 'E': 'enabled', + 'C': 'preferred', + 'B': 'primary boot', + 'p': 'printk buffer', + 'b': 'braille device', + 'a': 'safe when CPU offline' + } + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + # since parens are acting like quotation marks, use shlex.split() + # after converting parens to quotes. + line = line.replace('(', '"'). replace(')', '"') + device, operations, flags, maj_min = shlex.split(line) + + operations_str = operations.replace('-', '') + operations_list = [operations_map[i] for i in operations_str] + + flags_str = flags.replace (' ', '') + flags_list = [flags_map[i] for i in flags_str] + + raw_output.append( + { + 'device': device, + 'operations': operations, + 'operations_list': operations_list, + 'flags': flags, + 'flags_list': flags_list, + 'major': maj_min.split(':')[0], + 'minor': maj_min.split(':')[1] + } + ) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 37af3839..3dc3f98f 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -402,6 +402,11 @@ PLIST file parser \fB--proc-buddyinfo\fP `/proc/buddyinfo` file parser +.TP +.B +\fB--proc-consoles\fP +`/proc/consoles` file parser + .TP .B \fB--proc-meminfo\fP From b83cd24d5742a7660b90bfc7d80489ef182181e7 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 6 Sep 2022 17:05:14 -0700 Subject: [PATCH 13/71] add proc-cpuinfo parser and doc update --- docs/parsers/proc_consoles.md | 2 +- docs/parsers/proc_cpuinfo.md | 247 +++++++++++++++++++++++++ docs/parsers/proc_modules.md | 2 +- jc/lib.py | 1 + jc/parsers/proc.py | 2 +- jc/parsers/proc_consoles.py | 2 +- jc/parsers/proc_cpuinfo.py | 333 ++++++++++++++++++++++++++++++++++ jc/parsers/proc_modules.py | 2 +- man/jc.1 | 5 + 9 files changed, 591 insertions(+), 5 deletions(-) create mode 100644 docs/parsers/proc_cpuinfo.md create mode 100644 jc/parsers/proc_cpuinfo.py diff --git a/docs/parsers/proc_consoles.md b/docs/parsers/proc_consoles.md index 72093124..e3441ec1 100644 --- a/docs/parsers/proc_consoles.md +++ b/docs/parsers/proc_consoles.md @@ -3,7 +3,7 @@ # jc.parsers.proc\_consoles -jc - JSON Convert `/proc/consoles` command output parser +jc - JSON Convert `/proc/consoles` file parser Usage (cli): diff --git a/docs/parsers/proc_cpuinfo.md b/docs/parsers/proc_cpuinfo.md new file mode 100644 index 00000000..3bf9c967 --- /dev/null +++ b/docs/parsers/proc_cpuinfo.md @@ -0,0 +1,247 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_cpuinfo + +jc - JSON Convert `/proc/cpuinfo` file parser + +Usage (cli): + + $ cat /proc/cpuinfo | jc --proc + +or + + $ cat /proc/cpuinfo | jc --proc-cpuinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_cpuinfo_file) + +or + + import jc + result = jc.parse('proc_cpuinfo', proc_cpuinfo_file) + +Schema: + +Integer, float, and boolean ("yes"/"no") conversions are attempted. Blank +strings are converted to `null`. + +"Well-known" keys like `cache size`, `address types`, `bugs`, and `flags` +are processed into sensible data types. (see below) + +If this is not desired, then use the `--raw` (CLI) or `raw=True` (Module) +option. + + [ + { + "processor": integer, + "address sizes": string, + "address_size_physical": integer, # in bits + "address_size_virtual": integer, # in bits + "cache size": string, + "cache_size_num": integer, + "cache_size_unit": string, + "flags": string, + "flag_list": [ + string + ], + "bugs": string, + "bug_list": [ + string + ], + "bogomips": float, + : string/int/float/boolean/null + } + ] + +Examples: + + $ cat /proc/cpuinfo | jc --proc -p + [ + { + "processor": 0, + "vendor_id": "GenuineIntel", + "cpu family": 6, + "model": 142, + "model name": "Intel(R) Core(TM) i5-8279U CPU @ 2.40GHz", + "stepping": 10, + "cpu MHz": 2400.0, + "cache size": "6144 KB", + "physical id": 0, + "siblings": 1, + "core id": 0, + "cpu cores": 1, + "apicid": 0, + "initial apicid": 0, + "fpu": true, + "fpu_exception": true, + "cpuid level": 22, + "wp": true, + "flags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr ...", + "bugs": "cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass ...", + "bogomips": 4800.0, + "clflush size": 64, + "cache_alignment": 64, + "address sizes": "45 bits physical, 48 bits virtual", + "power management": null, + "address_size_physical": 45, + "address_size_virtual": 48, + "cache_size_num": 6144, + "cache_size_unit": "KB", + "flag_list": [ + "fpu", + "vme", + "de", + "pse", + "tsc", + "msr", + "pae", + "mce", + "cx8", + "apic", + "sep", + "mtrr", + "pge", + "mca", + "cmov", + "pat", + "pse36", + "clflush", + "mmx", + "fxsr", + "sse", + "sse2", + "ss", + "syscall", + "nx", + "pdpe1gb", + "rdtscp", + "lm", + "constant_tsc", + "arch_perfmon", + "nopl", + "xtopology", + "tsc_reliable", + "nonstop_tsc", + "cpuid", + "pni", + "pclmulqdq", + "ssse3", + "fma", + "cx16", + "pcid", + "sse4_1", + "sse4_2", + "x2apic", + "movbe", + "popcnt", + "tsc_deadline_timer", + "aes", + "xsave", + "avx", + "f16c", + "rdrand", + "hypervisor", + "lahf_lm", + "abm", + "3dnowprefetch", + "cpuid_fault", + "invpcid_single", + "pti", + "ssbd", + "ibrs", + "ibpb", + "stibp", + "fsgsbase", + "tsc_adjust", + "bmi1", + "avx2", + "smep", + "bmi2", + "invpcid", + "rdseed", + "adx", + "smap", + "clflushopt", + "xsaveopt", + "xsavec", + "xgetbv1", + "xsaves", + "arat", + "md_clear", + "flush_l1d", + "arch_capabilities" + ], + "bug_list": [ + "cpu_meltdown", + "spectre_v1", + "spectre_v2", + "spec_store_bypass", + "l1tf", + "mds", + "swapgs", + "itlb_multihit", + "srbds" + ] + }, + ... + ] + + $ proc_cpuinfo | jc --proc_cpuinfo -p -r + [ + { + "processor": "0", + "vendor_id": "GenuineIntel", + "cpu family": "6", + "model": "142", + "model name": "Intel(R) Core(TM) i5-8279U CPU @ 2.40GHz", + "stepping": "10", + "cpu MHz": "2400.000", + "cache size": "6144 KB", + "physical id": "0", + "siblings": "1", + "core id": "0", + "cpu cores": "1", + "apicid": "0", + "initial apicid": "0", + "fpu": "yes", + "fpu_exception": "yes", + "cpuid level": "22", + "wp": "yes", + "flags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge ...", + "bugs": "cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass ...", + "bogomips": "4800.00", + "clflush size": "64", + "cache_alignment": "64", + "address sizes": "45 bits physical, 48 bits virtual", + "power management": "" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/proc_modules.md b/docs/parsers/proc_modules.md index 7497d141..6fda08cc 100644 --- a/docs/parsers/proc_modules.md +++ b/docs/parsers/proc_modules.md @@ -3,7 +3,7 @@ # jc.parsers.proc\_modules -jc - JSON Convert `/proc/modules` command output parser +jc - JSON Convert `/proc/modules` file parser Usage (cli): diff --git a/jc/lib.py b/jc/lib.py index 76bd1000..9cecbe5d 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -87,6 +87,7 @@ parsers = [ 'proc', 'proc-buddyinfo', 'proc-consoles', + 'proc-cpuinfo', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc.py b/jc/parsers/proc.py index d0a8495a..0317ff53 100644 --- a/jc/parsers/proc.py +++ b/jc/parsers/proc.py @@ -145,7 +145,7 @@ def parse( # signatures buddyinfo_p = re.compile(r'^Node \d+, zone\s+\w+\s+(?:\d+\s+){11}\n') consoles_p = re.compile(r'^\w+\s+[\-WUR]{3} \([ECBpba ]+\)\s+\d+:\d+\n') - cpuinfo_p = re.compile(r'^processor\s+:.*\nvendor_id\s+:.*\ncpu family\s+:.*\n') + cpuinfo_p = re.compile(r'^processor\t+: \d+.*bogomips\t+: \d+.\d\d\n', re.DOTALL) crypto_p = re.compile(r'^name\s+:.*\ndriver\s+:.*\nmodule\s+:.*\n') devices_p = re.compile(r'^Character devices:\n\s+\d+ .*\n') diskstats_p = re.compile(r'^\s*\d+\s+\d\s\w+\s(?:\d+\s){16}\d\n') diff --git a/jc/parsers/proc_consoles.py b/jc/parsers/proc_consoles.py index 5724a9b5..5f517f4b 100644 --- a/jc/parsers/proc_consoles.py +++ b/jc/parsers/proc_consoles.py @@ -1,4 +1,4 @@ -"""jc - JSON Convert `/proc/consoles` command output parser +"""jc - JSON Convert `/proc/consoles` file parser Usage (cli): diff --git a/jc/parsers/proc_cpuinfo.py b/jc/parsers/proc_cpuinfo.py new file mode 100644 index 00000000..a059eea5 --- /dev/null +++ b/jc/parsers/proc_cpuinfo.py @@ -0,0 +1,333 @@ +"""jc - JSON Convert `/proc/cpuinfo` file parser + +Usage (cli): + + $ cat /proc/cpuinfo | jc --proc + +or + + $ cat /proc/cpuinfo | jc --proc-cpuinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_cpuinfo_file) + +or + + import jc + result = jc.parse('proc_cpuinfo', proc_cpuinfo_file) + +Schema: + +Integer, float, and boolean ("yes"/"no") conversions are attempted. Blank +strings are converted to `null`. + +"Well-known" keys like `cache size`, `address types`, `bugs`, and `flags` +are processed into sensible data types. (see below) + +If this is not desired, then use the `--raw` (CLI) or `raw=True` (Module) +option. + + [ + { + "processor": integer, + "address sizes": string, + "address_size_physical": integer, # in bits + "address_size_virtual": integer, # in bits + "cache size": string, + "cache_size_num": integer, + "cache_size_unit": string, + "flags": string, + "flag_list": [ + string + ], + "bugs": string, + "bug_list": [ + string + ], + "bogomips": float, + : string/int/float/boolean/null + } + ] + +Examples: + + $ cat /proc/cpuinfo | jc --proc -p + [ + { + "processor": 0, + "vendor_id": "GenuineIntel", + "cpu family": 6, + "model": 142, + "model name": "Intel(R) Core(TM) i5-8279U CPU @ 2.40GHz", + "stepping": 10, + "cpu MHz": 2400.0, + "cache size": "6144 KB", + "physical id": 0, + "siblings": 1, + "core id": 0, + "cpu cores": 1, + "apicid": 0, + "initial apicid": 0, + "fpu": true, + "fpu_exception": true, + "cpuid level": 22, + "wp": true, + "flags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr ...", + "bugs": "cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass ...", + "bogomips": 4800.0, + "clflush size": 64, + "cache_alignment": 64, + "address sizes": "45 bits physical, 48 bits virtual", + "power management": null, + "address_size_physical": 45, + "address_size_virtual": 48, + "cache_size_num": 6144, + "cache_size_unit": "KB", + "flag_list": [ + "fpu", + "vme", + "de", + "pse", + "tsc", + "msr", + "pae", + "mce", + "cx8", + "apic", + "sep", + "mtrr", + "pge", + "mca", + "cmov", + "pat", + "pse36", + "clflush", + "mmx", + "fxsr", + "sse", + "sse2", + "ss", + "syscall", + "nx", + "pdpe1gb", + "rdtscp", + "lm", + "constant_tsc", + "arch_perfmon", + "nopl", + "xtopology", + "tsc_reliable", + "nonstop_tsc", + "cpuid", + "pni", + "pclmulqdq", + "ssse3", + "fma", + "cx16", + "pcid", + "sse4_1", + "sse4_2", + "x2apic", + "movbe", + "popcnt", + "tsc_deadline_timer", + "aes", + "xsave", + "avx", + "f16c", + "rdrand", + "hypervisor", + "lahf_lm", + "abm", + "3dnowprefetch", + "cpuid_fault", + "invpcid_single", + "pti", + "ssbd", + "ibrs", + "ibpb", + "stibp", + "fsgsbase", + "tsc_adjust", + "bmi1", + "avx2", + "smep", + "bmi2", + "invpcid", + "rdseed", + "adx", + "smap", + "clflushopt", + "xsaveopt", + "xsavec", + "xgetbv1", + "xsaves", + "arat", + "md_clear", + "flush_l1d", + "arch_capabilities" + ], + "bug_list": [ + "cpu_meltdown", + "spectre_v1", + "spectre_v2", + "spec_store_bypass", + "l1tf", + "mds", + "swapgs", + "itlb_multihit", + "srbds" + ] + }, + ... + ] + + $ proc_cpuinfo | jc --proc_cpuinfo -p -r + [ + { + "processor": "0", + "vendor_id": "GenuineIntel", + "cpu family": "6", + "model": "142", + "model name": "Intel(R) Core(TM) i5-8279U CPU @ 2.40GHz", + "stepping": "10", + "cpu MHz": "2400.000", + "cache size": "6144 KB", + "physical id": "0", + "siblings": "1", + "core id": "0", + "cpu cores": "1", + "apicid": "0", + "initial apicid": "0", + "fpu": "yes", + "fpu_exception": "yes", + "cpuid level": "22", + "wp": "yes", + "flags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge ...", + "bugs": "cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass ...", + "bogomips": "4800.00", + "clflush size": "64", + "cache_alignment": "64", + "address sizes": "45 bits physical, 48 bits virtual", + "power management": "" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/cpuinfo` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'size', 'used'} + + for entry in proc_data: + for key in entry: + if entry[key] == '': + entry[key] = None + + try: + entry[key] = int(entry[key]) + except Exception: + pass + + try: + if isinstance(entry[key], str) and (entry[key] == 'yes' or entry[key] == 'no'): + entry[key] = jc.utils.convert_to_bool(entry[key]) + except Exception: + pass + + try: + if isinstance(entry[key], str) and '.' in entry[key]: + entry[key] = float(entry[key]) + except Exception: + pass + + if 'address sizes' in entry: + phy = int(entry['address sizes'].split()[0]) + virt = int(entry['address sizes'].split()[3]) + entry['address_size_physical'] = phy + entry['address_size_virtual'] = virt + + if 'cache size' in entry: + cache_size_int, unit = entry['cache size'].split() + entry['cache_size_num'] = int(cache_size_int) + entry['cache_size_unit'] = unit + + if 'flags' in entry: + flag_list = entry['flags'].split() + entry['flag_list'] = flag_list + + if 'bugs' in entry: + bug_list = entry['bugs'].split() + entry['bug_list'] = bug_list + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + output_line: Dict = {} + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + if line.startswith('processor'): + if output_line: + raw_output.append(output_line) + output_line = {} + + key, val = line.split(':', maxsplit=1) + output_line[key.strip()] = val.strip() + + if output_line: + raw_output.append(output_line) + + return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/proc_modules.py b/jc/parsers/proc_modules.py index 59715dff..1404c66a 100644 --- a/jc/parsers/proc_modules.py +++ b/jc/parsers/proc_modules.py @@ -1,4 +1,4 @@ -"""jc - JSON Convert `/proc/modules` command output parser +"""jc - JSON Convert `/proc/modules` file parser Usage (cli): diff --git a/man/jc.1 b/man/jc.1 index 3dc3f98f..f21b2ea9 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -407,6 +407,11 @@ PLIST file parser \fB--proc-consoles\fP `/proc/consoles` file parser +.TP +.B +\fB--proc-cpuinfo\fP +`/proc/cpuinfo` file parser + .TP .B \fB--proc-meminfo\fP From 1f2fe65185544a399216f29ce1d5233fef44a920 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 6 Sep 2022 17:29:56 -0700 Subject: [PATCH 14/71] add proc-crypto parser and doc update --- docs/parsers/proc_crypto.md | 139 ++++++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_cpuinfo.py | 6 +- jc/parsers/proc_crypto.py | 189 ++++++++++++++++++++++++++++++++++++ man/jc.1 | 5 + 5 files changed, 336 insertions(+), 4 deletions(-) create mode 100644 docs/parsers/proc_crypto.md create mode 100644 jc/parsers/proc_crypto.py diff --git a/docs/parsers/proc_crypto.md b/docs/parsers/proc_crypto.md new file mode 100644 index 00000000..119a3378 --- /dev/null +++ b/docs/parsers/proc_crypto.md @@ -0,0 +1,139 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_crypto + +jc - JSON Convert `/proc/crypto` file parser + +Usage (cli): + + $ cat /proc/crypto | jc --proc + +or + + $ cat /proc/crypto | jc --proc-crypto + +Usage (module): + + import jc + result = jc.parse('proc', proc_crypto_file) + +or + + import jc + result = jc.parse('proc_crypto', proc_crypto_file) + +Schema: + +"Well-known" keys like `priority` and `refcnt` are converted to integers. +Also, keynames ending in "size" are converted to integers. + +If this is not desired, then use the `--raw` (CLI) or `raw=True` (Module) +option. + + [ + { + "name": string, + "driver": string, + "module": string, + "priority": integer, + "refcnt": integer, + "selftest": string, + "internal": string, + "type": string, + "*size": integer + } + ] + +Examples: + + $ cat /proc/crypto | jc --proc -p + [ + { + "name": "ecdh", + "driver": "ecdh-generic", + "module": "ecdh_generic", + "priority": 100, + "refcnt": 1, + "selftest": "passed", + "internal": "no", + "type": "kpp" + }, + { + "name": "blake2b-512", + "driver": "blake2b-512-generic", + "module": "blake2b_generic", + "priority": 100, + "refcnt": 1, + "selftest": "passed", + "internal": "no", + "type": "shash", + "blocksize": 128, + "digestsize": 64 + }, + ... + ] + + $ proc_crypto | jc --proc_crypto -p -r + [ + { + "name": "ecdh", + "driver": "ecdh-generic", + "module": "ecdh_generic", + "priority": "100", + "refcnt": "1", + "selftest": "passed", + "internal": "no", + "type": "kpp" + }, + { + "name": "blake2b-512", + "driver": "blake2b-512-generic", + "module": "blake2b_generic", + "priority": "100", + "refcnt": "1", + "selftest": "passed", + "internal": "no", + "type": "shash", + "blocksize": "128", + "digestsize": "64" + }, + { + "name": "blake2b-384", + "driver": "blake2b-384-generic", + "module": "blake2b_generic", + "priority": "100", + "refcnt": "1", + "selftest": "passed", + "internal": "no", + "type": "shash", + "blocksize": "128", + "digestsize": "48" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 9cecbe5d..d5624c06 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -88,6 +88,7 @@ parsers = [ 'proc-buddyinfo', 'proc-consoles', 'proc-cpuinfo', + 'proc-crypto', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_cpuinfo.py b/jc/parsers/proc_cpuinfo.py index a059eea5..0627783f 100644 --- a/jc/parsers/proc_cpuinfo.py +++ b/jc/parsers/proc_cpuinfo.py @@ -245,8 +245,6 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = {'size', 'used'} - for entry in proc_data: for key in entry: if entry[key] == '': @@ -327,7 +325,7 @@ def parse( key, val = line.split(':', maxsplit=1) output_line[key.strip()] = val.strip() - if output_line: - raw_output.append(output_line) + if output_line: + raw_output.append(output_line) return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/proc_crypto.py b/jc/parsers/proc_crypto.py new file mode 100644 index 00000000..8e75c21f --- /dev/null +++ b/jc/parsers/proc_crypto.py @@ -0,0 +1,189 @@ +"""jc - JSON Convert `/proc/crypto` file parser + +Usage (cli): + + $ cat /proc/crypto | jc --proc + +or + + $ cat /proc/crypto | jc --proc-crypto + +Usage (module): + + import jc + result = jc.parse('proc', proc_crypto_file) + +or + + import jc + result = jc.parse('proc_crypto', proc_crypto_file) + +Schema: + +"Well-known" keys like `priority` and `refcnt` are converted to integers. +Also, keynames ending in "size" are converted to integers. + +If this is not desired, then use the `--raw` (CLI) or `raw=True` (Module) +option. + + [ + { + "name": string, + "driver": string, + "module": string, + "priority": integer, + "refcnt": integer, + "selftest": string, + "internal": string, + "type": string, + "*size": integer + } + ] + +Examples: + + $ cat /proc/crypto | jc --proc -p + [ + { + "name": "ecdh", + "driver": "ecdh-generic", + "module": "ecdh_generic", + "priority": 100, + "refcnt": 1, + "selftest": "passed", + "internal": "no", + "type": "kpp" + }, + { + "name": "blake2b-512", + "driver": "blake2b-512-generic", + "module": "blake2b_generic", + "priority": 100, + "refcnt": 1, + "selftest": "passed", + "internal": "no", + "type": "shash", + "blocksize": 128, + "digestsize": 64 + }, + ... + ] + + $ proc_crypto | jc --proc_crypto -p -r + [ + { + "name": "ecdh", + "driver": "ecdh-generic", + "module": "ecdh_generic", + "priority": "100", + "refcnt": "1", + "selftest": "passed", + "internal": "no", + "type": "kpp" + }, + { + "name": "blake2b-512", + "driver": "blake2b-512-generic", + "module": "blake2b_generic", + "priority": "100", + "refcnt": "1", + "selftest": "passed", + "internal": "no", + "type": "shash", + "blocksize": "128", + "digestsize": "64" + }, + { + "name": "blake2b-384", + "driver": "blake2b-384-generic", + "module": "blake2b_generic", + "priority": "100", + "refcnt": "1", + "selftest": "passed", + "internal": "no", + "type": "shash", + "blocksize": "128", + "digestsize": "48" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/crypto` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'priority', 'refcnt'} + + for entry in proc_data: + for key in entry: + if key in int_list or key.endswith('size'): + entry[key] = jc.utils.convert_to_int(entry[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + output_line: Dict = {} + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + if line.startswith('name'): + if output_line: + raw_output.append(output_line) + output_line = {} + + key, val = line.split(':', maxsplit=1) + output_line[key.strip()] = val.strip() + + if output_line: + raw_output.append(output_line) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index f21b2ea9..3ba16bea 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -412,6 +412,11 @@ PLIST file parser \fB--proc-cpuinfo\fP `/proc/cpuinfo` file parser +.TP +.B +\fB--proc-crypto\fP +`/proc/crypto` file parser + .TP .B \fB--proc-meminfo\fP From 966fe97759b0751e92a81556f0ead4ae1b4d8499 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 6 Sep 2022 17:59:26 -0700 Subject: [PATCH 15/71] add proc-devices parser and doc update --- docs/parsers/proc_devices.md | 99 +++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_devices.py | 162 +++++++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 4 files changed, 267 insertions(+) create mode 100644 docs/parsers/proc_devices.md create mode 100644 jc/parsers/proc_devices.py diff --git a/docs/parsers/proc_devices.md b/docs/parsers/proc_devices.md new file mode 100644 index 00000000..b269b321 --- /dev/null +++ b/docs/parsers/proc_devices.md @@ -0,0 +1,99 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_devices + +jc - JSON Convert `/proc/devices` file parser + +Usage (cli): + + $ cat /proc/devices | jc --proc + +or + + $ cat /proc/devices | jc --proc-devices + +Usage (module): + + import jc + result = jc.parse('proc', proc_devices_file) + +or + + import jc + result = jc.parse('proc_devices', proc_devices_file) + +Schema: + +Since devices can be members of multiple groups, the value for each device +is a list. + + { + "character": { + "": [ + string + ] + }, + "block": { + "": [ + string + ] + } + } + +Examples: + + $ cat /proc/devices | jc --proc -p + { + "character": { + "1": [ + "mem" + ], + "4": [ + "/dev/vc/0", + "tty", + "ttyS" + ], + "5": [ + "/dev/tty", + "/dev/console", + "/dev/ptmx", + "ttyprintk" + ], + "block": { + "7": [ + "loop" + ], + "8": [ + "sd" + ], + "9": [ + "md" + ] + } + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index d5624c06..625bdcea 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -89,6 +89,7 @@ parsers = [ 'proc-consoles', 'proc-cpuinfo', 'proc-crypto', + 'proc-devices', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_devices.py b/jc/parsers/proc_devices.py new file mode 100644 index 00000000..deec4866 --- /dev/null +++ b/jc/parsers/proc_devices.py @@ -0,0 +1,162 @@ +"""jc - JSON Convert `/proc/devices` file parser + +Usage (cli): + + $ cat /proc/devices | jc --proc + +or + + $ cat /proc/devices | jc --proc-devices + +Usage (module): + + import jc + result = jc.parse('proc', proc_devices_file) + +or + + import jc + result = jc.parse('proc_devices', proc_devices_file) + +Schema: + +Since devices can be members of multiple groups, the value for each device +is a list. + + { + "character": { + "": [ + string + ] + }, + "block": { + "": [ + string + ] + } + } + +Examples: + + $ cat /proc/devices | jc --proc -p + { + "character": { + "1": [ + "mem" + ], + "4": [ + "/dev/vc/0", + "tty", + "ttyS" + ], + "5": [ + "/dev/tty", + "/dev/console", + "/dev/ptmx", + "ttyprintk" + ], + "block": { + "7": [ + "loop" + ], + "8": [ + "sd" + ], + "9": [ + "md" + ] + } + } +""" +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/devices` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + character: Dict = {} + block: Dict = {} + section = '' + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + if 'Character devices:' in line: + section = 'character' + continue + + if 'Block devices:' in line: + section = 'block' + continue + + devnum, group = line.split() + + if section == 'character': + if not devnum in character: + character[devnum] = [] + + character[devnum].append(group) + continue + + if section == 'block': + if not devnum in block: + block[devnum] = [] + + block[devnum].append(group) + continue + + if character or block: + raw_output = { + 'character': character, + 'block': block + } + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 3ba16bea..33be6f91 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -417,6 +417,11 @@ PLIST file parser \fB--proc-crypto\fP `/proc/crypto` file parser +.TP +.B +\fB--proc-devices\fP +`/proc/devices` file parser + .TP .B \fB--proc-meminfo\fP From cc6287c12451c5f3f618ffaec17cb758804ccab7 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 7 Sep 2022 08:40:37 -0700 Subject: [PATCH 16/71] try/except int conversions --- jc/parsers/proc_crypto.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jc/parsers/proc_crypto.py b/jc/parsers/proc_crypto.py index 8e75c21f..c2c9352f 100644 --- a/jc/parsers/proc_crypto.py +++ b/jc/parsers/proc_crypto.py @@ -142,7 +142,10 @@ def _process(proc_data: List[Dict]) -> List[Dict]: for entry in proc_data: for key in entry: if key in int_list or key.endswith('size'): - entry[key] = jc.utils.convert_to_int(entry[key]) + try: + entry[key] = int(entry[key]) + except Exception: + pass return proc_data From c976c3226d76a07f698412d0cb12529fadab653b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 7 Sep 2022 11:59:54 -0700 Subject: [PATCH 17/71] add magic support for /proc files --- jc/cli.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index b272f277..c8615f90 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -419,6 +419,9 @@ def magic_parser(args): options # jc options to preserve ) +def open_text_file(path_string): + with open(path_string, 'r') as f: + return f.read() def run_user_command(command): """ @@ -571,13 +574,19 @@ def main(): # if magic syntax used, try to run the command and error if it's not found, etc. magic_stdout, magic_stderr, magic_exit_code = None, None, 0 + run_command_str = '' if run_command: try: run_command_str = shlex.join(run_command) # python 3.8+ except AttributeError: run_command_str = ' '.join(run_command) # older python versions - if valid_command: + if run_command_str.startswith('/proc'): + magic_found_parser = 'proc' + magic_stdout = open_text_file(run_command_str) + print(f'this is a procfile. Magic parser={magic_found_parser}, runstring={run_command_str}') + + elif valid_command: try: magic_stdout, magic_stderr, magic_exit_code = run_user_command(run_command) if magic_stderr: @@ -625,12 +634,7 @@ def main(): utils.error_message(['Missing or incorrect arguments. Use "jc -h" for help.']) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) - # check for input errors (pipe vs magic) - if not sys.stdin.isatty() and magic_stdout: - utils.error_message(['Piped data and Magic syntax used simultaneously. Use "jc -h" for help.']) - sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) - - elif sys.stdin.isatty() and magic_stdout is None: + if sys.stdin.isatty() and magic_stdout is None: utils.error_message(['Missing piped data. Use "jc -h" for help.']) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) From c1f36f74557f61d083286983fdabc2aff178f4ce Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 7 Sep 2022 12:31:17 -0700 Subject: [PATCH 18/71] add magic syntax for /proc to docs --- docs/parsers/proc.md | 15 ++++++++++++--- docs/parsers/proc_buddyinfo.md | 4 ++++ docs/parsers/proc_consoles.md | 4 ++++ docs/parsers/proc_cpuinfo.md | 4 ++++ docs/parsers/proc_crypto.md | 4 ++++ docs/parsers/proc_devices.md | 4 ++++ docs/parsers/proc_meminfo.md | 4 ++++ docs/parsers/proc_modules.md | 4 ++++ jc/cli.py | 24 +++++++++++++++++++----- jc/parsers/proc.py | 15 ++++++++++++--- jc/parsers/proc_buddyinfo.py | 4 ++++ jc/parsers/proc_consoles.py | 4 ++++ jc/parsers/proc_cpuinfo.py | 4 ++++ jc/parsers/proc_crypto.py | 4 ++++ jc/parsers/proc_devices.py | 4 ++++ jc/parsers/proc_meminfo.py | 4 ++++ jc/parsers/proc_modules.py | 4 ++++ man/jc.1 | 2 +- 18 files changed, 100 insertions(+), 12 deletions(-) diff --git a/docs/parsers/proc.md b/docs/parsers/proc.md index 531d601d..d51a25d9 100644 --- a/docs/parsers/proc.md +++ b/docs/parsers/proc.md @@ -6,14 +6,23 @@ jc - JSON Convert Proc file output parser This parser automatically identifies the Proc file and calls the -corresponding parser to peform the parsing. The specific parsers can also -be called directly, if desired and have a naming convention of -`proc-` (cli) or `proc_` (module). +corresponding parser to peform the parsing. + +Magic syntax for converting `/proc` files is also supported by running +`jc /proc/`. Any `jc` options must be specified before the +`/proc` path. + +specific Proc file parsers can also be called directly, if desired and have +a naming convention of `proc-` (cli) or `proc_` (module). Usage (cli): $ cat /proc/meminfo | jc --proc +or + + $ jc /proc/meminfo + or $ cat /proc/meminfo | jc --proc-memifno diff --git a/docs/parsers/proc_buddyinfo.md b/docs/parsers/proc_buddyinfo.md index df66ef22..7bed14e5 100644 --- a/docs/parsers/proc_buddyinfo.md +++ b/docs/parsers/proc_buddyinfo.md @@ -9,6 +9,10 @@ Usage (cli): $ cat /proc/buddyinfo | jc --proc +or + + $ jc /proc/buddyinfo + or $ cat /proc/buddyinfo | jc --proc-buddyinfo diff --git a/docs/parsers/proc_consoles.md b/docs/parsers/proc_consoles.md index e3441ec1..db00dced 100644 --- a/docs/parsers/proc_consoles.md +++ b/docs/parsers/proc_consoles.md @@ -9,6 +9,10 @@ Usage (cli): $ cat /proc/consoles | jc --proc +or + + $ jc /proc/consoles + or $ cat /proc/consoles | jc --proc-consoles diff --git a/docs/parsers/proc_cpuinfo.md b/docs/parsers/proc_cpuinfo.md index 3bf9c967..39e39643 100644 --- a/docs/parsers/proc_cpuinfo.md +++ b/docs/parsers/proc_cpuinfo.md @@ -9,6 +9,10 @@ Usage (cli): $ cat /proc/cpuinfo | jc --proc +or + + $ jc /proc/cpuinfo + or $ cat /proc/cpuinfo | jc --proc-cpuinfo diff --git a/docs/parsers/proc_crypto.md b/docs/parsers/proc_crypto.md index 119a3378..835489d7 100644 --- a/docs/parsers/proc_crypto.md +++ b/docs/parsers/proc_crypto.md @@ -9,6 +9,10 @@ Usage (cli): $ cat /proc/crypto | jc --proc +or + + $ jc /proc/crypto + or $ cat /proc/crypto | jc --proc-crypto diff --git a/docs/parsers/proc_devices.md b/docs/parsers/proc_devices.md index b269b321..c5b82a06 100644 --- a/docs/parsers/proc_devices.md +++ b/docs/parsers/proc_devices.md @@ -9,6 +9,10 @@ Usage (cli): $ cat /proc/devices | jc --proc +or + + $ jc /proc/devices + or $ cat /proc/devices | jc --proc-devices diff --git a/docs/parsers/proc_meminfo.md b/docs/parsers/proc_meminfo.md index edf76d2e..45328325 100644 --- a/docs/parsers/proc_meminfo.md +++ b/docs/parsers/proc_meminfo.md @@ -9,6 +9,10 @@ Usage (cli): $ cat /proc/meminfo | jc --proc +or + + $ jc /proc/meminfo + or $ cat /proc/meminfo | jc --proc-meminfo diff --git a/docs/parsers/proc_modules.md b/docs/parsers/proc_modules.md index 6fda08cc..82833a42 100644 --- a/docs/parsers/proc_modules.md +++ b/docs/parsers/proc_modules.md @@ -9,6 +9,10 @@ Usage (cli): $ cat /proc/modules | jc --proc +or + + $ jc /proc/modules + or $ cat /proc/modules | jc --proc-modules diff --git a/jc/cli.py b/jc/cli.py index c8615f90..3077513d 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -179,14 +179,23 @@ def helptext(show_hidden=False): options_string = options_text(indent=4, pad=20) helptext_string = f'''\ -jc converts the output of many commands and file-types to JSON or YAML +jc converts the output of many commands, file-types, and strings to JSON or YAML Usage: - COMMAND | jc PARSER [OPTIONS] - or magic syntax: + Standard syntax: - jc [OPTIONS] COMMAND + COMMAND | jc PARSER [OPTIONS] + + cat FILE | jc PARSER [OPTIONS] + + echo STRING | jc PARSER [OPTIONS] + + Magic syntax: + + jc [OPTIONS] COMMAND + + jc [OPTIONS] /proc/ Parsers: {parsers_string} @@ -195,12 +204,18 @@ Options: Examples: Standard Syntax: $ dig www.google.com | jc --dig --pretty + $ cat /proc/meminfo | jc --proc --pretty Magic Syntax: $ jc --pretty dig www.google.com + $ jc --pretty /proc/meminfo Parser Documentation: $ jc --help --dig + + Show Hidden Parsers: + $ jc -hh + $ jc --about --pretty ''' return helptext_string @@ -584,7 +599,6 @@ def main(): if run_command_str.startswith('/proc'): magic_found_parser = 'proc' magic_stdout = open_text_file(run_command_str) - print(f'this is a procfile. Magic parser={magic_found_parser}, runstring={run_command_str}') elif valid_command: try: diff --git a/jc/parsers/proc.py b/jc/parsers/proc.py index 0317ff53..a4309329 100644 --- a/jc/parsers/proc.py +++ b/jc/parsers/proc.py @@ -1,14 +1,23 @@ """jc - JSON Convert Proc file output parser This parser automatically identifies the Proc file and calls the -corresponding parser to peform the parsing. The specific parsers can also -be called directly, if desired and have a naming convention of -`proc-` (cli) or `proc_` (module). +corresponding parser to peform the parsing. + +Magic syntax for converting `/proc` files is also supported by running +`jc /proc/`. Any `jc` options must be specified before the +`/proc` path. + +specific Proc file parsers can also be called directly, if desired and have +a naming convention of `proc-` (cli) or `proc_` (module). Usage (cli): $ cat /proc/meminfo | jc --proc +or + + $ jc /proc/meminfo + or $ cat /proc/meminfo | jc --proc-memifno diff --git a/jc/parsers/proc_buddyinfo.py b/jc/parsers/proc_buddyinfo.py index 3c2bd44e..1efd5680 100644 --- a/jc/parsers/proc_buddyinfo.py +++ b/jc/parsers/proc_buddyinfo.py @@ -4,6 +4,10 @@ Usage (cli): $ cat /proc/buddyinfo | jc --proc +or + + $ jc /proc/buddyinfo + or $ cat /proc/buddyinfo | jc --proc-buddyinfo diff --git a/jc/parsers/proc_consoles.py b/jc/parsers/proc_consoles.py index 5f517f4b..3423d0b0 100644 --- a/jc/parsers/proc_consoles.py +++ b/jc/parsers/proc_consoles.py @@ -4,6 +4,10 @@ Usage (cli): $ cat /proc/consoles | jc --proc +or + + $ jc /proc/consoles + or $ cat /proc/consoles | jc --proc-consoles diff --git a/jc/parsers/proc_cpuinfo.py b/jc/parsers/proc_cpuinfo.py index 0627783f..16c93892 100644 --- a/jc/parsers/proc_cpuinfo.py +++ b/jc/parsers/proc_cpuinfo.py @@ -4,6 +4,10 @@ Usage (cli): $ cat /proc/cpuinfo | jc --proc +or + + $ jc /proc/cpuinfo + or $ cat /proc/cpuinfo | jc --proc-cpuinfo diff --git a/jc/parsers/proc_crypto.py b/jc/parsers/proc_crypto.py index c2c9352f..f2b4fa4a 100644 --- a/jc/parsers/proc_crypto.py +++ b/jc/parsers/proc_crypto.py @@ -4,6 +4,10 @@ Usage (cli): $ cat /proc/crypto | jc --proc +or + + $ jc /proc/crypto + or $ cat /proc/crypto | jc --proc-crypto diff --git a/jc/parsers/proc_devices.py b/jc/parsers/proc_devices.py index deec4866..4d2a4441 100644 --- a/jc/parsers/proc_devices.py +++ b/jc/parsers/proc_devices.py @@ -4,6 +4,10 @@ Usage (cli): $ cat /proc/devices | jc --proc +or + + $ jc /proc/devices + or $ cat /proc/devices | jc --proc-devices diff --git a/jc/parsers/proc_meminfo.py b/jc/parsers/proc_meminfo.py index daed6188..7b069f12 100644 --- a/jc/parsers/proc_meminfo.py +++ b/jc/parsers/proc_meminfo.py @@ -4,6 +4,10 @@ Usage (cli): $ cat /proc/meminfo | jc --proc +or + + $ jc /proc/meminfo + or $ cat /proc/meminfo | jc --proc-meminfo diff --git a/jc/parsers/proc_modules.py b/jc/parsers/proc_modules.py index 1404c66a..f74e9539 100644 --- a/jc/parsers/proc_modules.py +++ b/jc/parsers/proc_modules.py @@ -4,6 +4,10 @@ Usage (cli): $ cat /proc/modules | jc --proc +or + + $ jc /proc/modules + or $ cat /proc/modules | jc --proc-modules diff --git a/man/jc.1 b/man/jc.1 index 33be6f91..6291f633 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-09-06 1.21.2 "JSON Convert" +.TH jc 1 2022-09-07 1.21.2 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types .SH SYNOPSIS From 4d761d7e8ae2cd38b1e09a05c188b5f25ee7f15e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 7 Sep 2022 12:52:05 -0700 Subject: [PATCH 19/71] add exception handling for file open errors --- jc/cli.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index 3077513d..8a5100cc 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -434,10 +434,12 @@ def magic_parser(args): options # jc options to preserve ) + def open_text_file(path_string): with open(path_string, 'r') as f: return f.read() + def run_user_command(command): """ Use subprocess to run the user's command. Returns the STDOUT, STDERR, @@ -597,8 +599,28 @@ def main(): run_command_str = ' '.join(run_command) # older python versions if run_command_str.startswith('/proc'): - magic_found_parser = 'proc' - magic_stdout = open_text_file(run_command_str) + try: + magic_found_parser = 'proc' + magic_stdout = open_text_file(run_command_str) + + except OSError as e: + if debug: + raise + + error_msg = os.strerror(e.errno) + utils.error_message([ + f'"{run_command_str}" file could not be opened: {error_msg}.' + ]) + sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) + + except Exception: + if debug: + raise + + utils.error_message([ + f'"{run_command_str}" file could not be opened. For details use the -d or -dd option.' + ]) + sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) elif valid_command: try: From c4c159f0560ed7c81fd4e2d031dee4f905d0421b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 7 Sep 2022 13:56:16 -0700 Subject: [PATCH 20/71] clean up examples --- README.md | 37 ++++++++++++++++++++----------------- jc/cli.py | 1 - man/jc.1 | 29 +++++++++++++++++++++++------ templates/manpage_template | 29 +++++++++++++++++++++++------ templates/readme_template | 37 ++++++++++++++++++++----------------- 5 files changed, 86 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 6092740b..cfb329f6 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,9 @@ on Github. `jc` accepts piped input from `STDIN` and outputs a JSON representation of the previous command's output to `STDOUT`. ```bash -COMMAND | jc PARSER [OPTIONS] +COMMAND | jc [OPTIONS] PARSER +cat FILE | jc [OPTIONS] PARSER +echo STRING | jc [OPTIONS] PARSER ``` Alternatively, the "magic" syntax can be used by prepending `jc` to the command @@ -141,6 +143,7 @@ to be converted. Options can be passed to `jc` immediately before the command is given. (Note: command aliases and shell builtins are not supported) ```bash jc [OPTIONS] COMMAND +jc [OPTIONS] /proc/ ``` The JSON output can be compact (default) or pretty formatted with the `-p` @@ -538,12 +541,12 @@ that case you can suppress the warning message with the `-q` cli option or the macOS: ```bash -cat lsof.out | jc --lsof -q +cat lsof.out | jc -q --lsof ``` or Windows: ```bash -type lsof.out | jc --lsof -q +type lsof.out | jc -q --lsof ``` Tested on: @@ -587,7 +590,7 @@ documentation. ### arp ```bash -arp | jc --arp -p # or: jc -p arp +arp | jc -p --arp # or: jc -p arp ``` ```json [ @@ -626,7 +629,7 @@ cat homes.csv ... ``` ```bash -cat homes.csv | jc --csv -p +cat homes.csv | jc -p --csv ``` ```json [ @@ -667,7 +670,7 @@ cat homes.csv | jc --csv -p ``` ### /etc/hosts file ```bash -cat /etc/hosts | jc --hosts -p +cat /etc/hosts | jc -p --hosts ``` ```json [ @@ -694,7 +697,7 @@ cat /etc/hosts | jc --hosts -p ``` ### ifconfig ```bash -ifconfig | jc --ifconfig -p # or: jc -p ifconfig +ifconfig | jc -p --ifconfig # or: jc -p ifconfig ``` ```json [ @@ -752,7 +755,7 @@ Port = 50022 ForwardX11 = no ``` ```bash -cat example.ini | jc --ini -p +cat example.ini | jc -p --ini ``` ```json { @@ -774,7 +777,7 @@ cat example.ini | jc --ini -p ``` ### ls ```bash -$ ls -l /usr/bin | jc --ls -p # or: jc -p ls -l /usr/bin +$ ls -l /usr/bin | jc -p --ls # or: jc -p ls -l /usr/bin ``` ```json [ @@ -810,7 +813,7 @@ $ ls -l /usr/bin | jc --ls -p # or: jc -p ls -l /usr/bin ``` ### netstat ```bash -netstat -apee | jc --netstat -p # or: jc -p netstat -apee +netstat -apee | jc -p --netstat # or: jc -p netstat -apee ``` ```json [ @@ -898,7 +901,7 @@ netstat -apee | jc --netstat -p # or: jc -p netstat -apee ``` ### /etc/passwd file ```bash -cat /etc/passwd | jc --passwd -p +cat /etc/passwd | jc -p --passwd ``` ```json [ @@ -924,7 +927,7 @@ cat /etc/passwd | jc --passwd -p ``` ### ping ```bash -ping 8.8.8.8 -c 3 | jc --ping -p # or: jc -p ping 8.8.8.8 -c 3 +ping 8.8.8.8 -c 3 | jc -p --ping # or: jc -p ping 8.8.8.8 -c 3 ``` ```json { @@ -977,7 +980,7 @@ ping 8.8.8.8 -c 3 | jc --ping -p # or: jc -p ping 8.8.8.8 -c 3 ``` ### ps ```bash -ps axu | jc --ps -p # or: jc -p ps axu +ps axu | jc -p --ps # or: jc -p ps axu ``` ```json [ @@ -1024,7 +1027,7 @@ ps axu | jc --ps -p # or: jc -p ps axu ``` ### traceroute ```bash -traceroute -m 2 8.8.8.8 | jc --traceroute -p +traceroute -m 2 8.8.8.8 | jc -p --traceroute # or: jc -p traceroute -m 2 8.8.8.8 ``` ```json @@ -1089,7 +1092,7 @@ traceroute -m 2 8.8.8.8 | jc --traceroute -p ``` ### uptime ```bash -uptime | jc --uptime -p # or: jc -p uptime +uptime | jc -p --uptime # or: jc -p uptime ``` ```json { @@ -1134,7 +1137,7 @@ cat cd_catalog.xml ... ``` ```bash -cat cd_catalog.xml | jc --xml -p +cat cd_catalog.xml | jc -p --xml ``` ```json { @@ -1186,7 +1189,7 @@ spec: mode: ISTIO_MUTUAL ``` ```bash -cat istio.yaml | jc --yaml -p +cat istio.yaml | jc -p --yaml ``` ```json [ diff --git a/jc/cli.py b/jc/cli.py index 8a5100cc..cab5581c 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -215,7 +215,6 @@ Examples: Show Hidden Parsers: $ jc -hh - $ jc --about --pretty ''' return helptext_string diff --git a/man/jc.1 b/man/jc.1 index 6291f633..0cbd86e9 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,13 +1,26 @@ .TH jc 1 2022-09-07 1.21.2 "JSON Convert" .SH NAME -\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types +\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS -COMMAND | jc PARSER [OPTIONS] -or "Magic" syntax: +Standard syntax: +.RS +COMMAND | \fBjc\fP [OPTIONS] PARSER + +cat FILE | \fBjc\fP [OPTIONS] PARSER + +echo STRING | \fBjc\fP [OPTIONS] PARSER +.RE + +Magic syntax: + +.RS \fBjc\fP [OPTIONS] COMMAND +\fBjc\fP [OPTIONS] /proc/ +.RE + .SH DESCRIPTION \fBjc\fP JSONifies the output of many CLI tools, file-types, and common strings for easier parsing in scripts. \fBjc\fP accepts piped input from \fBSTDIN\fP and outputs a JSON representation of the previous command's output to \fBSTDOUT\fP. Alternatively, the "Magic" syntax can be used by prepending \fBjc\fP to the command to be converted. Options can be passed to \fBjc\fP immediately before the command is given. (Note: "Magic" syntax does not support shell builtins or command aliases) @@ -912,17 +925,21 @@ If a UTC timezone can be detected in the text of the command output, the timesta .SH EXAMPLES Standard Syntax: .RS -$ dig www.google.com | jc \fB--dig\fP \fB-p\fP +$ dig www.google.com | jc \fB-p\fP \fB--dig\fP + +$ cat /proc/meminfo | jc \fB--pretty\fP \fB--proc\fP .RE Magic Syntax: .RS -$ jc \fB-p\fP dig www.google.com +$ jc \fB--pretty\fP dig www.google.com + +$ jc \fB--pretty\fP /proc/meminfo .RE For parser documentation: .RS -$ jc \fB-h\fP \fB--dig\fP +$ jc \fB--help\fP \fB--dig\fP .RE .SH AUTHOR Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/templates/manpage_template b/templates/manpage_template index 03ebcd57..d6a8cc40 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -1,13 +1,26 @@ .TH jc 1 {{ today }} {{ jc.version}} "JSON Convert" .SH NAME -\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types +\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS -COMMAND | jc PARSER [OPTIONS] -or "Magic" syntax: +Standard syntax: +.RS +COMMAND | \fBjc\fP [OPTIONS] PARSER + +cat FILE | \fBjc\fP [OPTIONS] PARSER + +echo STRING | \fBjc\fP [OPTIONS] PARSER +.RE + +Magic syntax: + +.RS \fBjc\fP [OPTIONS] COMMAND +\fBjc\fP [OPTIONS] /proc/ +.RE + .SH DESCRIPTION \fBjc\fP JSONifies the output of many CLI tools, file-types, and common strings for easier parsing in scripts. \fBjc\fP accepts piped input from \fBSTDIN\fP and outputs a JSON representation of the previous command's output to \fBSTDOUT\fP. Alternatively, the "Magic" syntax can be used by prepending \fBjc\fP to the command to be converted. Options can be passed to \fBjc\fP immediately before the command is given. (Note: "Magic" syntax does not support shell builtins or command aliases) @@ -277,17 +290,21 @@ If a UTC timezone can be detected in the text of the command output, the timesta .SH EXAMPLES Standard Syntax: .RS -$ dig www.google.com | jc \fB--dig\fP \fB-p\fP +$ dig www.google.com | jc \fB-p\fP \fB--dig\fP + +$ cat /proc/meminfo | jc \fB--pretty\fP \fB--proc\fP .RE Magic Syntax: .RS -$ jc \fB-p\fP dig www.google.com +$ jc \fB--pretty\fP dig www.google.com + +$ jc \fB--pretty\fP /proc/meminfo .RE For parser documentation: .RS -$ jc \fB-h\fP \fB--dig\fP +$ jc \fB--help\fP \fB--dig\fP .RE .SH AUTHOR {{ jc.author }} ({{ jc.author_email }}) diff --git a/templates/readme_template b/templates/readme_template index 6e98f351..9a5c2c1a 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -133,7 +133,9 @@ on Github. `jc` accepts piped input from `STDIN` and outputs a JSON representation of the previous command's output to `STDOUT`. ```bash -COMMAND | jc PARSER [OPTIONS] +COMMAND | jc [OPTIONS] PARSER +cat FILE | jc [OPTIONS] PARSER +echo STRING | jc [OPTIONS] PARSER ``` Alternatively, the "magic" syntax can be used by prepending `jc` to the command @@ -141,6 +143,7 @@ to be converted. Options can be passed to `jc` immediately before the command is given. (Note: command aliases and shell builtins are not supported) ```bash jc [OPTIONS] COMMAND +jc [OPTIONS] /proc/ ``` The JSON output can be compact (default) or pretty formatted with the `-p` @@ -418,12 +421,12 @@ that case you can suppress the warning message with the `-q` cli option or the macOS: ```bash -cat lsof.out | jc --lsof -q +cat lsof.out | jc -q --lsof ``` or Windows: ```bash -type lsof.out | jc --lsof -q +type lsof.out | jc -q --lsof ``` Tested on: @@ -467,7 +470,7 @@ documentation. ### arp ```bash -arp | jc --arp -p # or: jc -p arp +arp | jc -p --arp # or: jc -p arp ``` ```json [ @@ -506,7 +509,7 @@ cat homes.csv ... ``` ```bash -cat homes.csv | jc --csv -p +cat homes.csv | jc -p --csv ``` ```json [ @@ -547,7 +550,7 @@ cat homes.csv | jc --csv -p ``` ### /etc/hosts file ```bash -cat /etc/hosts | jc --hosts -p +cat /etc/hosts | jc -p --hosts ``` ```json [ @@ -574,7 +577,7 @@ cat /etc/hosts | jc --hosts -p ``` ### ifconfig ```bash -ifconfig | jc --ifconfig -p # or: jc -p ifconfig +ifconfig | jc -p --ifconfig # or: jc -p ifconfig ``` ```json [ @@ -632,7 +635,7 @@ Port = 50022 ForwardX11 = no ``` ```bash -cat example.ini | jc --ini -p +cat example.ini | jc -p --ini ``` ```json { @@ -654,7 +657,7 @@ cat example.ini | jc --ini -p ``` ### ls ```bash -$ ls -l /usr/bin | jc --ls -p # or: jc -p ls -l /usr/bin +$ ls -l /usr/bin | jc -p --ls # or: jc -p ls -l /usr/bin ``` ```json [ @@ -690,7 +693,7 @@ $ ls -l /usr/bin | jc --ls -p # or: jc -p ls -l /usr/bin ``` ### netstat ```bash -netstat -apee | jc --netstat -p # or: jc -p netstat -apee +netstat -apee | jc -p --netstat # or: jc -p netstat -apee ``` ```json [ @@ -778,7 +781,7 @@ netstat -apee | jc --netstat -p # or: jc -p netstat -apee ``` ### /etc/passwd file ```bash -cat /etc/passwd | jc --passwd -p +cat /etc/passwd | jc -p --passwd ``` ```json [ @@ -804,7 +807,7 @@ cat /etc/passwd | jc --passwd -p ``` ### ping ```bash -ping 8.8.8.8 -c 3 | jc --ping -p # or: jc -p ping 8.8.8.8 -c 3 +ping 8.8.8.8 -c 3 | jc -p --ping # or: jc -p ping 8.8.8.8 -c 3 ``` ```json { @@ -857,7 +860,7 @@ ping 8.8.8.8 -c 3 | jc --ping -p # or: jc -p ping 8.8.8.8 -c 3 ``` ### ps ```bash -ps axu | jc --ps -p # or: jc -p ps axu +ps axu | jc -p --ps # or: jc -p ps axu ``` ```json [ @@ -904,7 +907,7 @@ ps axu | jc --ps -p # or: jc -p ps axu ``` ### traceroute ```bash -traceroute -m 2 8.8.8.8 | jc --traceroute -p +traceroute -m 2 8.8.8.8 | jc -p --traceroute # or: jc -p traceroute -m 2 8.8.8.8 ``` ```json @@ -969,7 +972,7 @@ traceroute -m 2 8.8.8.8 | jc --traceroute -p ``` ### uptime ```bash -uptime | jc --uptime -p # or: jc -p uptime +uptime | jc -p --uptime # or: jc -p uptime ``` ```json { @@ -1014,7 +1017,7 @@ cat cd_catalog.xml ... ``` ```bash -cat cd_catalog.xml | jc --xml -p +cat cd_catalog.xml | jc -p --xml ``` ```json { @@ -1066,7 +1069,7 @@ spec: mode: ISTIO_MUTUAL ``` ```bash -cat istio.yaml | jc --yaml -p +cat istio.yaml | jc -p --yaml ``` ```json [ From cfe98506a56f095b59e4d46e48c3a5cdf0cfd0d7 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 7 Sep 2022 14:25:09 -0700 Subject: [PATCH 21/71] fix help examples --- jc/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jc/cli.py b/jc/cli.py index cab5581c..6696651e 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -185,11 +185,11 @@ Usage: Standard syntax: - COMMAND | jc PARSER [OPTIONS] + COMMAND | jc [OPTIONS] PARSER - cat FILE | jc PARSER [OPTIONS] + cat FILE | jc [OPTIONS] PARSER - echo STRING | jc PARSER [OPTIONS] + echo STRING | jc [OPTIONS] PARSER Magic syntax: @@ -203,8 +203,8 @@ Options: {options_string} Examples: Standard Syntax: - $ dig www.google.com | jc --dig --pretty - $ cat /proc/meminfo | jc --proc --pretty + $ dig www.google.com | jc --pretty --dig + $ cat /proc/meminfo | jc --pretty --proc Magic Syntax: $ jc --pretty dig www.google.com From bc9cdadfb06abd57597b5e8dd7a5b8aadb47b7fe Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Wed, 7 Sep 2022 15:23:48 -0700 Subject: [PATCH 22/71] updated schema --- jc/parsers/proc_cpuinfo.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/jc/parsers/proc_cpuinfo.py b/jc/parsers/proc_cpuinfo.py index 16c93892..5e826e99 100644 --- a/jc/parsers/proc_cpuinfo.py +++ b/jc/parsers/proc_cpuinfo.py @@ -42,12 +42,10 @@ option. "cache size": string, "cache_size_num": integer, "cache_size_unit": string, - "flags": string, - "flag_list": [ + "flags": [ string ], - "bugs": string, - "bug_list": [ + "bugs": [ string ], "bogomips": float, @@ -78,8 +76,6 @@ Examples: "fpu_exception": true, "cpuid level": 22, "wp": true, - "flags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr ...", - "bugs": "cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass ...", "bogomips": 4800.0, "clflush size": 64, "cache_alignment": 64, @@ -89,7 +85,7 @@ Examples: "address_size_virtual": 48, "cache_size_num": 6144, "cache_size_unit": "KB", - "flag_list": [ + "flags": [ "fpu", "vme", "de", @@ -173,7 +169,7 @@ Examples: "flush_l1d", "arch_capabilities" ], - "bug_list": [ + "bugs": [ "cpu_meltdown", "spectre_v1", "spectre_v2", @@ -283,12 +279,10 @@ def _process(proc_data: List[Dict]) -> List[Dict]: entry['cache_size_unit'] = unit if 'flags' in entry: - flag_list = entry['flags'].split() - entry['flag_list'] = flag_list + entry['flags'] = entry['flags'].split() if 'bugs' in entry: - bug_list = entry['bugs'].split() - entry['bug_list'] = bug_list + entry['bugs'] = entry['bugs'].split() return proc_data From 6d6054d1dc63e51fe797ea9e596ae772d0748424 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 8 Sep 2022 16:52:26 -0700 Subject: [PATCH 23/71] update compatibility --- jc/parsers/lsof.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/lsof.py b/jc/parsers/lsof.py index ef14cd56..ac167fd0 100644 --- a/jc/parsers/lsof.py +++ b/jc/parsers/lsof.py @@ -124,7 +124,7 @@ class info(): description = '`lsof` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - compatible = ['linux'] + compatible = ['linux', 'darwin', 'aix', 'freebsd'] magic_commands = ['lsof'] From 0a89652ae5a9be7d6d55a3a051ea6a4dd8d86481 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 8 Sep 2022 16:53:26 -0700 Subject: [PATCH 24/71] doc update --- docs/parsers/lsof.md | 2 +- docs/parsers/proc_cpuinfo.md | 12 ++++-------- man/jc.1 | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/parsers/lsof.md b/docs/parsers/lsof.md index 6a0f7945..2fbdc0af 100644 --- a/docs/parsers/lsof.md +++ b/docs/parsers/lsof.md @@ -140,6 +140,6 @@ Returns: List of Dictionaries. Raw or processed structured data. ### Parser Information -Compatibility: linux +Compatibility: linux, darwin, aix, freebsd Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/proc_cpuinfo.md b/docs/parsers/proc_cpuinfo.md index 39e39643..0c24ddc7 100644 --- a/docs/parsers/proc_cpuinfo.md +++ b/docs/parsers/proc_cpuinfo.md @@ -47,12 +47,10 @@ option. "cache size": string, "cache_size_num": integer, "cache_size_unit": string, - "flags": string, - "flag_list": [ + "flags": [ string ], - "bugs": string, - "bug_list": [ + "bugs": [ string ], "bogomips": float, @@ -83,8 +81,6 @@ Examples: "fpu_exception": true, "cpuid level": 22, "wp": true, - "flags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr ...", - "bugs": "cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass ...", "bogomips": 4800.0, "clflush size": 64, "cache_alignment": 64, @@ -94,7 +90,7 @@ Examples: "address_size_virtual": 48, "cache_size_num": 6144, "cache_size_unit": "KB", - "flag_list": [ + "flags": [ "fpu", "vme", "de", @@ -178,7 +174,7 @@ Examples: "flush_l1d", "arch_capabilities" ], - "bug_list": [ + "bugs": [ "cpu_meltdown", "spectre_v1", "spectre_v2", diff --git a/man/jc.1 b/man/jc.1 index 0cbd86e9..3a70c413 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-09-07 1.21.2 "JSON Convert" +.TH jc 1 2022-09-08 1.21.2 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS From 8a239b8f9c4e8931983e5f75aed3925677633283 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 9 Sep 2022 14:33:12 -0700 Subject: [PATCH 25/71] add proc-diskstats parser --- docs/parsers/proc_diskstats.md | 202 +++++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_diskstats.py | 269 +++++++++++++++++++++++++++++++++ man/jc.1 | 7 +- 4 files changed, 478 insertions(+), 1 deletion(-) create mode 100644 docs/parsers/proc_diskstats.md create mode 100644 jc/parsers/proc_diskstats.py diff --git a/docs/parsers/proc_diskstats.md b/docs/parsers/proc_diskstats.md new file mode 100644 index 00000000..5273814a --- /dev/null +++ b/docs/parsers/proc_diskstats.md @@ -0,0 +1,202 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_diskstats + +jc - JSON Convert `/proc/diskstats` file parser + +Usage (cli): + + $ cat /proc/diskstats | jc --proc + +or + + $ jc /proc/diskstats + +or + + $ cat /proc/diskstats | jc --proc-diskstats + +Usage (module): + + import jc + result = jc.parse('proc', proc_diskstats_file) + +or + + import jc + result = jc.parse('proc_diskstats', proc_diskstats_file) + +Schema: + + [ + { + "maj": integer, + "min": integer, + "device": string, + "reads_completed": integer, + "reads_merged": integer, + "sectors_read": integer, + "read_time_ms": integer, + "writes_completed": integer, + "writes_merged": integer, + "sectors_written": integer, + "write_time_ms": integer, + "io_in_progress": integer, + "io_time_ms": integer, + "weighted_io_time_ms": integer, + "discards_completed_successfully": integer, + "discards_merged": integer, + "sectors_discarded": integer, + "discarding_time_ms": integer, + "flush_requests_completed_successfully": integer, + "flushing_time_ms": integer + } + ] + +Examples: + + $ cat /proc/diskstats | jc --proc -p + [ + { + "maj": 7, + "min": 0, + "device": "loop0", + "reads_completed": 48, + "reads_merged": 0, + "sectors_read": 718, + "read_time_ms": 19, + "writes_completed": 0, + "writes_merged": 0, + "sectors_written": 0, + "write_time_ms": 0, + "io_in_progress": 0, + "io_time_ms": 36, + "weighted_io_time_ms": 19, + "discards_completed_successfully": 0, + "discards_merged": 0, + "sectors_discarded": 0, + "discarding_time_ms": 0, + "flush_requests_completed_successfully": 0, + "flushing_time_ms": 0 + }, + { + "maj": 7, + "min": 1, + "device": "loop1", + "reads_completed": 41, + "reads_merged": 0, + "sectors_read": 688, + "read_time_ms": 17, + "writes_completed": 0, + "writes_merged": 0, + "sectors_written": 0, + "write_time_ms": 0, + "io_in_progress": 0, + "io_time_ms": 28, + "weighted_io_time_ms": 17, + "discards_completed_successfully": 0, + "discards_merged": 0, + "sectors_discarded": 0, + "discarding_time_ms": 0, + "flush_requests_completed_successfully": 0, + "flushing_time_ms": 0 + }, + ... + ] + + $ proc_diskstats | jc --proc_diskstats -p -r + [ + { + "maj": "7", + "min": "0", + "device": "loop0", + "reads_completed": "48", + "reads_merged": "0", + "sectors_read": "718", + "read_time_ms": "19", + "writes_completed": "0", + "writes_merged": "0", + "sectors_written": "0", + "write_time_ms": "0", + "io_in_progress": "0", + "io_time_ms": "36", + "weighted_io_time_ms": "19", + "discards_completed_successfully": "0", + "discards_merged": "0", + "sectors_discarded": "0", + "discarding_time_ms": "0", + "flush_requests_completed_successfully": "0", + "flushing_time_ms": "0" + }, + { + "maj": "7", + "min": "1", + "device": "loop1", + "reads_completed": "41", + "reads_merged": "0", + "sectors_read": "688", + "read_time_ms": "17", + "writes_completed": "0", + "writes_merged": "0", + "sectors_written": "0", + "write_time_ms": "0", + "io_in_progress": "0", + "io_time_ms": "28", + "weighted_io_time_ms": "17", + "discards_completed_successfully": "0", + "discards_merged": "0", + "sectors_discarded": "0", + "discarding_time_ms": "0", + "flush_requests_completed_successfully": "0", + "flushing_time_ms": "0" + }, + { + "maj": "7", + "min": "2", + "device": "loop2", + "reads_completed": "119", + "reads_merged": "0", + "sectors_read": "2956", + "read_time_ms": "18", + "writes_completed": "0", + "writes_merged": "0", + "sectors_written": "0", + "write_time_ms": "0", + "io_in_progress": "0", + "io_time_ms": "56", + "weighted_io_time_ms": "18", + "discards_completed_successfully": "0", + "discards_merged": "0", + "sectors_discarded": "0", + "discarding_time_ms": "0", + "flush_requests_completed_successfully": "0", + "flushing_time_ms": "0" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 625bdcea..d7eda470 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -90,6 +90,7 @@ parsers = [ 'proc-cpuinfo', 'proc-crypto', 'proc-devices', + 'proc-diskstats', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_diskstats.py b/jc/parsers/proc_diskstats.py new file mode 100644 index 00000000..630c8def --- /dev/null +++ b/jc/parsers/proc_diskstats.py @@ -0,0 +1,269 @@ +"""jc - JSON Convert `/proc/diskstats` file parser + +Usage (cli): + + $ cat /proc/diskstats | jc --proc + +or + + $ jc /proc/diskstats + +or + + $ cat /proc/diskstats | jc --proc-diskstats + +Usage (module): + + import jc + result = jc.parse('proc', proc_diskstats_file) + +or + + import jc + result = jc.parse('proc_diskstats', proc_diskstats_file) + +Schema: + + [ + { + "maj": integer, + "min": integer, + "device": string, + "reads_completed": integer, + "reads_merged": integer, + "sectors_read": integer, + "read_time_ms": integer, + "writes_completed": integer, + "writes_merged": integer, + "sectors_written": integer, + "write_time_ms": integer, + "io_in_progress": integer, + "io_time_ms": integer, + "weighted_io_time_ms": integer, + "discards_completed_successfully": integer, + "discards_merged": integer, + "sectors_discarded": integer, + "discarding_time_ms": integer, + "flush_requests_completed_successfully": integer, + "flushing_time_ms": integer + } + ] + +Examples: + + $ cat /proc/diskstats | jc --proc -p + [ + { + "maj": 7, + "min": 0, + "device": "loop0", + "reads_completed": 48, + "reads_merged": 0, + "sectors_read": 718, + "read_time_ms": 19, + "writes_completed": 0, + "writes_merged": 0, + "sectors_written": 0, + "write_time_ms": 0, + "io_in_progress": 0, + "io_time_ms": 36, + "weighted_io_time_ms": 19, + "discards_completed_successfully": 0, + "discards_merged": 0, + "sectors_discarded": 0, + "discarding_time_ms": 0, + "flush_requests_completed_successfully": 0, + "flushing_time_ms": 0 + }, + { + "maj": 7, + "min": 1, + "device": "loop1", + "reads_completed": 41, + "reads_merged": 0, + "sectors_read": 688, + "read_time_ms": 17, + "writes_completed": 0, + "writes_merged": 0, + "sectors_written": 0, + "write_time_ms": 0, + "io_in_progress": 0, + "io_time_ms": 28, + "weighted_io_time_ms": 17, + "discards_completed_successfully": 0, + "discards_merged": 0, + "sectors_discarded": 0, + "discarding_time_ms": 0, + "flush_requests_completed_successfully": 0, + "flushing_time_ms": 0 + }, + ... + ] + + $ proc_diskstats | jc --proc_diskstats -p -r + [ + { + "maj": "7", + "min": "0", + "device": "loop0", + "reads_completed": "48", + "reads_merged": "0", + "sectors_read": "718", + "read_time_ms": "19", + "writes_completed": "0", + "writes_merged": "0", + "sectors_written": "0", + "write_time_ms": "0", + "io_in_progress": "0", + "io_time_ms": "36", + "weighted_io_time_ms": "19", + "discards_completed_successfully": "0", + "discards_merged": "0", + "sectors_discarded": "0", + "discarding_time_ms": "0", + "flush_requests_completed_successfully": "0", + "flushing_time_ms": "0" + }, + { + "maj": "7", + "min": "1", + "device": "loop1", + "reads_completed": "41", + "reads_merged": "0", + "sectors_read": "688", + "read_time_ms": "17", + "writes_completed": "0", + "writes_merged": "0", + "sectors_written": "0", + "write_time_ms": "0", + "io_in_progress": "0", + "io_time_ms": "28", + "weighted_io_time_ms": "17", + "discards_completed_successfully": "0", + "discards_merged": "0", + "sectors_discarded": "0", + "discarding_time_ms": "0", + "flush_requests_completed_successfully": "0", + "flushing_time_ms": "0" + }, + { + "maj": "7", + "min": "2", + "device": "loop2", + "reads_completed": "119", + "reads_merged": "0", + "sectors_read": "2956", + "read_time_ms": "18", + "writes_completed": "0", + "writes_merged": "0", + "sectors_written": "0", + "write_time_ms": "0", + "io_in_progress": "0", + "io_time_ms": "56", + "weighted_io_time_ms": "18", + "discards_completed_successfully": "0", + "discards_merged": "0", + "sectors_discarded": "0", + "discarding_time_ms": "0", + "flush_requests_completed_successfully": "0", + "flushing_time_ms": "0" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/diskstats` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + for entry in proc_data: + for key in entry: + if key != 'device': + entry[key] = int(entry[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + split_line = line.split() + + output_line = { + 'maj': split_line[0], + 'min': split_line[1], + 'device': split_line[2], + 'reads_completed': split_line[3], + 'reads_merged': split_line[4], + 'sectors_read': split_line[5], + 'read_time_ms': split_line[6], + 'writes_completed': split_line[7], + 'writes_merged': split_line[8], + 'sectors_written': split_line[9], + 'write_time_ms': split_line[10], + 'io_in_progress': split_line[11], + 'io_time_ms': split_line[12], + 'weighted_io_time_ms': split_line[13] + } + + if len(split_line) > 14: + output_line['discards_completed_successfully'] = split_line[14] + output_line['discards_merged'] = split_line[15] + output_line['sectors_discarded'] = split_line[16] + output_line['discarding_time_ms'] = split_line[17] + + if len(split_line) > 18: + output_line['flush_requests_completed_successfully'] = split_line[18] + output_line['flushing_time_ms'] = split_line[19] + + raw_output.append(output_line) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 3a70c413..4adf2055 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-09-08 1.21.2 "JSON Convert" +.TH jc 1 2022-09-09 1.21.2 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS @@ -435,6 +435,11 @@ PLIST file parser \fB--proc-devices\fP `/proc/devices` file parser +.TP +.B +\fB--proc-diskstats\fP +`/proc/diskstats` file parser + .TP .B \fB--proc-meminfo\fP From 23d5204634104cff857a5b04077a58102c1ca041 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 9 Sep 2022 15:07:44 -0700 Subject: [PATCH 26/71] simplify parser by using simple_table_parse --- jc/parsers/proc_diskstats.py | 43 +++++++++--------------------------- 1 file changed, 11 insertions(+), 32 deletions(-) diff --git a/jc/parsers/proc_diskstats.py b/jc/parsers/proc_diskstats.py index 630c8def..309bb609 100644 --- a/jc/parsers/proc_diskstats.py +++ b/jc/parsers/proc_diskstats.py @@ -173,6 +173,7 @@ Examples: """ from typing import List, Dict import jc.utils +from jc.parsers.universal import simple_table_parse class info(): @@ -233,37 +234,15 @@ def parse( if jc.utils.has_data(data): - for line in filter(None, data.splitlines()): - - split_line = line.split() - - output_line = { - 'maj': split_line[0], - 'min': split_line[1], - 'device': split_line[2], - 'reads_completed': split_line[3], - 'reads_merged': split_line[4], - 'sectors_read': split_line[5], - 'read_time_ms': split_line[6], - 'writes_completed': split_line[7], - 'writes_merged': split_line[8], - 'sectors_written': split_line[9], - 'write_time_ms': split_line[10], - 'io_in_progress': split_line[11], - 'io_time_ms': split_line[12], - 'weighted_io_time_ms': split_line[13] - } - - if len(split_line) > 14: - output_line['discards_completed_successfully'] = split_line[14] - output_line['discards_merged'] = split_line[15] - output_line['sectors_discarded'] = split_line[16] - output_line['discarding_time_ms'] = split_line[17] - - if len(split_line) > 18: - output_line['flush_requests_completed_successfully'] = split_line[18] - output_line['flushing_time_ms'] = split_line[19] - - raw_output.append(output_line) + header = ( + 'maj min device reads_completed reads_merged sectors_read read_time_ms ' + 'writes_completed writes_merged sectors_written write_time_ms io_in_progress ' + 'io_time_ms weighted_io_time_ms discards_completed_successfully discards_merged ' + 'sectors_discarded discarding_time_ms flush_requests_completed_successfully ' + 'flushing_time_ms\n' + ) + data = header + data + cleandata = filter(None, data.splitlines()) + raw_output = simple_table_parse(cleandata) return raw_output if raw else _process(raw_output) From 65d647bc0a32ce09a145521c412517bffbc24b84 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 9 Sep 2022 15:20:38 -0700 Subject: [PATCH 27/71] tighten up diskstats signature --- jc/parsers/proc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/proc.py b/jc/parsers/proc.py index a4309329..69c5d550 100644 --- a/jc/parsers/proc.py +++ b/jc/parsers/proc.py @@ -157,7 +157,7 @@ def parse( cpuinfo_p = re.compile(r'^processor\t+: \d+.*bogomips\t+: \d+.\d\d\n', re.DOTALL) crypto_p = re.compile(r'^name\s+:.*\ndriver\s+:.*\nmodule\s+:.*\n') devices_p = re.compile(r'^Character devices:\n\s+\d+ .*\n') - diskstats_p = re.compile(r'^\s*\d+\s+\d\s\w+\s(?:\d+\s){16}\d\n') + diskstats_p = re.compile(r'^\s*\d+\s+\d\s\w+\s(?:\d+\s){10,16}\d+\n') filesystems_p = re.compile(r'^(?:(?:nodev\t|\t)\w+\n){3}') interrupts_p = re.compile(r'^\s+(?:CPU\d+ +)+\n\s*\d+:\s+\d+') iomem_p = re.compile(r'^00000000-[0-9a-f]{8} : .*\n[0-9a-f]{8}-[0-9a-f]{8} : ') From 993fcd989b226b2e27e32bb207c2b2387859ce33 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 9 Sep 2022 16:28:34 -0700 Subject: [PATCH 28/71] add driver_rtc parser --- docs/parsers/proc_driver_rtc.md | 126 +++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_driver_rtc.py | 173 ++++++++++++++++++++++++++++++++ man/jc.1 | 5 + 4 files changed, 305 insertions(+) create mode 100644 docs/parsers/proc_driver_rtc.md create mode 100644 jc/parsers/proc_driver_rtc.py diff --git a/docs/parsers/proc_driver_rtc.md b/docs/parsers/proc_driver_rtc.md new file mode 100644 index 00000000..199f2396 --- /dev/null +++ b/docs/parsers/proc_driver_rtc.md @@ -0,0 +1,126 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_driver\_rtc + +jc - JSON Convert `/proc/driver-rtc` file parser + +Usage (cli): + + $ cat /proc/driver_rtc | jc --proc + +or + + $ jc /proc/driver_rtc + +or + + $ cat /proc/driver_rtc | jc --proc-driver-rtc + +Usage (module): + + import jc + result = jc.parse('proc', proc_driver_rtc_file) + +or + + import jc + result = jc.parse('proc_driver_rtc', proc_driver_rtc_file) + +Schema: + +"yes" and "no" values are converted to true/false. Integer conversions are +attempted. If you do not want this behavior, then use `--raw` (CLI) or +`raw=True` (module). + + { + "rtc_time": string, + "rtc_date": string, + "alrm_time": string, + "alrm_date": string, + "alarm_IRQ": boolean, + "alrm_pending": boolean, + "update IRQ enabled": boolean, + "periodic IRQ enabled": boolean, + "periodic IRQ frequency": integer, + "max user IRQ frequency": integer, + "24hr": boolean, + "periodic_IRQ": boolean, + "update_IRQ": boolean, + "HPET_emulated": boolean, + "BCD": boolean, + "DST_enable": boolean, + "periodic_freq": integer, + "batt_status": string + } + +Examples: + + $ cat /proc/driver_rtc | jc --proc -p + { + "rtc_time": "16:09:21", + "rtc_date": "2022-09-03", + "alrm_time": "00:00:00", + "alrm_date": "2022-09-03", + "alarm_IRQ": false, + "alrm_pending": false, + "update IRQ enabled": false, + "periodic IRQ enabled": false, + "periodic IRQ frequency": 1024, + "max user IRQ frequency": 64, + "24hr": true, + "periodic_IRQ": false, + "update_IRQ": false, + "HPET_emulated": true, + "BCD": true, + "DST_enable": false, + "periodic_freq": 1024, + "batt_status": "okay" + } + + $ cat /proc/driver_rtc | jc --proc -p -r + { + "rtc_time": "16:09:21", + "rtc_date": "2022-09-03", + "alrm_time": "00:00:00", + "alrm_date": "2022-09-03", + "alarm_IRQ": "no", + "alrm_pending": "no", + "update IRQ enabled": "no", + "periodic IRQ enabled": "no", + "periodic IRQ frequency": "1024", + "max user IRQ frequency": "64", + "24hr": "yes", + "periodic_IRQ": "no", + "update_IRQ": "no", + "HPET_emulated": "yes", + "BCD": "yes", + "DST_enable": "no", + "periodic_freq": "1024", + "batt_status": "okay" + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index d7eda470..fe5d7d2c 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -91,6 +91,7 @@ parsers = [ 'proc-crypto', 'proc-devices', 'proc-diskstats', + 'proc-driver-rtc', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_driver_rtc.py b/jc/parsers/proc_driver_rtc.py new file mode 100644 index 00000000..54c1066f --- /dev/null +++ b/jc/parsers/proc_driver_rtc.py @@ -0,0 +1,173 @@ +"""jc - JSON Convert `/proc/driver-rtc` file parser + +Usage (cli): + + $ cat /proc/driver_rtc | jc --proc + +or + + $ jc /proc/driver_rtc + +or + + $ cat /proc/driver_rtc | jc --proc-driver-rtc + +Usage (module): + + import jc + result = jc.parse('proc', proc_driver_rtc_file) + +or + + import jc + result = jc.parse('proc_driver_rtc', proc_driver_rtc_file) + +Schema: + +"yes" and "no" values are converted to true/false. Integer conversions are +attempted. If you do not want this behavior, then use `--raw` (CLI) or +`raw=True` (module). + + { + "rtc_time": string, + "rtc_date": string, + "alrm_time": string, + "alrm_date": string, + "alarm_IRQ": boolean, + "alrm_pending": boolean, + "update IRQ enabled": boolean, + "periodic IRQ enabled": boolean, + "periodic IRQ frequency": integer, + "max user IRQ frequency": integer, + "24hr": boolean, + "periodic_IRQ": boolean, + "update_IRQ": boolean, + "HPET_emulated": boolean, + "BCD": boolean, + "DST_enable": boolean, + "periodic_freq": integer, + "batt_status": string + } + +Examples: + + $ cat /proc/driver_rtc | jc --proc -p + { + "rtc_time": "16:09:21", + "rtc_date": "2022-09-03", + "alrm_time": "00:00:00", + "alrm_date": "2022-09-03", + "alarm_IRQ": false, + "alrm_pending": false, + "update IRQ enabled": false, + "periodic IRQ enabled": false, + "periodic IRQ frequency": 1024, + "max user IRQ frequency": 64, + "24hr": true, + "periodic_IRQ": false, + "update_IRQ": false, + "HPET_emulated": true, + "BCD": true, + "DST_enable": false, + "periodic_freq": 1024, + "batt_status": "okay" + } + + $ cat /proc/driver_rtc | jc --proc -p -r + { + "rtc_time": "16:09:21", + "rtc_date": "2022-09-03", + "alrm_time": "00:00:00", + "alrm_date": "2022-09-03", + "alarm_IRQ": "no", + "alrm_pending": "no", + "update IRQ enabled": "no", + "periodic IRQ enabled": "no", + "periodic IRQ frequency": "1024", + "max user IRQ frequency": "64", + "24hr": "yes", + "periodic_IRQ": "no", + "update_IRQ": "no", + "HPET_emulated": "yes", + "BCD": "yes", + "DST_enable": "no", + "periodic_freq": "1024", + "batt_status": "okay" + } +""" +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/driver_rtc` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + for key, val in proc_data.items(): + try: + proc_data[key] = int(val) + except: + pass + + if val == 'yes': + proc_data[key] = True + + if val == 'no': + proc_data[key] = False + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + split_line = line.split(':', maxsplit=1) + key = split_line[0].strip() + val = split_line[1].rsplit(maxsplit=1)[0] + raw_output[key] = val + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 4adf2055..bb13036b 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -440,6 +440,11 @@ PLIST file parser \fB--proc-diskstats\fP `/proc/diskstats` file parser +.TP +.B +\fB--proc-driver-rtc\fP +`/proc/driver_rtc` file parser + .TP .B \fB--proc-meminfo\fP From 0508256d280cb882de0d535cc6efae14a0aa8439 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 9 Sep 2022 16:32:33 -0700 Subject: [PATCH 29/71] formatting --- docs/parsers/proc_driver_rtc.md | 2 +- jc/parsers/proc_driver_rtc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parsers/proc_driver_rtc.md b/docs/parsers/proc_driver_rtc.md index 199f2396..26c7193f 100644 --- a/docs/parsers/proc_driver_rtc.md +++ b/docs/parsers/proc_driver_rtc.md @@ -3,7 +3,7 @@ # jc.parsers.proc\_driver\_rtc -jc - JSON Convert `/proc/driver-rtc` file parser +jc - JSON Convert `/proc/driver_rtc` file parser Usage (cli): diff --git a/jc/parsers/proc_driver_rtc.py b/jc/parsers/proc_driver_rtc.py index 54c1066f..6b384b42 100644 --- a/jc/parsers/proc_driver_rtc.py +++ b/jc/parsers/proc_driver_rtc.py @@ -1,4 +1,4 @@ -"""jc - JSON Convert `/proc/driver-rtc` file parser +"""jc - JSON Convert `/proc/driver_rtc` file parser Usage (cli): From e1f6007dea6647115e58c6f7965b5d632611182b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 9 Sep 2022 17:03:51 -0700 Subject: [PATCH 30/71] add proc-filesystems parser --- docs/parsers/proc_filesystems.md | 81 ++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_filesystems.py | 122 +++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 4 files changed, 209 insertions(+) create mode 100644 docs/parsers/proc_filesystems.md create mode 100644 jc/parsers/proc_filesystems.py diff --git a/docs/parsers/proc_filesystems.md b/docs/parsers/proc_filesystems.md new file mode 100644 index 00000000..b4b8b443 --- /dev/null +++ b/docs/parsers/proc_filesystems.md @@ -0,0 +1,81 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_filesystems + +jc - JSON Convert `/proc/filesystems` file parser + +Usage (cli): + + $ cat /proc/filesystems | jc --proc + +or + + $ jc /proc/filesystems + +or + + $ cat /proc/filesystems | jc --proc-filesystems + +Usage (module): + + import jc + result = jc.parse('proc', proc_filesystems_file) + +or + + import jc + result = jc.parse('proc_filesystems', proc_filesystems_file) + +Schema: + + [ + { + "filesystem": string, + "nodev": boolean + } + ] + +Examples: + + $ cat /proc/filesystems | jc --proc -p + [ + { + "filesystem": "sysfs", + "nodev": true + }, + { + "filesystem": "tmpfs", + "nodev": true + }, + { + "filesystem": "bdev", + "nodev": true + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index fe5d7d2c..f6bbc733 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -92,6 +92,7 @@ parsers = [ 'proc-devices', 'proc-diskstats', 'proc-driver-rtc', + 'proc-filesystems', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_filesystems.py b/jc/parsers/proc_filesystems.py new file mode 100644 index 00000000..91170de4 --- /dev/null +++ b/jc/parsers/proc_filesystems.py @@ -0,0 +1,122 @@ +"""jc - JSON Convert `/proc/filesystems` file parser + +Usage (cli): + + $ cat /proc/filesystems | jc --proc + +or + + $ jc /proc/filesystems + +or + + $ cat /proc/filesystems | jc --proc-filesystems + +Usage (module): + + import jc + result = jc.parse('proc', proc_filesystems_file) + +or + + import jc + result = jc.parse('proc_filesystems', proc_filesystems_file) + +Schema: + + [ + { + "filesystem": string, + "nodev": boolean + } + ] + +Examples: + + $ cat /proc/filesystems | jc --proc -p + [ + { + "filesystem": "sysfs", + "nodev": true + }, + { + "filesystem": "tmpfs", + "nodev": true + }, + { + "filesystem": "bdev", + "nodev": true + }, + ... + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/filesystems` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + split_line = line.split() + output_line = {'filesystem': split_line[-1]} + + if len(split_line) == 2: + output_line['nodev'] = True # type: ignore + else: + output_line['nodev'] = False # type: ignore + + raw_output.append(output_line) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index bb13036b..23069a54 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -445,6 +445,11 @@ PLIST file parser \fB--proc-driver-rtc\fP `/proc/driver_rtc` file parser +.TP +.B +\fB--proc-filesystems\fP +`/proc/filesystems` file parser + .TP .B \fB--proc-meminfo\fP From 03f0984e1d48a2eb3d302a72b6fd482d16af1202 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 9 Sep 2022 17:13:43 -0700 Subject: [PATCH 31/71] doc update --- README.md | 9 +++++---- docs/parsers/proc_driver_rtc.md | 4 ++-- jc/cli.py | 2 +- jc/parsers/proc_driver_rtc.py | 4 ++-- man/jc.1 | 2 +- templates/manpage_template | 2 +- templates/readme_template | 9 +++++---- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index cfb329f6..24bd24c0 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ values are converted, and, in some cases, additional semantic context fields are added. To access the raw, pre-processed JSON, use the `-r` cli option or the `raw=True` -function parameter in `parse()`. +function parameter in `parse()` when using `jc` as a python library. Schemas for each parser can be found at the documentation link beside each [**Parser**](#parsers) below. @@ -139,11 +139,12 @@ echo STRING | jc [OPTIONS] PARSER ``` Alternatively, the "magic" syntax can be used by prepending `jc` to the command -to be converted. Options can be passed to `jc` immediately before the command is -given. (Note: command aliases and shell builtins are not supported) +to be converted or in front of the absolute path for Proc files. Options can be +passed to `jc` immediately before the command or Proc file path is given. +(Note: command aliases and shell builtins are not supported) ```bash jc [OPTIONS] COMMAND -jc [OPTIONS] /proc/ +jc [OPTIONS] /proc/ ``` The JSON output can be compact (default) or pretty formatted with the `-p` diff --git a/docs/parsers/proc_driver_rtc.md b/docs/parsers/proc_driver_rtc.md index 26c7193f..05ea5972 100644 --- a/docs/parsers/proc_driver_rtc.md +++ b/docs/parsers/proc_driver_rtc.md @@ -29,8 +29,8 @@ or Schema: -"yes" and "no" values are converted to true/false. Integer conversions are -attempted. If you do not want this behavior, then use `--raw` (CLI) or +"yes" and "no" values are converted to `true`/`false`. Integer conversions +are attempted. If you do not want this behavior, then use `--raw` (cli) or `raw=True` (module). { diff --git a/jc/cli.py b/jc/cli.py index 6696651e..9c35ee95 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -195,7 +195,7 @@ Usage: jc [OPTIONS] COMMAND - jc [OPTIONS] /proc/ + jc [OPTIONS] /proc/ Parsers: {parsers_string} diff --git a/jc/parsers/proc_driver_rtc.py b/jc/parsers/proc_driver_rtc.py index 6b384b42..2bc40f33 100644 --- a/jc/parsers/proc_driver_rtc.py +++ b/jc/parsers/proc_driver_rtc.py @@ -24,8 +24,8 @@ or Schema: -"yes" and "no" values are converted to true/false. Integer conversions are -attempted. If you do not want this behavior, then use `--raw` (CLI) or +"yes" and "no" values are converted to `true`/`false`. Integer conversions +are attempted. If you do not want this behavior, then use `--raw` (cli) or `raw=True` (module). { diff --git a/man/jc.1 b/man/jc.1 index 23069a54..aa896893 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -18,7 +18,7 @@ Magic syntax: .RS \fBjc\fP [OPTIONS] COMMAND -\fBjc\fP [OPTIONS] /proc/ +\fBjc\fP [OPTIONS] /proc/ .RE .SH DESCRIPTION diff --git a/templates/manpage_template b/templates/manpage_template index d6a8cc40..8854f421 100644 --- a/templates/manpage_template +++ b/templates/manpage_template @@ -18,7 +18,7 @@ Magic syntax: .RS \fBjc\fP [OPTIONS] COMMAND -\fBjc\fP [OPTIONS] /proc/ +\fBjc\fP [OPTIONS] /proc/ .RE .SH DESCRIPTION diff --git a/templates/readme_template b/templates/readme_template index 9a5c2c1a..5f25970a 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -70,7 +70,7 @@ values are converted, and, in some cases, additional semantic context fields are added. To access the raw, pre-processed JSON, use the `-r` cli option or the `raw=True` -function parameter in `parse()`. +function parameter in `parse()` when using `jc` as a python library. Schemas for each parser can be found at the documentation link beside each [**Parser**](#parsers) below. @@ -139,11 +139,12 @@ echo STRING | jc [OPTIONS] PARSER ``` Alternatively, the "magic" syntax can be used by prepending `jc` to the command -to be converted. Options can be passed to `jc` immediately before the command is -given. (Note: command aliases and shell builtins are not supported) +to be converted or in front of the absolute path for Proc files. Options can be +passed to `jc` immediately before the command or Proc file path is given. +(Note: command aliases and shell builtins are not supported) ```bash jc [OPTIONS] COMMAND -jc [OPTIONS] /proc/ +jc [OPTIONS] /proc/ ``` The JSON output can be compact (default) or pretty formatted with the `-p` From da51b2b5a07716c2d4909a1ca58fe21f9e778bc5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 12 Sep 2022 22:43:37 -0700 Subject: [PATCH 32/71] initial proc_interrupts parser --- jc/lib.py | 1 + jc/parsers/proc_interrupts.py | 213 ++++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 jc/parsers/proc_interrupts.py diff --git a/jc/lib.py b/jc/lib.py index f6bbc733..106052cd 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -93,6 +93,7 @@ parsers = [ 'proc-diskstats', 'proc-driver-rtc', 'proc-filesystems', + 'proc-interrupts', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_interrupts.py b/jc/parsers/proc_interrupts.py new file mode 100644 index 00000000..11057953 --- /dev/null +++ b/jc/parsers/proc_interrupts.py @@ -0,0 +1,213 @@ +"""jc - JSON Convert `/proc/interrupts` file parser + +Usage (cli): + + $ cat /proc/interrupts | jc --proc + +or + + $ jc /proc/interrupts + +or + + $ cat /proc/interrupts | jc --proc-interrupts + +Usage (module): + + import jc + result = jc.parse('proc', proc_interrupts_file) + +or + + import jc + result = jc.parse('proc_interrupts', proc_interrupts_file) + +Schema: + + [ + { + "module": string, + "size": integer, + "used": integer, + "used_by": [ + string + ], + "status": string, + "location": string + } + ] + +Examples: + + $ cat /proc/interrupts | jc --proc -p + [ + { + "module": "binfmt_misc", + "size": 24576, + "used": 1, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": 16384, + "used": 0, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": 36864, + "used": 1, + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] + + $ proc_interrupts | jc --proc_interrupts -p -r + [ + { + "module": "binfmt_misc", + "size": "24576", + "used": "1", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": "16384", + "used": "0", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": "36864", + "used": "1", + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/interrupts` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'size', 'used'} + + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + data_lines = data.splitlines() + + # get the number of cpus + cpu_num = len(data_lines[0].split()) + + for line in filter(None, data_lines): + + # skip non-data lines + if not ':' in line: + continue + + # process data lines + split_line = line.split() + irq = split_line.pop(0)[:-1] + + interrupts = [] + if irq == 'ERR' or irq == 'MIS': + interrupts.extend(split_line) + + elif irq.isdigit(): + for _ in range(cpu_num): + interrupts.append(split_line.pop(0)) + + interrupt_type = split_line.pop(0) + interrupt_info = split_line + + else: + for _ in range(cpu_num): + interrupts.append(split_line.pop(0)) + + interrupt_type = ' '.join(split_line) + interrupt_info = [] + + + print(f'{cpu_num=}, {irq=}, {interrupts=}, {interrupt_type=}, {interrupt_info=}') + + raw_output.append( + { + 'irq': irq, + 'cpu_num': cpu_num, + 'interrupts': interrupts, + 'interrupt_type': interrupt_type, + 'interrupt_info': interrupt_info or None + } + ) + + return raw_output if raw else _process(raw_output) From de6307dc382406d2294e9f2920b8231bd79d7345 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 12 Sep 2022 22:59:14 -0700 Subject: [PATCH 33/71] remove debug print --- jc/parsers/proc_interrupts.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/jc/parsers/proc_interrupts.py b/jc/parsers/proc_interrupts.py index 11057953..0ba288c2 100644 --- a/jc/parsers/proc_interrupts.py +++ b/jc/parsers/proc_interrupts.py @@ -197,9 +197,6 @@ def parse( interrupt_type = ' '.join(split_line) interrupt_info = [] - - print(f'{cpu_num=}, {irq=}, {interrupts=}, {interrupt_type=}, {interrupt_info=}') - raw_output.append( { 'irq': irq, From cb684fa6de33caa3cb45a56ddcf43a37865b3962 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 13 Sep 2022 13:42:06 -0700 Subject: [PATCH 34/71] update schema and docs --- docs/parsers/proc_interrupts.md | 133 ++++++++++++++++++++++++++++++++ jc/parsers/proc_interrupts.py | 121 ++++++++++++++--------------- man/jc.1 | 7 +- 3 files changed, 198 insertions(+), 63 deletions(-) create mode 100644 docs/parsers/proc_interrupts.md diff --git a/docs/parsers/proc_interrupts.md b/docs/parsers/proc_interrupts.md new file mode 100644 index 00000000..412449eb --- /dev/null +++ b/docs/parsers/proc_interrupts.md @@ -0,0 +1,133 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_interrupts + +jc - JSON Convert `/proc/interrupts` file parser + +Usage (cli): + + $ cat /proc/interrupts | jc --proc + +or + + $ jc /proc/interrupts + +or + + $ cat /proc/interrupts | jc --proc-interrupts + +Usage (module): + + import jc + result = jc.parse('proc', proc_interrupts_file) + +or + + import jc + result = jc.parse('proc_interrupts', proc_interrupts_file) + +Schema: + + [ + { + "irq": string, + "cpu_num": integer, + "interrupts": [ + integer + ], + "type": string, + "device": [ + string + ] + } + ] + +Examples: + + $ cat /proc/interrupts | jc --proc -p + [ + { + "irq": "0", + "cpu_num": 2, + "interrupts": [ + 18, + 0 + ], + "type": "IO-APIC", + "device": [ + "2-edge", + "timer" + ] + }, + { + "irq": "1", + "cpu_num": 2, + "interrupts": [ + 0, + 73 + ], + "type": "IO-APIC", + "device": [ + "1-edge", + "i8042" + ] + }, + ... + ] + + $ proc_interrupts | jc --proc_interrupts -p -r + [ + { + "irq": "0", + "cpu_num": 2, + "interrupts": [ + "18", + "0" + ], + "type": "IO-APIC", + "device": [ + "2-edge", + "timer" + ] + }, + { + "irq": "1", + "cpu_num": 2, + "interrupts": [ + "0", + "73" + ], + "type": "IO-APIC", + "device": [ + "1-edge", + "i8042" + ] + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/proc_interrupts.py b/jc/parsers/proc_interrupts.py index 0ba288c2..76b76fe3 100644 --- a/jc/parsers/proc_interrupts.py +++ b/jc/parsers/proc_interrupts.py @@ -26,14 +26,15 @@ Schema: [ { - "module": string, - "size": integer, - "used": integer, - "used_by": [ - string + "irq": string, + "cpu_num": integer, + "interrupts": [ + integer ], - "status": string, - "location": string + "type": string, + "device": [ + string + ] } ] @@ -42,30 +43,30 @@ Examples: $ cat /proc/interrupts | jc --proc -p [ { - "module": "binfmt_misc", - "size": 24576, - "used": 1, - "used_by": [], - "status": "Live", - "location": "0xffffffffc0ab4000" - }, - { - "module": "vsock_loopback", - "size": 16384, - "used": 0, - "used_by": [], - "status": "Live", - "location": "0xffffffffc0a14000" - }, - { - "module": "vmw_vsock_virtio_transport_common", - "size": 36864, - "used": 1, - "used_by": [ - "vsock_loopback" + "irq": "0", + "cpu_num": 2, + "interrupts": [ + 18, + 0 ], - "status": "Live", - "location": "0xffffffffc0a03000" + "type": "IO-APIC", + "device": [ + "2-edge", + "timer" + ] + }, + { + "irq": "1", + "cpu_num": 2, + "interrupts": [ + 0, + 73 + ], + "type": "IO-APIC", + "device": [ + "1-edge", + "i8042" + ] }, ... ] @@ -73,30 +74,30 @@ Examples: $ proc_interrupts | jc --proc_interrupts -p -r [ { - "module": "binfmt_misc", - "size": "24576", - "used": "1", - "used_by": [], - "status": "Live", - "location": "0xffffffffc0ab4000" - }, - { - "module": "vsock_loopback", - "size": "16384", - "used": "0", - "used_by": [], - "status": "Live", - "location": "0xffffffffc0a14000" - }, - { - "module": "vmw_vsock_virtio_transport_common", - "size": "36864", - "used": "1", - "used_by": [ - "vsock_loopback" + "irq": "0", + "cpu_num": 2, + "interrupts": [ + "18", + "0" ], - "status": "Live", - "location": "0xffffffffc0a03000" + "type": "IO-APIC", + "device": [ + "2-edge", + "timer" + ] + }, + { + "irq": "1", + "cpu_num": 2, + "interrupts": [ + "0", + "73" + ], + "type": "IO-APIC", + "device": [ + "1-edge", + "i8042" + ] }, ... ] @@ -130,12 +131,8 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = {'size', 'used'} - for entry in proc_data: - for key in entry: - if key in int_list: - entry[key] = jc.utils.convert_to_int(entry[key]) + entry['interrupts'] = [int(x) for x in entry['interrupts']] return proc_data @@ -188,22 +185,22 @@ def parse( interrupts.append(split_line.pop(0)) interrupt_type = split_line.pop(0) - interrupt_info = split_line + device = split_line else: for _ in range(cpu_num): interrupts.append(split_line.pop(0)) interrupt_type = ' '.join(split_line) - interrupt_info = [] + device = [] raw_output.append( { 'irq': irq, 'cpu_num': cpu_num, 'interrupts': interrupts, - 'interrupt_type': interrupt_type, - 'interrupt_info': interrupt_info or None + 'type': interrupt_type, + 'device': device or None } ) diff --git a/man/jc.1 b/man/jc.1 index aa896893..c9df998f 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-09-09 1.21.2 "JSON Convert" +.TH jc 1 2022-09-13 1.21.2 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS @@ -450,6 +450,11 @@ PLIST file parser \fB--proc-filesystems\fP `/proc/filesystems` file parser +.TP +.B +\fB--proc-interrupts\fP +`/proc/interrupts` file parser + .TP .B \fB--proc-meminfo\fP From ab33836637634269d25f08b12b2c39caceaab48d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 13 Sep 2022 14:04:15 -0700 Subject: [PATCH 35/71] add proc-iomem parser --- docs/parsers/proc_iomem.md | 85 ++++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_iomem.py | 130 +++++++++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 4 files changed, 221 insertions(+) create mode 100644 docs/parsers/proc_iomem.md create mode 100644 jc/parsers/proc_iomem.py diff --git a/docs/parsers/proc_iomem.md b/docs/parsers/proc_iomem.md new file mode 100644 index 00000000..4bd0ce2f --- /dev/null +++ b/docs/parsers/proc_iomem.md @@ -0,0 +1,85 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_iomem + +jc - JSON Convert `/proc/iomem` file parser + +Usage (cli): + + $ cat /proc/iomem | jc --proc + +or + + $ jc /proc/iomem + +or + + $ cat /proc/iomem | jc --proc-iomem + +Usage (module): + + import jc + result = jc.parse('proc', proc_iomem_file) + +or + + import jc + result = jc.parse('proc_iomem', proc_iomem_file) + +Schema: + + [ + { + "start": string, + "end": string, + "device": string + } + ] + +Examples: + + $ cat /proc/iomem | jc --proc -p + [ + { + "start": "00000000", + "end": "00000fff", + "device": "Reserved" + }, + { + "start": "00001000", + "end": "0009e7ff", + "device": "System RAM" + }, + { + "start": "0009e800", + "end": "0009ffff", + "device": "Reserved" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 106052cd..a693fb7c 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -94,6 +94,7 @@ parsers = [ 'proc-driver-rtc', 'proc-filesystems', 'proc-interrupts', + 'proc-iomem', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_iomem.py b/jc/parsers/proc_iomem.py new file mode 100644 index 00000000..4b526d7a --- /dev/null +++ b/jc/parsers/proc_iomem.py @@ -0,0 +1,130 @@ +"""jc - JSON Convert `/proc/iomem` file parser + +Usage (cli): + + $ cat /proc/iomem | jc --proc + +or + + $ jc /proc/iomem + +or + + $ cat /proc/iomem | jc --proc-iomem + +Usage (module): + + import jc + result = jc.parse('proc', proc_iomem_file) + +or + + import jc + result = jc.parse('proc_iomem', proc_iomem_file) + +Schema: + + [ + { + "start": string, + "end": string, + "device": string + } + ] + +Examples: + + $ cat /proc/iomem | jc --proc -p + [ + { + "start": "00000000", + "end": "00000fff", + "device": "Reserved" + }, + { + "start": "00001000", + "end": "0009e7ff", + "device": "System RAM" + }, + { + "start": "0009e800", + "end": "0009ffff", + "device": "Reserved" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/iomem` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + colon_split = line.split(':', maxsplit=1) + device = colon_split[1].strip() + mem_split = colon_split[0].split('-', maxsplit=1) + start = mem_split[0].strip() + end = mem_split[1].strip() + + raw_output.append( + { + 'start': start, + 'end': end, + 'device': device + } + ) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index c9df998f..db93f147 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -455,6 +455,11 @@ PLIST file parser \fB--proc-interrupts\fP `/proc/interrupts` file parser +.TP +.B +\fB--proc-iomem\fP +`/proc/iomem` file parser + .TP .B \fB--proc-meminfo\fP From 9e5c3ae6fb5cd0e59e1cec4cf1d813104536bb7b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 13 Sep 2022 14:12:24 -0700 Subject: [PATCH 36/71] add proc-ioports parser --- docs/parsers/proc_ioports.md | 85 ++++++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_ioports.py | 113 +++++++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 4 files changed, 204 insertions(+) create mode 100644 docs/parsers/proc_ioports.md create mode 100644 jc/parsers/proc_ioports.py diff --git a/docs/parsers/proc_ioports.md b/docs/parsers/proc_ioports.md new file mode 100644 index 00000000..9606509c --- /dev/null +++ b/docs/parsers/proc_ioports.md @@ -0,0 +1,85 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_ioports + +jc - JSON Convert `/proc/ioports` file parser + +Usage (cli): + + $ cat /proc/ioports | jc --proc + +or + + $ jc /proc/ioports + +or + + $ cat /proc/ioports | jc --proc-ioports + +Usage (module): + + import jc + result = jc.parse('proc', proc_ioports_file) + +or + + import jc + result = jc.parse('proc_ioports', proc_ioports_file) + +Schema: + + [ + { + "start": string, + "end": string, + "device": string + } + ] + +Examples: + + $ cat /proc/ioports | jc --proc -p + [ + { + "start": "0000", + "end": "0cf7", + "device": "PCI Bus 0000:00" + }, + { + "start": "0000", + "end": "001f", + "device": "dma1" + }, + { + "start": "0020", + "end": "0021", + "device": "PNP0001:00" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index a693fb7c..dac00e19 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -95,6 +95,7 @@ parsers = [ 'proc-filesystems', 'proc-interrupts', 'proc-iomem', + 'proc-ioports', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_ioports.py b/jc/parsers/proc_ioports.py new file mode 100644 index 00000000..0c861b0d --- /dev/null +++ b/jc/parsers/proc_ioports.py @@ -0,0 +1,113 @@ +"""jc - JSON Convert `/proc/ioports` file parser + +Usage (cli): + + $ cat /proc/ioports | jc --proc + +or + + $ jc /proc/ioports + +or + + $ cat /proc/ioports | jc --proc-ioports + +Usage (module): + + import jc + result = jc.parse('proc', proc_ioports_file) + +or + + import jc + result = jc.parse('proc_ioports', proc_ioports_file) + +Schema: + + [ + { + "start": string, + "end": string, + "device": string + } + ] + +Examples: + + $ cat /proc/ioports | jc --proc -p + [ + { + "start": "0000", + "end": "0cf7", + "device": "PCI Bus 0000:00" + }, + { + "start": "0000", + "end": "001f", + "device": "dma1" + }, + { + "start": "0020", + "end": "0021", + "device": "PNP0001:00" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils +import jc.parsers.proc_iomem as proc_iomem + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/ioports` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = proc_iomem.parse(data) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index db93f147..78bc4903 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -460,6 +460,11 @@ PLIST file parser \fB--proc-iomem\fP `/proc/iomem` file parser +.TP +.B +\fB--proc-ioports\fP +`/proc/ioports` file parser + .TP .B \fB--proc-meminfo\fP From 2cad23a7f3ecd71c89bd72181387518bbf98194a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 13 Sep 2022 16:27:11 -0700 Subject: [PATCH 37/71] add proc-loadavg parser --- docs/parsers/proc_loadavg.md | 88 ++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_loadavg.py | 139 +++++++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 4 files changed, 233 insertions(+) create mode 100644 docs/parsers/proc_loadavg.md create mode 100644 jc/parsers/proc_loadavg.py diff --git a/docs/parsers/proc_loadavg.md b/docs/parsers/proc_loadavg.md new file mode 100644 index 00000000..05168398 --- /dev/null +++ b/docs/parsers/proc_loadavg.md @@ -0,0 +1,88 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_loadavg + +jc - JSON Convert `/proc/loadavg` file parser + +Usage (cli): + + $ cat /proc/loadavg | jc --proc + +or + + $ jc /proc/loadavg + +or + + $ cat /proc/loadavg | jc --proc-loadavg + +Usage (module): + + import jc + result = jc.parse('proc', proc_loadavg_file) + +or + + import jc + result = jc.parse('proc_loadavg', proc_loadavg_file) + +Schema: + +All values are integers. + + { + "load_1m": float, + "load_5m": float, + "load_15m": float, + "running": integer, + "available": integer, + "last_pid": integer + } + +Examples: + + $ cat /proc/loadavg | jc --proc -p + { + "load_1m": 0.0, + "load_5m": 0.01, + "load_15m": 0.03, + "running": 2, + "available": 111, + "last_pid": 2039 + } + + $ cat /proc/loadavg | jc --proc -p -r + { + "load_1m": "0.00", + "load_5m": "0.01", + "load_15m": "0.03", + "running": "2", + "available": "111", + "last_pid": "2039" + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index dac00e19..e9537127 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -96,6 +96,7 @@ parsers = [ 'proc-interrupts', 'proc-iomem', 'proc-ioports', + 'proc-loadavg', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_loadavg.py b/jc/parsers/proc_loadavg.py new file mode 100644 index 00000000..dc234c71 --- /dev/null +++ b/jc/parsers/proc_loadavg.py @@ -0,0 +1,139 @@ +"""jc - JSON Convert `/proc/loadavg` file parser + +Usage (cli): + + $ cat /proc/loadavg | jc --proc + +or + + $ jc /proc/loadavg + +or + + $ cat /proc/loadavg | jc --proc-loadavg + +Usage (module): + + import jc + result = jc.parse('proc', proc_loadavg_file) + +or + + import jc + result = jc.parse('proc_loadavg', proc_loadavg_file) + +Schema: + +All values are integers. + + { + "load_1m": float, + "load_5m": float, + "load_15m": float, + "running": integer, + "available": integer, + "last_pid": integer + } + +Examples: + + $ cat /proc/loadavg | jc --proc -p + { + "load_1m": 0.0, + "load_5m": 0.01, + "load_15m": 0.03, + "running": 2, + "available": 111, + "last_pid": 2039 + } + + $ cat /proc/loadavg | jc --proc -p -r + { + "load_1m": "0.00", + "load_5m": "0.01", + "load_15m": "0.03", + "running": "2", + "available": "111", + "last_pid": "2039" + } +""" +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/loadavg` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + float_list = {'load_1m', 'load_5m', 'load_15m'} + int_list = {'running', 'available', 'last_pid'} + + for key in proc_data: + if key in float_list: + proc_data[key] = float(proc_data[key]) + + if key in int_list: + proc_data[key] = int(proc_data[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + + if jc.utils.has_data(data): + + load_1m, load_5m, load_15m, runnable, last_pid = data.split() + running, available = runnable.split('/') + + raw_output = { + 'load_1m': load_1m, + 'load_5m': load_5m, + 'load_15m': load_15m, + 'running': running, + 'available': available, + 'last_pid': last_pid + } + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 78bc4903..54b939e7 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -465,6 +465,11 @@ PLIST file parser \fB--proc-ioports\fP `/proc/ioports` file parser +.TP +.B +\fB--proc-loadavg\fP +`/proc/loadavg` file parser + .TP .B \fB--proc-meminfo\fP From 140dc656a25100745d4890c1022c01c3e04b8dd6 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 13 Sep 2022 16:45:33 -0700 Subject: [PATCH 38/71] initial proc-locks parser --- jc/lib.py | 1 + jc/parsers/proc_locks.py | 188 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 jc/parsers/proc_locks.py diff --git a/jc/lib.py b/jc/lib.py index e9537127..29e28d78 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -97,6 +97,7 @@ parsers = [ 'proc-iomem', 'proc-ioports', 'proc-loadavg', + 'proc-locks', 'proc-meminfo', 'proc-modules', 'ps', diff --git a/jc/parsers/proc_locks.py b/jc/parsers/proc_locks.py new file mode 100644 index 00000000..9bf54c6e --- /dev/null +++ b/jc/parsers/proc_locks.py @@ -0,0 +1,188 @@ +"""jc - JSON Convert `/proc/locks` file parser + +Usage (cli): + + $ cat /proc/locks | jc --proc + +or + + $ jc /proc/locks + +or + + $ cat /proc/locks | jc --proc-locks + +Usage (module): + + import jc + result = jc.parse('proc', proc_locks_file) + +or + + import jc + result = jc.parse('proc_locks', proc_locks_file) + +Schema: + + [ + { + "module": string, + "size": integer, + "used": integer, + "used_by": [ + string + ], + "status": string, + "location": string + } + ] + +Examples: + + $ cat /proc/locks | jc --proc -p + [ + { + "module": "binfmt_misc", + "size": 24576, + "used": 1, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": 16384, + "used": 0, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": 36864, + "used": 1, + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] + + $ proc_locks | jc --proc_locks -p -r + [ + { + "module": "binfmt_misc", + "size": "24576", + "used": "1", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": "16384", + "used": "0", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": "36864", + "used": "1", + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/locks` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'size', 'used'} + + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + id, class_, type_, access, pid, file, start, end = line.split() + maj, min, inode = file.split(':') + + raw_output.append( + { + 'id': id[:-1], + 'class': class_, + 'type': type_, + 'access': access, + 'pid': pid, + 'maj': maj, + 'min': min, + 'inode': inode, + 'start': start, + 'end': end + } + ) + + return raw_output if raw else _process(raw_output) From c348fa89a938099662c4fe8f89bdfd9422230ab5 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Tue, 13 Sep 2022 20:40:48 -0700 Subject: [PATCH 39/71] proc-locks fixes --- docs/parsers/proc_locks.md | 130 +++++++++++++++++++++++++++++++++++++ jc/parsers/proc_locks.py | 106 +++++++++++++++--------------- man/jc.1 | 5 ++ 3 files changed, 187 insertions(+), 54 deletions(-) create mode 100644 docs/parsers/proc_locks.md diff --git a/docs/parsers/proc_locks.md b/docs/parsers/proc_locks.md new file mode 100644 index 00000000..38ebb3f9 --- /dev/null +++ b/docs/parsers/proc_locks.md @@ -0,0 +1,130 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_locks + +jc - JSON Convert `/proc/locks` file parser + +Usage (cli): + + $ cat /proc/locks | jc --proc + +or + + $ jc /proc/locks + +or + + $ cat /proc/locks | jc --proc-locks + +Usage (module): + + import jc + result = jc.parse('proc', proc_locks_file) + +or + + import jc + result = jc.parse('proc_locks', proc_locks_file) + +Schema: + + [ + { + "id": integer, + "class": string, + "type": string, + "access": string, + "pid": integer, + "maj": string, + "min": string, + "inode": integer, + "start": string, + "end": string + } + ] + +Examples: + + $ cat /proc/locks | jc --proc -p + [ + { + "id": 1, + "class": "POSIX", + "type": "ADVISORY", + "access": "WRITE", + "pid": 877, + "maj": "00", + "min": "19", + "inode": 812, + "start": "0", + "end": "EOF" + }, + { + "id": 2, + "class": "FLOCK", + "type": "ADVISORY", + "access": "WRITE", + "pid": 854, + "maj": "00", + "min": "19", + "inode": 805, + "start": "0", + "end": "EOF" + }, + ... + ] + + $ proc_locks | jc --proc_locks -p -r + [ + { + "id": "1", + "class": "POSIX", + "type": "ADVISORY", + "access": "WRITE", + "pid": "877", + "maj": "00", + "min": "19", + "inode": "812", + "start": "0", + "end": "EOF" + }, + { + "id": "2", + "class": "FLOCK", + "type": "ADVISORY", + "access": "WRITE", + "pid": "854", + "maj": "00", + "min": "19", + "inode": "805", + "start": "0", + "end": "EOF" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/parsers/proc_locks.py b/jc/parsers/proc_locks.py index 9bf54c6e..9550645a 100644 --- a/jc/parsers/proc_locks.py +++ b/jc/parsers/proc_locks.py @@ -26,14 +26,16 @@ Schema: [ { - "module": string, - "size": integer, - "used": integer, - "used_by": [ - string - ], - "status": string, - "location": string + "id": integer, + "class": string, + "type": string, + "access": string, + "pid": integer, + "maj": string, + "min": string, + "inode": integer, + "start": string, + "end": string } ] @@ -42,30 +44,28 @@ Examples: $ cat /proc/locks | jc --proc -p [ { - "module": "binfmt_misc", - "size": 24576, - "used": 1, - "used_by": [], - "status": "Live", - "location": "0xffffffffc0ab4000" + "id": 1, + "class": "POSIX", + "type": "ADVISORY", + "access": "WRITE", + "pid": 877, + "maj": "00", + "min": "19", + "inode": 812, + "start": "0", + "end": "EOF" }, { - "module": "vsock_loopback", - "size": 16384, - "used": 0, - "used_by": [], - "status": "Live", - "location": "0xffffffffc0a14000" - }, - { - "module": "vmw_vsock_virtio_transport_common", - "size": 36864, - "used": 1, - "used_by": [ - "vsock_loopback" - ], - "status": "Live", - "location": "0xffffffffc0a03000" + "id": 2, + "class": "FLOCK", + "type": "ADVISORY", + "access": "WRITE", + "pid": 854, + "maj": "00", + "min": "19", + "inode": 805, + "start": "0", + "end": "EOF" }, ... ] @@ -73,30 +73,28 @@ Examples: $ proc_locks | jc --proc_locks -p -r [ { - "module": "binfmt_misc", - "size": "24576", - "used": "1", - "used_by": [], - "status": "Live", - "location": "0xffffffffc0ab4000" + "id": "1", + "class": "POSIX", + "type": "ADVISORY", + "access": "WRITE", + "pid": "877", + "maj": "00", + "min": "19", + "inode": "812", + "start": "0", + "end": "EOF" }, { - "module": "vsock_loopback", - "size": "16384", - "used": "0", - "used_by": [], - "status": "Live", - "location": "0xffffffffc0a14000" - }, - { - "module": "vmw_vsock_virtio_transport_common", - "size": "36864", - "used": "1", - "used_by": [ - "vsock_loopback" - ], - "status": "Live", - "location": "0xffffffffc0a03000" + "id": "2", + "class": "FLOCK", + "type": "ADVISORY", + "access": "WRITE", + "pid": "854", + "maj": "00", + "min": "19", + "inode": "805", + "start": "0", + "end": "EOF" }, ... ] @@ -130,12 +128,12 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = {'size', 'used'} + int_list = {'id', 'pid', 'inode'} for entry in proc_data: for key in entry: if key in int_list: - entry[key] = jc.utils.convert_to_int(entry[key]) + entry[key] = int(entry[key]) return proc_data diff --git a/man/jc.1 b/man/jc.1 index 54b939e7..78afe6e0 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -470,6 +470,11 @@ PLIST file parser \fB--proc-loadavg\fP `/proc/loadavg` file parser +.TP +.B +\fB--proc-locks\fP +`/proc/locks` file parser + .TP .B \fB--proc-meminfo\fP From ab5e9a46b4fff0e9fdb6be7da2cad6f3f0b40b8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20B=C5=99ezina?= Date: Wed, 14 Sep 2022 13:36:53 +0200 Subject: [PATCH 40/71] id: support space in names id parser did not work correctly if space is present in user or group name. --- jc/parsers/id.py | 32 ++++++++++++++++---------------- tests/test_id.py | 24 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/jc/parsers/id.py b/jc/parsers/id.py index 897ba494..6f52f9c2 100644 --- a/jc/parsers/id.py +++ b/jc/parsers/id.py @@ -100,6 +100,7 @@ Examples: } } """ +import re import jc.utils @@ -170,28 +171,28 @@ def parse(data, raw=False, quiet=False): raw_output = {} - # Clear any blank lines - cleandata = list(filter(None, data.split())) + # re.split produces first element empty + cleandata = re.split(r' ?(uid|gid|groups|context)=', data.strip())[1:] if jc.utils.has_data(data): - for section in cleandata: - if section.startswith('uid'): - uid_parsed = section.replace('(', '=').replace(')', '=') + for key, value in zip(cleandata[0::2], cleandata[1::2]): + if key == 'uid': + uid_parsed = value.replace('(', '=').replace(')', '=') uid_parsed = uid_parsed.split('=') raw_output['uid'] = {} - raw_output['uid']['id'] = uid_parsed[1] - raw_output['uid']['name'] = _get_item(uid_parsed, 2) + raw_output['uid']['id'] = uid_parsed[0] + raw_output['uid']['name'] = _get_item(uid_parsed, 1) - if section.startswith('gid'): - gid_parsed = section.replace('(', '=').replace(')', '=') + if key == 'gid': + gid_parsed = value.replace('(', '=').replace(')', '=') gid_parsed = gid_parsed.split('=') raw_output['gid'] = {} - raw_output['gid']['id'] = gid_parsed[1] - raw_output['gid']['name'] = _get_item(gid_parsed, 2) + raw_output['gid']['id'] = gid_parsed[0] + raw_output['gid']['name'] = _get_item(gid_parsed, 1) - if section.startswith('groups'): - groups_parsed = section.replace('(', '=').replace(')', '=') + if key == 'groups': + groups_parsed = value.replace('(', '=').replace(')', '=') groups_parsed = groups_parsed.replace('groups=', '') groups_parsed = groups_parsed.split(',') raw_output['groups'] = [] @@ -203,9 +204,8 @@ def parse(data, raw=False, quiet=False): group_dict['name'] = _get_item(grp_parsed, 1) raw_output['groups'].append(group_dict) - if section.startswith('context'): - context_parsed = section.replace('context=', '') - context_parsed = context_parsed.split(':', maxsplit=3) + if key == 'context': + context_parsed = value.split(':', maxsplit=3) raw_output['context'] = {} raw_output['context']['user'] = context_parsed[0] raw_output['context']['role'] = context_parsed[1] diff --git a/tests/test_id.py b/tests/test_id.py index b3752dfe..bcd6a9eb 100644 --- a/tests/test_id.py +++ b/tests/test_id.py @@ -43,6 +43,30 @@ class MyTests(unittest.TestCase): {'uid': {'id': 1000, 'name': 'user'}, 'gid': {'id': 1000, 'name': None}, 'groups': [{'id': 1000, 'name': None}, {'id': 10, 'name': 'wheel'}]} ) + def test_id_space(self): + """ + Test 'id' with with space in name + """ + self.assertEqual( + jc.parsers.id.parse('uid=1000(user 1) gid=1000 groups=1000,10(wheel)', quiet=True), + {'uid': {'id': 1000, 'name': 'user 1'}, 'gid': {'id': 1000, 'name': None}, 'groups': [{'id': 1000, 'name': None}, {'id': 10, 'name': 'wheel'}]} + ) + + self.assertEqual( + jc.parsers.id.parse('uid=1000(user) gid=1000(group 1) groups=1000,10(wheel)', quiet=True), + {'uid': {'id': 1000, 'name': 'user'}, 'gid': {'id': 1000, 'name': 'group 1'}, 'groups': [{'id': 1000, 'name': None}, {'id': 10, 'name': 'wheel'}]} + ) + + self.assertEqual( + jc.parsers.id.parse('uid=1000(user) gid=1000 groups=1000,10(wheel 1)', quiet=True), + {'uid': {'id': 1000, 'name': 'user'}, 'gid': {'id': 1000, 'name': None}, 'groups': [{'id': 1000, 'name': None}, {'id': 10, 'name': 'wheel 1'}]} + ) + + self.assertEqual( + jc.parsers.id.parse('uid=1000(user 1) gid=1000(group 1) groups=1000,10(wheel 1)', quiet=True), + {'uid': {'id': 1000, 'name': 'user 1'}, 'gid': {'id': 1000, 'name': 'group 1'}, 'groups': [{'id': 1000, 'name': None}, {'id': 10, 'name': 'wheel 1'}]} + ) + def test_id_centos_7_7(self): """ Test 'id' on Centos 7.7 From 4ebde3af5f8d6019db51c7c3e9d53dcecebb667a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 15 Sep 2022 08:40:58 -0700 Subject: [PATCH 41/71] parser version bump --- jc/parsers/id.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/id.py b/jc/parsers/id.py index 6f52f9c2..d3f310a2 100644 --- a/jc/parsers/id.py +++ b/jc/parsers/id.py @@ -106,7 +106,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`id` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' From d3c1a73ced56c13a5e3079e49cde03ffa9ab81bc Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 15 Sep 2022 08:49:12 -0700 Subject: [PATCH 42/71] doc update --- CHANGELOG | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index e42e2302..c8491521 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,9 @@ jc changelog +20220915 v1.22.0 +- Add /proc file parsers +- Fix `id` command parser to allow usernames and groupnames with spaces + 20220829 v1.21.2 - Fix IP Address string parser for older python versions that don't cleanly accept decimal input format - IPv6 fix (e.g. python 3.6) From e4e07b76ec724f96b2532b659a54089379313186 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 15 Sep 2022 16:57:45 -0700 Subject: [PATCH 43/71] doc update --- docs/parsers/id.md | 2 +- docs/parsers/proc_cpuinfo.md | 2 +- docs/parsers/proc_crypto.md | 2 +- docs/parsers/proc_diskstats.md | 2 +- docs/parsers/proc_interrupts.md | 2 +- docs/parsers/proc_locks.md | 2 +- docs/parsers/proc_modules.md | 2 +- jc/lib.py | 1 + jc/parsers/proc_cpuinfo.py | 2 +- jc/parsers/proc_crypto.py | 2 +- jc/parsers/proc_diskstats.py | 2 +- jc/parsers/proc_interrupts.py | 2 +- jc/parsers/proc_locks.py | 2 +- jc/parsers/proc_modules.py | 2 +- jc/parsers/proc_mtrr.py | 183 ++++++++++++++++++++++++++++++++ man/jc.1 | 2 +- 16 files changed, 198 insertions(+), 14 deletions(-) create mode 100644 jc/parsers/proc_mtrr.py diff --git a/docs/parsers/id.md b/docs/parsers/id.md index eaf32554..0bac2060 100644 --- a/docs/parsers/id.md +++ b/docs/parsers/id.md @@ -128,4 +128,4 @@ Returns: ### Parser Information Compatibility: linux, darwin, aix, freebsd -Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com) +Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/parsers/proc_cpuinfo.md b/docs/parsers/proc_cpuinfo.md index 0c24ddc7..574c919f 100644 --- a/docs/parsers/proc_cpuinfo.md +++ b/docs/parsers/proc_cpuinfo.md @@ -189,7 +189,7 @@ Examples: ... ] - $ proc_cpuinfo | jc --proc_cpuinfo -p -r + $ cat /proc/cpuinfo | jc --proc_cpuinfo -p -r [ { "processor": "0", diff --git a/docs/parsers/proc_crypto.md b/docs/parsers/proc_crypto.md index 835489d7..25e9adc0 100644 --- a/docs/parsers/proc_crypto.md +++ b/docs/parsers/proc_crypto.md @@ -78,7 +78,7 @@ Examples: ... ] - $ proc_crypto | jc --proc_crypto -p -r + $ cat /proc/crypto | jc --proc_crypto -p -r [ { "name": "ecdh", diff --git a/docs/parsers/proc_diskstats.md b/docs/parsers/proc_diskstats.md index 5273814a..43accf9e 100644 --- a/docs/parsers/proc_diskstats.md +++ b/docs/parsers/proc_diskstats.md @@ -105,7 +105,7 @@ Examples: ... ] - $ proc_diskstats | jc --proc_diskstats -p -r + $ cat /proc/diskstats | jc --proc_diskstats -p -r [ { "maj": "7", diff --git a/docs/parsers/proc_interrupts.md b/docs/parsers/proc_interrupts.md index 412449eb..f0f8c123 100644 --- a/docs/parsers/proc_interrupts.md +++ b/docs/parsers/proc_interrupts.md @@ -76,7 +76,7 @@ Examples: ... ] - $ proc_interrupts | jc --proc_interrupts -p -r + $ cat /proc/interrupts | jc --proc_interrupts -p -r [ { "irq": "0", diff --git a/docs/parsers/proc_locks.md b/docs/parsers/proc_locks.md index 38ebb3f9..a611bb62 100644 --- a/docs/parsers/proc_locks.md +++ b/docs/parsers/proc_locks.md @@ -75,7 +75,7 @@ Examples: ... ] - $ proc_locks | jc --proc_locks -p -r + $ cat /proc/locks | jc --proc_locks -p -r [ { "id": "1", diff --git a/docs/parsers/proc_modules.md b/docs/parsers/proc_modules.md index 82833a42..f300ae06 100644 --- a/docs/parsers/proc_modules.md +++ b/docs/parsers/proc_modules.md @@ -75,7 +75,7 @@ Examples: ... ] - $ proc_modules | jc --proc_modules -p -r + $ cat /proc/modules | jc --proc_modules -p -r [ { "module": "binfmt_misc", diff --git a/jc/lib.py b/jc/lib.py index 29e28d78..69065411 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -100,6 +100,7 @@ parsers = [ 'proc-locks', 'proc-meminfo', 'proc-modules', + 'proc-mtrr', 'ps', 'route', 'rpm-qi', diff --git a/jc/parsers/proc_cpuinfo.py b/jc/parsers/proc_cpuinfo.py index 5e826e99..5de13991 100644 --- a/jc/parsers/proc_cpuinfo.py +++ b/jc/parsers/proc_cpuinfo.py @@ -184,7 +184,7 @@ Examples: ... ] - $ proc_cpuinfo | jc --proc_cpuinfo -p -r + $ cat /proc/cpuinfo | jc --proc_cpuinfo -p -r [ { "processor": "0", diff --git a/jc/parsers/proc_crypto.py b/jc/parsers/proc_crypto.py index f2b4fa4a..93cab164 100644 --- a/jc/parsers/proc_crypto.py +++ b/jc/parsers/proc_crypto.py @@ -73,7 +73,7 @@ Examples: ... ] - $ proc_crypto | jc --proc_crypto -p -r + $ cat /proc/crypto | jc --proc_crypto -p -r [ { "name": "ecdh", diff --git a/jc/parsers/proc_diskstats.py b/jc/parsers/proc_diskstats.py index 309bb609..c230b2f8 100644 --- a/jc/parsers/proc_diskstats.py +++ b/jc/parsers/proc_diskstats.py @@ -100,7 +100,7 @@ Examples: ... ] - $ proc_diskstats | jc --proc_diskstats -p -r + $ cat /proc/diskstats | jc --proc_diskstats -p -r [ { "maj": "7", diff --git a/jc/parsers/proc_interrupts.py b/jc/parsers/proc_interrupts.py index 76b76fe3..606cadc1 100644 --- a/jc/parsers/proc_interrupts.py +++ b/jc/parsers/proc_interrupts.py @@ -71,7 +71,7 @@ Examples: ... ] - $ proc_interrupts | jc --proc_interrupts -p -r + $ cat /proc/interrupts | jc --proc_interrupts -p -r [ { "irq": "0", diff --git a/jc/parsers/proc_locks.py b/jc/parsers/proc_locks.py index 9550645a..24d3cd58 100644 --- a/jc/parsers/proc_locks.py +++ b/jc/parsers/proc_locks.py @@ -70,7 +70,7 @@ Examples: ... ] - $ proc_locks | jc --proc_locks -p -r + $ cat /proc/locks | jc --proc_locks -p -r [ { "id": "1", diff --git a/jc/parsers/proc_modules.py b/jc/parsers/proc_modules.py index f74e9539..18e35b15 100644 --- a/jc/parsers/proc_modules.py +++ b/jc/parsers/proc_modules.py @@ -70,7 +70,7 @@ Examples: ... ] - $ proc_modules | jc --proc_modules -p -r + $ cat /proc/modules | jc --proc_modules -p -r [ { "module": "binfmt_misc", diff --git a/jc/parsers/proc_mtrr.py b/jc/parsers/proc_mtrr.py new file mode 100644 index 00000000..bc325f26 --- /dev/null +++ b/jc/parsers/proc_mtrr.py @@ -0,0 +1,183 @@ +"""jc - JSON Convert `/proc/mtrr` file parser + +Usage (cli): + + $ cat /proc/mtrr | jc --proc + +or + + $ jc /proc/mtrr + +or + + $ cat /proc/mtrr | jc --proc-mtrr + +Usage (module): + + import jc + result = jc.parse('proc', proc_mtrr_file) + +or + + import jc + result = jc.parse('proc_mtrr', proc_mtrr_file) + +Schema: + + [ + { + "register": string, + "type": string, + "base": string, + "base_mb": integer, + "size": integer, + "count": integer, + "": string # additional key/values are strings + } + ] + +Examples: + + $ cat /proc/mtrr | jc --proc -p + [ + { + "register": "reg00", + "type": "write-back", + "base": "0x000000000", + "base_mb": 0, + "size": 2048, + "count": 1 + }, + { + "register": "reg01", + "type": "write-back", + "base": "0x080000000", + "base_mb": 2048, + "size": 1024, + "count": 1 + }, + ... + ] + + $ cat /proc/mtrr | jc --proc_mtrr -p -r + [ + { + "register": "reg00", + "type": "write-back", + "base": "0x000000000", + "base_mb": "0", + "size": "2048MB", + "count": "1" + }, + { + "register": "reg01", + "type": "write-back", + "base": "0x080000000", + "base_mb": "2048", + "size": "1024MB", + "count": "1" + }, + ... + ] +""" +import re +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/mtrr` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'size', 'count', 'base_mb'} + + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + split_line = re.split(r',|:', line) + register = split_line.pop(0) + type_ = None + key_vals: list = [] + + base, base_mb = split_line.pop(0).split(maxsplit=1) + key_vals.append(base) + + base_mb = base_mb.replace('(', '').replace(')', '').replace('MB', '').strip() + key_vals.append(f'base_mb={base_mb}') + + for item in split_line: + if '=' in item: + key_vals.append(item.strip()) + + else: + type_ = item.strip() + + output_line = { + 'register': register, + 'type': type_ + } + + kv_dict = {} + + for item in key_vals: + key, val = item.split('=') + kv_dict[key.strip()] = val.strip() + + output_line.update(kv_dict) + raw_output.append(output_line) + + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 78afe6e0..0a41ffe5 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-09-13 1.21.2 "JSON Convert" +.TH jc 1 2022-09-15 1.21.2 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS From bdb218cb0f65834bf4e682f0dca731abae5e6325 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Thu, 15 Sep 2022 16:59:15 -0700 Subject: [PATCH 44/71] doc update --- docs/parsers/proc_mtrr.md | 111 ++++++++++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 2 files changed, 116 insertions(+) create mode 100644 docs/parsers/proc_mtrr.md diff --git a/docs/parsers/proc_mtrr.md b/docs/parsers/proc_mtrr.md new file mode 100644 index 00000000..d6e905ab --- /dev/null +++ b/docs/parsers/proc_mtrr.md @@ -0,0 +1,111 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_mtrr + +jc - JSON Convert `/proc/mtrr` file parser + +Usage (cli): + + $ cat /proc/mtrr | jc --proc + +or + + $ jc /proc/mtrr + +or + + $ cat /proc/mtrr | jc --proc-mtrr + +Usage (module): + + import jc + result = jc.parse('proc', proc_mtrr_file) + +or + + import jc + result = jc.parse('proc_mtrr', proc_mtrr_file) + +Schema: + + [ + { + "register": string, + "type": string, + "base": string, + "base_mb": integer, + "size": integer, + "count": integer, + "": string # additional key/values are strings + } + ] + +Examples: + + $ cat /proc/mtrr | jc --proc -p + [ + { + "register": "reg00", + "type": "write-back", + "base": "0x000000000", + "base_mb": 0, + "size": 2048, + "count": 1 + }, + { + "register": "reg01", + "type": "write-back", + "base": "0x080000000", + "base_mb": 2048, + "size": 1024, + "count": 1 + }, + ... + ] + + $ cat /proc/mtrr | jc --proc_mtrr -p -r + [ + { + "register": "reg00", + "type": "write-back", + "base": "0x000000000", + "base_mb": "0", + "size": "2048MB", + "count": "1" + }, + { + "register": "reg01", + "type": "write-back", + "base": "0x080000000", + "base_mb": "2048", + "size": "1024MB", + "count": "1" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/man/jc.1 b/man/jc.1 index 0a41ffe5..251f6bcf 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -485,6 +485,11 @@ PLIST file parser \fB--proc-modules\fP `/proc/modules` file parser +.TP +.B +\fB--proc-mtrr\fP +`/proc/mtrr` file parser + .TP .B \fB--ps\fP From d115d435590f7e21b3bd02254c52ce83f5dbc915 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 16 Sep 2022 10:02:44 -0700 Subject: [PATCH 45/71] add proc-pid-numa-maps parser --- docs/parsers/proc_pid_numa_maps.md | 126 ++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_mtrr.py | 1 - jc/parsers/proc_pid_numa_maps.py | 185 +++++++++++++++++++++++++++++ man/jc.1 | 7 +- 5 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 docs/parsers/proc_pid_numa_maps.md create mode 100644 jc/parsers/proc_pid_numa_maps.py diff --git a/docs/parsers/proc_pid_numa_maps.md b/docs/parsers/proc_pid_numa_maps.md new file mode 100644 index 00000000..94e444a6 --- /dev/null +++ b/docs/parsers/proc_pid_numa_maps.md @@ -0,0 +1,126 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_pid\_numa\_maps + +jc - JSON Convert `/proc//numa_maps` file parser + +This parser will attempt to convert number values to integers. If that is +not desired, please use the `--raw` option (cli) or `raw=True` argument +(module). + +Usage (cli): + + $ cat /proc/1/numa_maps | jc --proc + +or + + $ jc /proc/1/numa_maps + +or + + $ cat /proc/1/numa_maps | jc --proc-numa-maps + +Usage (module): + + import jc + result = jc.parse('proc', proc_numa_maps_file) + +or + + import jc + result = jc.parse('proc_numa_maps', proc_numa_maps_file) + +Schema: + +Integer conversion for Key/value pairs will be attempted. + + [ + { + "address": string, + "policy": string, + "": string/integer, + "options": [ + string # [0] + ] + } + ] + + [0] remaining individual words that are not part of a key/value pair + +Examples: + + $ cat /proc/1/numa_maps | jc --proc -p + [ + { + "address": "7f53b5083000", + "policy": "default", + "file": "/usr/lib/x86_64-linux-gnu/ld-2.32.so", + "anon": 2, + "dirty": 2, + "N0": 2, + "kernelpagesize_kB": 4 + }, + { + "address": "7ffd1b23e000", + "policy": "default", + "anon": 258, + "dirty": 258, + "N0": 258, + "kernelpagesize_kB": 4, + "options": [ + "stack" + ] + }, + ... + ] + + $ cat /proc/1/numa_maps | jc --proc_numa_maps -p -r + [ + { + "address": "7f53b5083000", + "policy": "default", + "file": "/usr/lib/x86_64-linux-gnu/ld-2.32.so", + "anon": "2", + "dirty": "2", + "N0": "2", + "kernelpagesize_kB": "4" + }, + { + "address": "7ffd1b23e000", + "policy": "default", + "anon": "258", + "dirty": "258", + "N0": "258", + "kernelpagesize_kB": "4", + "options": [ + "stack" + ] + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 69065411..d8dfcb5c 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -101,6 +101,7 @@ parsers = [ 'proc-meminfo', 'proc-modules', 'proc-mtrr', + 'proc-pid-numa-maps', 'ps', 'route', 'rpm-qi', diff --git a/jc/parsers/proc_mtrr.py b/jc/parsers/proc_mtrr.py index bc325f26..ea6e89e8 100644 --- a/jc/parsers/proc_mtrr.py +++ b/jc/parsers/proc_mtrr.py @@ -179,5 +179,4 @@ def parse( output_line.update(kv_dict) raw_output.append(output_line) - return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/proc_pid_numa_maps.py b/jc/parsers/proc_pid_numa_maps.py new file mode 100644 index 00000000..f05714bc --- /dev/null +++ b/jc/parsers/proc_pid_numa_maps.py @@ -0,0 +1,185 @@ +"""jc - JSON Convert `/proc//numa_maps` file parser + +This parser will attempt to convert number values to integers. If that is +not desired, please use the `--raw` option (cli) or `raw=True` argument +(module). + +Usage (cli): + + $ cat /proc/1/numa_maps | jc --proc + +or + + $ jc /proc/1/numa_maps + +or + + $ cat /proc/1/numa_maps | jc --proc-numa-maps + +Usage (module): + + import jc + result = jc.parse('proc', proc_numa_maps_file) + +or + + import jc + result = jc.parse('proc_numa_maps', proc_numa_maps_file) + +Schema: + +Integer conversion for Key/value pairs will be attempted. + + [ + { + "address": string, + "policy": string, + "": string/integer, + "options": [ + string # [0] + ] + } + ] + + [0] remaining individual words that are not part of a key/value pair + +Examples: + + $ cat /proc/1/numa_maps | jc --proc -p + [ + { + "address": "7f53b5083000", + "policy": "default", + "file": "/usr/lib/x86_64-linux-gnu/ld-2.32.so", + "anon": 2, + "dirty": 2, + "N0": 2, + "kernelpagesize_kB": 4 + }, + { + "address": "7ffd1b23e000", + "policy": "default", + "anon": 258, + "dirty": 258, + "N0": 258, + "kernelpagesize_kB": 4, + "options": [ + "stack" + ] + }, + ... + ] + + $ cat /proc/1/numa_maps | jc --proc_numa_maps -p -r + [ + { + "address": "7f53b5083000", + "policy": "default", + "file": "/usr/lib/x86_64-linux-gnu/ld-2.32.so", + "anon": "2", + "dirty": "2", + "N0": "2", + "kernelpagesize_kB": "4" + }, + { + "address": "7ffd1b23e000", + "policy": "default", + "anon": "258", + "dirty": "258", + "N0": "258", + "kernelpagesize_kB": "4", + "options": [ + "stack" + ] + }, + ... + ] +""" +from typing import List, Dict +import jc.utils +from jc.parsers.universal import simple_table_parse + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc//numa_maps` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + for entry in proc_data: + for key, val in entry.items(): + try: + entry[key] = int(val) + except Exception: + pass + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + header = 'address policy details\n' + data = header + data + + raw_output = simple_table_parse(data.splitlines()) + + for row in raw_output: + if 'details' in row: + detail_split = row['details'].split() + + options = [] + for item in detail_split: + if '=' in item: + key, val = item.split('=') + row.update({key: val}) + else: + options.append(item) + + if options: + row['options'] = options + + del row['details'] + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 251f6bcf..0448d21e 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-09-15 1.21.2 "JSON Convert" +.TH jc 1 2022-09-16 1.21.2 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS @@ -490,6 +490,11 @@ PLIST file parser \fB--proc-mtrr\fP `/proc/mtrr` file parser +.TP +.B +\fB--proc-pid-numa-maps\fP +`/proc//numa_maps` file parser + .TP .B \fB--ps\fP From edbae09a178f167b32a02263c18b5f439bd2074c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 16 Sep 2022 11:06:33 -0700 Subject: [PATCH 46/71] add proc-pagetypeinfo parser --- docs/parsers/proc_pagetypeinfo.md | 139 +++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_pagetypeinfo.py | 222 ++++++++++++++++++++++++++++++ man/jc.1 | 5 + 4 files changed, 367 insertions(+) create mode 100644 docs/parsers/proc_pagetypeinfo.md create mode 100644 jc/parsers/proc_pagetypeinfo.py diff --git a/docs/parsers/proc_pagetypeinfo.md b/docs/parsers/proc_pagetypeinfo.md new file mode 100644 index 00000000..3fdebf03 --- /dev/null +++ b/docs/parsers/proc_pagetypeinfo.md @@ -0,0 +1,139 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_pagetypeinfo + +jc - JSON Convert `/proc/pagetypeinfo` file parser + +Usage (cli): + + $ cat /proc/pagetypeinfo | jc --proc + +or + + $ jc /proc/pagetypeinfo + +or + + $ cat /proc/pagetypeinfo | jc --proc-pagetypeinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_pagetypeinfo_file) + +or + + import jc + result = jc.parse('proc_pagetypeinfo', proc_pagetypeinfo_file) + +Schema: + + { + "page_block_order": integer, + "pages_per_block": integer, + "free_pages": [ + { + "node": integer, + "zone": string, + "type": string, + "free": [ + integer + ] + ], + "num_blocks_type": [ + { + "node": integer, + "zone": string, + "unmovable": integer, + "movable": integer, + "reclaimable": integer, + "high_atomic": integer, + "isolate": integer + } + ] + } + + +Examples: + + $ cat /proc/pagetypeinfo | jc --proc -p + { + "page_block_order": 9, + "pages_per_block": 512, + "free_pages": [ + { + "node": 0, + "zone": "DMA", + "type": "Unmovable", + "free": [ + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0 + ] + }, + ... + ], + "num_blocks_type": [ + { + "node": 0, + "zone": "DMA", + "unmovable": 1, + "movable": 7, + "reclaimable": 0, + "high_atomic": 0, + "isolate": 0 + }, + { + "node": 0, + "zone": "DMA32", + "unmovable": 8, + "movable": 1472, + "reclaimable": 48, + "high_atomic": 0, + "isolate": 0 + }, + { + "node": 0, + "zone": "Normal", + "unmovable": 120, + "movable": 345, + "reclaimable": 47, + "high_atomic": 0, + "isolate": 0 + } + ] + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index d8dfcb5c..d4683bd7 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -101,6 +101,7 @@ parsers = [ 'proc-meminfo', 'proc-modules', 'proc-mtrr', + 'proc-pagetypeinfo', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_pagetypeinfo.py b/jc/parsers/proc_pagetypeinfo.py new file mode 100644 index 00000000..9eaeadb4 --- /dev/null +++ b/jc/parsers/proc_pagetypeinfo.py @@ -0,0 +1,222 @@ +"""jc - JSON Convert `/proc/pagetypeinfo` file parser + +Usage (cli): + + $ cat /proc/pagetypeinfo | jc --proc + +or + + $ jc /proc/pagetypeinfo + +or + + $ cat /proc/pagetypeinfo | jc --proc-pagetypeinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_pagetypeinfo_file) + +or + + import jc + result = jc.parse('proc_pagetypeinfo', proc_pagetypeinfo_file) + +Schema: + + { + "page_block_order": integer, + "pages_per_block": integer, + "free_pages": [ + { + "node": integer, + "zone": string, + "type": string, + "free": [ + integer + ] + ], + "num_blocks_type": [ + { + "node": integer, + "zone": string, + "unmovable": integer, + "movable": integer, + "reclaimable": integer, + "high_atomic": integer, + "isolate": integer + } + ] + } + + +Examples: + + $ cat /proc/pagetypeinfo | jc --proc -p + { + "page_block_order": 9, + "pages_per_block": 512, + "free_pages": [ + { + "node": 0, + "zone": "DMA", + "type": "Unmovable", + "free": [ + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0 + ] + }, + ... + ], + "num_blocks_type": [ + { + "node": 0, + "zone": "DMA", + "unmovable": 1, + "movable": 7, + "reclaimable": 0, + "high_atomic": 0, + "isolate": 0 + }, + { + "node": 0, + "zone": "DMA32", + "unmovable": 8, + "movable": 1472, + "reclaimable": 48, + "high_atomic": 0, + "isolate": 0 + }, + { + "node": 0, + "zone": "Normal", + "unmovable": 120, + "movable": 345, + "reclaimable": 47, + "high_atomic": 0, + "isolate": 0 + } + ] + } +""" +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/pagetypeinfo` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + section = '' + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + if line.startswith('Page block order:'): + raw_output['page_block_order'] = int(line.split(':', maxsplit=1)[1]) + continue + + if line.startswith('Pages per block:'): + raw_output['pages_per_block'] = int(line.split(':', maxsplit=1)[1]) + continue + + if line.startswith('Free pages count per migrate type at order'): + section = 'free_pages' + raw_output['free_pages'] = [] + continue + + if line.startswith('Number of blocks type'): + section = 'num_blocks_type' + raw_output['num_blocks_type'] = [] + continue + + # Free pages count per migrate type at order 0 1 2 3 4 5 6 7 8 9 10 + # Node 0, zone DMA, type Unmovable 0 0 0 1 1 1 1 1 0 0 0 + if section == 'free_pages': + split_line = line.replace(',', ' ').split() + + output_line = { + 'node': int(split_line[1]), + 'zone': split_line[3], + 'type': split_line[5], + 'free': [int(x) for x in split_line[6:]] + } + + raw_output['free_pages'].append(output_line) + continue + + # Number of blocks type Unmovable Movable Reclaimable HighAtomic Isolate + # Node 0, zone DMA 1 7 0 0 0 + if section == 'num_blocks_type': + split_line = line.replace(',', ' ').split() + + output_line = { + 'node': int(split_line[1]), + 'zone': split_line[3], + 'unmovable': int(split_line[4]), + 'movable': int(split_line[5]), + 'reclaimable': int(split_line[6]), + 'high_atomic': int(split_line[7]), + 'isolate': int(split_line[8]), + } + + raw_output['num_blocks_type'].append(output_line) + continue + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 0448d21e..5d493eab 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -490,6 +490,11 @@ PLIST file parser \fB--proc-mtrr\fP `/proc/mtrr` file parser +.TP +.B +\fB--proc-pagetypeinfo\fP +`/proc/pagetypeinfo` file parser + .TP .B \fB--proc-pid-numa-maps\fP From 8bd935791ef31467020c328c1f1b974bea82d0e7 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 16 Sep 2022 11:17:02 -0700 Subject: [PATCH 47/71] add proc-partitions parser --- docs/parsers/proc_partitions.md | 100 ++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_partitions.py | 141 ++++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 4 files changed, 247 insertions(+) create mode 100644 docs/parsers/proc_partitions.md create mode 100644 jc/parsers/proc_partitions.py diff --git a/docs/parsers/proc_partitions.md b/docs/parsers/proc_partitions.md new file mode 100644 index 00000000..78e0b3f1 --- /dev/null +++ b/docs/parsers/proc_partitions.md @@ -0,0 +1,100 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_partitions + +jc - JSON Convert `/proc/partitions` file parser + +Usage (cli): + + $ cat /proc/partitions | jc --proc + +or + + $ jc /proc/partitions + +or + + $ cat /proc/partitions | jc --proc-partitions + +Usage (module): + + import jc + result = jc.parse('proc', proc_partitions_file) + +or + + import jc + result = jc.parse('proc_partitions', proc_partitions_file) + +Schema: + + [ + { + "major": integer, + "minor": integer, + "num_blocks": integer, + "name": string + } + ] + +Examples: + + $ cat /proc/partitions | jc --proc -p + [ + { + "major": 7, + "minor": 0, + "num_blocks": 56896, + "name": "loop0" + }, + { + "major": 7, + "minor": 1, + "num_blocks": 56868, + "name": "loop1" + }, + ... + ] + + $ cat /proc/partitions | jc --proc_partitions -p -r + [ + { + "major": "7", + "minor": "0", + "num_blocks": "56896", + "name": "loop0" + }, + { + "major": "7", + "minor": "1", + "num_blocks": "56868", + "name": "loop1" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index d4683bd7..d18af816 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -102,6 +102,7 @@ parsers = [ 'proc-modules', 'proc-mtrr', 'proc-pagetypeinfo', + 'proc-partitions', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_partitions.py b/jc/parsers/proc_partitions.py new file mode 100644 index 00000000..c04ab215 --- /dev/null +++ b/jc/parsers/proc_partitions.py @@ -0,0 +1,141 @@ +"""jc - JSON Convert `/proc/partitions` file parser + +Usage (cli): + + $ cat /proc/partitions | jc --proc + +or + + $ jc /proc/partitions + +or + + $ cat /proc/partitions | jc --proc-partitions + +Usage (module): + + import jc + result = jc.parse('proc', proc_partitions_file) + +or + + import jc + result = jc.parse('proc_partitions', proc_partitions_file) + +Schema: + + [ + { + "major": integer, + "minor": integer, + "num_blocks": integer, + "name": string + } + ] + +Examples: + + $ cat /proc/partitions | jc --proc -p + [ + { + "major": 7, + "minor": 0, + "num_blocks": 56896, + "name": "loop0" + }, + { + "major": 7, + "minor": 1, + "num_blocks": 56868, + "name": "loop1" + }, + ... + ] + + $ cat /proc/partitions | jc --proc_partitions -p -r + [ + { + "major": "7", + "minor": "0", + "num_blocks": "56896", + "name": "loop0" + }, + { + "major": "7", + "minor": "1", + "num_blocks": "56868", + "name": "loop1" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils +from jc.parsers.universal import simple_table_parse + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/partitions` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + for entry in proc_data: + for key in entry: + try: + entry[key] = int(entry[key]) + except Exception: + pass + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + cleandata = list(filter(None, data.splitlines())) + cleandata[0] = cleandata[0].replace('#', 'num_') + raw_output = simple_table_parse(cleandata) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 5d493eab..e6f1380e 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -495,6 +495,11 @@ PLIST file parser \fB--proc-pagetypeinfo\fP `/proc/pagetypeinfo` file parser +.TP +.B +\fB--proc-partitions\fP +`/proc/partitions` file parser + .TP .B \fB--proc-pid-numa-maps\fP From 122a4d8f32ca76ce73bf0b8fe0d5e7d726c63a71 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 16 Sep 2022 11:40:54 -0700 Subject: [PATCH 48/71] add proc-slabinfo parser --- docs/parsers/proc_slabinfo.md | 100 ++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_slabinfo.py | 157 ++++++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 4 files changed, 263 insertions(+) create mode 100644 docs/parsers/proc_slabinfo.md create mode 100644 jc/parsers/proc_slabinfo.py diff --git a/docs/parsers/proc_slabinfo.md b/docs/parsers/proc_slabinfo.md new file mode 100644 index 00000000..736a5576 --- /dev/null +++ b/docs/parsers/proc_slabinfo.md @@ -0,0 +1,100 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_slabinfo + +jc - JSON Convert `/proc/slabinfo` file parser + +Usage (cli): + + $ cat /proc/slabinfo | jc --proc + +or + + $ jc /proc/slabinfo + +or + + $ cat /proc/slabinfo | jc --proc-slabinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_slabinfo_file) + +or + + import jc + result = jc.parse('proc_slabinfo', proc_slabinfo_file) + +Schema: + + [ + { + "name": string, + "active_objs": integer, + "num_objs": integer, + "obj_size": integer, + "obj_per_slab": integer, + "pages_per_slab": integer, + "tunables": { + "limit": integer, + "batch_count": integer, + "shared_factor": integer + }, + "slabdata": { + "active_slabs": integer, + "num_slabs": integer, + "shared_avail": integer + } + ] + +Examples: + + $ cat /proc/slabinfo | jc --proc -p + [ + { + "name": "ext4_groupinfo_4k", + "active_objs": 224, + "num_objs": 224, + "obj_size": 144, + "obj_per_slab": 56, + "pages_per_slab": 2, + "tunables": { + "limit": 0, + "batch_count": 0, + "shared_factor": 0 + }, + "slabdata": { + "active_slabs": 4, + "num_slabs": 4, + "shared_avail": 0 + } + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index d18af816..7620d6f6 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -103,6 +103,7 @@ parsers = [ 'proc-mtrr', 'proc-pagetypeinfo', 'proc-partitions', + 'proc-slabinfo', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_slabinfo.py b/jc/parsers/proc_slabinfo.py new file mode 100644 index 00000000..9ebbde1e --- /dev/null +++ b/jc/parsers/proc_slabinfo.py @@ -0,0 +1,157 @@ +"""jc - JSON Convert `/proc/slabinfo` file parser + +Usage (cli): + + $ cat /proc/slabinfo | jc --proc + +or + + $ jc /proc/slabinfo + +or + + $ cat /proc/slabinfo | jc --proc-slabinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_slabinfo_file) + +or + + import jc + result = jc.parse('proc_slabinfo', proc_slabinfo_file) + +Schema: + + [ + { + "name": string, + "active_objs": integer, + "num_objs": integer, + "obj_size": integer, + "obj_per_slab": integer, + "pages_per_slab": integer, + "tunables": { + "limit": integer, + "batch_count": integer, + "shared_factor": integer + }, + "slabdata": { + "active_slabs": integer, + "num_slabs": integer, + "shared_avail": integer + } + ] + +Examples: + + $ cat /proc/slabinfo | jc --proc -p + [ + { + "name": "ext4_groupinfo_4k", + "active_objs": 224, + "num_objs": 224, + "obj_size": 144, + "obj_per_slab": 56, + "pages_per_slab": 2, + "tunables": { + "limit": 0, + "batch_count": 0, + "shared_factor": 0 + }, + "slabdata": { + "active_slabs": 4, + "num_slabs": 4, + "shared_avail": 0 + } + }, + ... + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/slabinfo` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + cleandata = data.splitlines()[2:] + + for line in filter(None, cleandata): + + line = line.replace(':', ' ') + split_line = line.split() + + raw_output.append( + { + 'name': split_line[0], + 'active_objs': int(split_line[1]), + 'num_objs': int(split_line[2]), + 'obj_size': int(split_line[3]), + 'obj_per_slab': int(split_line[4]), + 'pages_per_slab': int(split_line[5]), + 'tunables': { + 'limit': int(split_line[7]), + 'batch_count': int(split_line[8]), + 'shared_factor': int(split_line[9]) + }, + 'slabdata': { + 'active_slabs': int(split_line[11]), + 'num_slabs': int(split_line[12]), + 'shared_avail': int(split_line[13]) + } + } + ) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index e6f1380e..ab202336 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -500,6 +500,11 @@ PLIST file parser \fB--proc-partitions\fP `/proc/partitions` file parser +.TP +.B +\fB--proc-slabinfo\fP +`/proc/slabinfo` file parser + .TP .B \fB--proc-pid-numa-maps\fP From 8f539af4ab4ad4c4fae2dbc4197ca383e6a75235 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 16 Sep 2022 14:50:14 -0700 Subject: [PATCH 49/71] add proc-softirqs parser --- docs/parsers/proc_pagetypeinfo.md | 4 +- docs/parsers/proc_softirqs.md | 85 ++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_pagetypeinfo.py | 4 +- jc/parsers/proc_softirqs.py | 129 ++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 6 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 docs/parsers/proc_softirqs.md create mode 100644 jc/parsers/proc_softirqs.py diff --git a/docs/parsers/proc_pagetypeinfo.md b/docs/parsers/proc_pagetypeinfo.md index 3fdebf03..f1cfb990 100644 --- a/docs/parsers/proc_pagetypeinfo.md +++ b/docs/parsers/proc_pagetypeinfo.md @@ -38,7 +38,7 @@ Schema: "zone": string, "type": string, "free": [ - integer + integer # [0] ] ], "num_blocks_type": [ @@ -54,6 +54,8 @@ Schema: ] } + [0] array index correlates to the Order number. + E.g. free[0] is the value for Order 0 Examples: diff --git a/docs/parsers/proc_softirqs.md b/docs/parsers/proc_softirqs.md new file mode 100644 index 00000000..150b5a73 --- /dev/null +++ b/docs/parsers/proc_softirqs.md @@ -0,0 +1,85 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_softirqs + +jc - JSON Convert `/proc/softirqs` file parser + +Usage (cli): + + $ cat /proc/softirqs | jc --proc + +or + + $ jc /proc/softirqs + +or + + $ cat /proc/softirqs | jc --proc-softirqs + +Usage (module): + + import jc + result = jc.parse('proc', proc_softirqs_file) + +or + + import jc + result = jc.parse('proc_softirqs', proc_softirqs_file) + +Schema: + + [ + { + "counter": string, + "CPU": integer, + } + ] + +Examples: + + $ cat /proc/softirqs | jc --proc -p + [ + { + "counter": "HI", + "CPU0": 1, + "CPU1": 34056, + "CPU2": 0, + "CPU3": 0, + "CPU4": 0 + }, + { + "counter": "TIMER", + "CPU0": 322970, + "CPU1": 888166, + "CPU2": 0, + "CPU3": 0, + "CPU4": 0 + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 7620d6f6..72c680bd 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -104,6 +104,7 @@ parsers = [ 'proc-pagetypeinfo', 'proc-partitions', 'proc-slabinfo', + 'proc-softirqs', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_pagetypeinfo.py b/jc/parsers/proc_pagetypeinfo.py index 9eaeadb4..08ef08ba 100644 --- a/jc/parsers/proc_pagetypeinfo.py +++ b/jc/parsers/proc_pagetypeinfo.py @@ -33,7 +33,7 @@ Schema: "zone": string, "type": string, "free": [ - integer + integer # [0] ] ], "num_blocks_type": [ @@ -49,6 +49,8 @@ Schema: ] } + [0] array index correlates to the Order number. + E.g. free[0] is the value for Order 0 Examples: diff --git a/jc/parsers/proc_softirqs.py b/jc/parsers/proc_softirqs.py new file mode 100644 index 00000000..d4068ec1 --- /dev/null +++ b/jc/parsers/proc_softirqs.py @@ -0,0 +1,129 @@ +"""jc - JSON Convert `/proc/softirqs` file parser + +Usage (cli): + + $ cat /proc/softirqs | jc --proc + +or + + $ jc /proc/softirqs + +or + + $ cat /proc/softirqs | jc --proc-softirqs + +Usage (module): + + import jc + result = jc.parse('proc', proc_softirqs_file) + +or + + import jc + result = jc.parse('proc_softirqs', proc_softirqs_file) + +Schema: + + [ + { + "counter": string, + "CPU": integer, + } + ] + +Examples: + + $ cat /proc/softirqs | jc --proc -p + [ + { + "counter": "HI", + "CPU0": 1, + "CPU1": 34056, + "CPU2": 0, + "CPU3": 0, + "CPU4": 0 + }, + { + "counter": "TIMER", + "CPU0": 322970, + "CPU1": 888166, + "CPU2": 0, + "CPU3": 0, + "CPU4": 0 + }, + ... + ] +""" +from typing import List, Dict +import jc.utils +from jc.parsers.universal import simple_table_parse + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/softirqs` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + cleandata = list(filter(None, data.splitlines())) + cleandata[0] = 'counter ' + cleandata[0] + raw_output = simple_table_parse(cleandata) + + for item in raw_output: + if 'counter' in item: + item['counter'] = item['counter'][:-1] + + for key in item: + try: + item[key] = int(item[key]) + except Exception: + pass + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index ab202336..506b65bd 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -505,6 +505,11 @@ PLIST file parser \fB--proc-slabinfo\fP `/proc/slabinfo` file parser +.TP +.B +\fB--proc-softirqs\fP +`/proc/softirqs` file parser + .TP .B \fB--proc-pid-numa-maps\fP From e171861629c878e3decb26c030c09d4cefbc76b3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 16 Sep 2022 15:54:13 -0700 Subject: [PATCH 50/71] add proc-stat parser --- docs/parsers/proc_stat.md | 161 ++++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc.py | 2 +- jc/parsers/proc_stat.py | 250 ++++++++++++++++++++++++++++++++++++++ man/jc.1 | 5 + 5 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 docs/parsers/proc_stat.md create mode 100644 jc/parsers/proc_stat.py diff --git a/docs/parsers/proc_stat.md b/docs/parsers/proc_stat.md new file mode 100644 index 00000000..1e7dc002 --- /dev/null +++ b/docs/parsers/proc_stat.md @@ -0,0 +1,161 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_stat + +jc - JSON Convert `/proc/stat` file parser + +Usage (cli): + + $ cat /proc/stat | jc --proc + +or + + $ jc /proc/stat + +or + + $ cat /proc/stat | jc --proc-stat + +Usage (module): + + import jc + result = jc.parse('proc', proc_stat_file) + +or + + import jc + result = jc.parse('proc_stat', proc_stat_file) + +Schema: + + { + "cpu": { + "user": integer, + "nice": integer, + "system": integer, + "idle": integer, + "iowait": integer, + "irq": integer, + "softirq": integer, + "steal": integer, + "guest": integer, + "guest_nice": integer + }, + "cpu": { + "user": integer, + "nice": integer, + "system": integer, + "idle": integer, + "iowait": integer, + "irq": integer, + "softirq": integer, + "steal": integer, + "guest": integer, + "guest_nice": integer + }, + "interrupts": [ + integer + ], + "context_switches": integer, + "boot_time": integer, + "processes": integer, + "processes_running": integer, + "processes_blocked": integer, + "softirq": [ + integer + ] + } + +Examples: + + $ cat /proc/stat | jc --proc -p + { + "cpu": { + "user": 6002, + "nice": 152, + "system": 8398, + "idle": 3444436, + "iowait": 448, + "irq": 0, + "softirq": 1174, + "steal": 0, + "guest": 0, + "guest_nice": 0 + }, + "cpu0": { + "user": 2784, + "nice": 137, + "system": 4367, + "idle": 1732802, + "iowait": 225, + "irq": 0, + "softirq": 221, + "steal": 0, + "guest": 0, + "guest_nice": 0 + }, + "cpu1": { + "user": 3218, + "nice": 15, + "system": 4031, + "idle": 1711634, + "iowait": 223, + "irq": 0, + "softirq": 953, + "steal": 0, + "guest": 0, + "guest_nice": 0 + }, + "interrupts": [ + 2496709, + 18, + 73, + 0, + 0, + ... + ], + "context_switches": 4622716, + "boot_time": 1662154781, + "processes": 9831, + "processes_running": 1, + "processes_blocked": 0, + "softirq": [ + 3478985, + 35230, + 1252057, + 3467, + 128583, + 51014, + 0, + 171199, + 1241297, + 0, + 596138 + ] + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 72c680bd..c43b3e33 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -105,6 +105,7 @@ parsers = [ 'proc-partitions', 'proc-slabinfo', 'proc-softirqs', + 'proc-stat', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc.py b/jc/parsers/proc.py index 69c5d550..872647de 100644 --- a/jc/parsers/proc.py +++ b/jc/parsers/proc.py @@ -171,7 +171,7 @@ def parse( partitions_p = re.compile(r'^major minor #blocks name\n\n\s*\d+\s+\d+\s+\d+ \w+\n') slabinfo_p = re.compile(r'^slabinfo - version: \d+.\d+\n') softirqs_p = re.compile(r'^\s+(CPU\d+\s+)+\n\s+HI:\s+\d') - stat_p = re.compile(r'^cpu\s+(?: \d+){10}.*intr ', re.DOTALL) + stat_p = re.compile(r'^cpu\s+(?: \d+){7,10}.*intr ', re.DOTALL) swaps_p = re.compile(r'^Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n') uptime_p = re.compile(r'^\d+.\d\d \d+.\d\d$') version_p = re.compile(r'^.+\sversion\s[^\n]+$') diff --git a/jc/parsers/proc_stat.py b/jc/parsers/proc_stat.py new file mode 100644 index 00000000..99f974cc --- /dev/null +++ b/jc/parsers/proc_stat.py @@ -0,0 +1,250 @@ +"""jc - JSON Convert `/proc/stat` file parser + +Usage (cli): + + $ cat /proc/stat | jc --proc + +or + + $ jc /proc/stat + +or + + $ cat /proc/stat | jc --proc-stat + +Usage (module): + + import jc + result = jc.parse('proc', proc_stat_file) + +or + + import jc + result = jc.parse('proc_stat', proc_stat_file) + +Schema: + + { + "cpu": { + "user": integer, + "nice": integer, + "system": integer, + "idle": integer, + "iowait": integer, + "irq": integer, + "softirq": integer, + "steal": integer, + "guest": integer, + "guest_nice": integer + }, + "cpu": { + "user": integer, + "nice": integer, + "system": integer, + "idle": integer, + "iowait": integer, + "irq": integer, + "softirq": integer, + "steal": integer, + "guest": integer, + "guest_nice": integer + }, + "interrupts": [ + integer + ], + "context_switches": integer, + "boot_time": integer, + "processes": integer, + "processes_running": integer, + "processes_blocked": integer, + "softirq": [ + integer + ] + } + +Examples: + + $ cat /proc/stat | jc --proc -p + { + "cpu": { + "user": 6002, + "nice": 152, + "system": 8398, + "idle": 3444436, + "iowait": 448, + "irq": 0, + "softirq": 1174, + "steal": 0, + "guest": 0, + "guest_nice": 0 + }, + "cpu0": { + "user": 2784, + "nice": 137, + "system": 4367, + "idle": 1732802, + "iowait": 225, + "irq": 0, + "softirq": 221, + "steal": 0, + "guest": 0, + "guest_nice": 0 + }, + "cpu1": { + "user": 3218, + "nice": 15, + "system": 4031, + "idle": 1711634, + "iowait": 223, + "irq": 0, + "softirq": 953, + "steal": 0, + "guest": 0, + "guest_nice": 0 + }, + "interrupts": [ + 2496709, + 18, + 73, + 0, + 0, + ... + ], + "context_switches": 4622716, + "boot_time": 1662154781, + "processes": 9831, + "processes_running": 1, + "processes_blocked": 0, + "softirq": [ + 3478985, + 35230, + 1252057, + 3467, + 128583, + 51014, + 0, + 171199, + 1241297, + 0, + 596138 + ] + } +""" +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/stat` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + if line.startswith('cpu'): + split_line = line.split() + cpu_num = split_line[0] + raw_output[cpu_num] = { + 'user': int(split_line[1]), + 'nice': int(split_line[2]), + 'system': int(split_line[3]), + 'idle': int(split_line[4]) + } + + if len(split_line) > 5: + raw_output[cpu_num]['iowait'] = int(split_line[5]) + + if len(split_line) > 6: + raw_output[cpu_num]['irq'] = int(split_line[6]) + raw_output[cpu_num]['softirq'] = int(split_line[7]) + + if len(split_line) > 8: + raw_output[cpu_num]['steal'] = int(split_line[8]) + + if len(split_line) > 9: + raw_output[cpu_num]['guest'] = int(split_line[9]) + + if len(split_line) > 10: + raw_output[cpu_num]['guest_nice'] = int(split_line[10]) + + continue + + if line.startswith('intr '): + split_line = line.split() + raw_output['interrupts'] = [int(x) for x in split_line[1:]] + continue + + if line.startswith('ctxt '): + raw_output['context_switches'] = int(line.split(maxsplit=1)[1]) + continue + + if line.startswith('btime '): + raw_output['boot_time'] = int(line.split(maxsplit=1)[1]) + continue + + if line.startswith('processes '): + raw_output['processes'] = int(line.split(maxsplit=1)[1]) + continue + + if line.startswith('procs_running '): + raw_output['processes_running'] = int(line.split(maxsplit=1)[1]) + continue + + if line.startswith('procs_blocked '): + raw_output['processes_blocked'] = int(line.split(maxsplit=1)[1]) + continue + + if line.startswith('softirq '): + split_line = line.split() + raw_output['softirq'] = [int(x) for x in split_line[1:]] + continue + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 506b65bd..d6d52b16 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -510,6 +510,11 @@ PLIST file parser \fB--proc-softirqs\fP `/proc/softirqs` file parser +.TP +.B +\fB--proc-stat\fP +`/proc/stat` file parser + .TP .B \fB--proc-pid-numa-maps\fP From bc816bb8584423ec39bd336d699c322e8ac1913e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 16 Sep 2022 16:04:25 -0700 Subject: [PATCH 51/71] add proc-swaps parser --- docs/parsers/proc_swaps.md | 91 +++++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_swaps.py | 132 +++++++++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 4 files changed, 229 insertions(+) create mode 100644 docs/parsers/proc_swaps.md create mode 100644 jc/parsers/proc_swaps.py diff --git a/docs/parsers/proc_swaps.md b/docs/parsers/proc_swaps.md new file mode 100644 index 00000000..47237358 --- /dev/null +++ b/docs/parsers/proc_swaps.md @@ -0,0 +1,91 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_swaps + +jc - JSON Convert `/proc/swaps` file parser + +Usage (cli): + + $ cat /proc/swaps | jc --proc + +or + + $ jc /proc/swaps + +or + + $ cat /proc/swaps | jc --proc-swaps + +Usage (module): + + import jc + result = jc.parse('proc', proc_swaps_file) + +or + + import jc + result = jc.parse('proc_swaps', proc_swaps_file) + +Schema: + + [ + { + "filename": string, + "type": string, + "size": integer, + "used": integer, + "priority": integer + } + ] + +Examples: + + $ cat /proc/swaps | jc --proc -p + [ + { + "filename": "/swap.img", + "type": "file", + "size": 3996668, + "used": 0, + "priority": -2 + }, + ... + ] + + $ cat /proc/swaps | jc --proc_swaps -p -r + [ + { + "filename": "/swap.img", + "type": "file", + "size": "3996668", + "used": "0", + "priority": "-2" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index c43b3e33..36e26946 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -106,6 +106,7 @@ parsers = [ 'proc-slabinfo', 'proc-softirqs', 'proc-stat', + 'proc-swaps', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_swaps.py b/jc/parsers/proc_swaps.py new file mode 100644 index 00000000..2669159d --- /dev/null +++ b/jc/parsers/proc_swaps.py @@ -0,0 +1,132 @@ +"""jc - JSON Convert `/proc/swaps` file parser + +Usage (cli): + + $ cat /proc/swaps | jc --proc + +or + + $ jc /proc/swaps + +or + + $ cat /proc/swaps | jc --proc-swaps + +Usage (module): + + import jc + result = jc.parse('proc', proc_swaps_file) + +or + + import jc + result = jc.parse('proc_swaps', proc_swaps_file) + +Schema: + + [ + { + "filename": string, + "type": string, + "size": integer, + "used": integer, + "priority": integer + } + ] + +Examples: + + $ cat /proc/swaps | jc --proc -p + [ + { + "filename": "/swap.img", + "type": "file", + "size": 3996668, + "used": 0, + "priority": -2 + }, + ... + ] + + $ cat /proc/swaps | jc --proc_swaps -p -r + [ + { + "filename": "/swap.img", + "type": "file", + "size": "3996668", + "used": "0", + "priority": "-2" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils +from jc.parsers.universal import simple_table_parse + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/swaps` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'size', 'used', 'priority'} + + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + cleandata = list(filter(None, data.splitlines())) + cleandata[0] = cleandata[0].lower() + raw_output = simple_table_parse(cleandata) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index d6d52b16..c2545fb4 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -515,6 +515,11 @@ PLIST file parser \fB--proc-stat\fP `/proc/stat` file parser +.TP +.B +\fB--proc-swaps\fP +`/proc/swaps` file parser + .TP .B \fB--proc-pid-numa-maps\fP From 0e35e857534adb2f6a713e3b85a8d5c4dcb7e4a2 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 16 Sep 2022 16:14:28 -0700 Subject: [PATCH 52/71] add proc-uptime parser --- docs/parsers/proc_uptime.md | 68 +++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_uptime.py | 104 ++++++++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 4 files changed, 178 insertions(+) create mode 100644 docs/parsers/proc_uptime.md create mode 100644 jc/parsers/proc_uptime.py diff --git a/docs/parsers/proc_uptime.md b/docs/parsers/proc_uptime.md new file mode 100644 index 00000000..47fadf70 --- /dev/null +++ b/docs/parsers/proc_uptime.md @@ -0,0 +1,68 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_uptime + +jc - JSON Convert `/proc/uptime` file parser + +Usage (cli): + + $ cat /proc/uptime | jc --proc + +or + + $ jc /proc/uptime + +or + + $ cat /proc/uptime | jc --proc-uptime + +Usage (module): + + import jc + result = jc.parse('proc', proc_uptime_file) + +or + + import jc + result = jc.parse('proc_uptime', proc_uptime_file) + +Schema: + + { + "up_time": float, + "idle_time": float + } + +Examples: + + $ cat /proc/uptime | jc --proc -p + { + "up_time": 46901.13, + "idle_time": 46856.66 + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 36e26946..535240de 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -107,6 +107,7 @@ parsers = [ 'proc-softirqs', 'proc-stat', 'proc-swaps', + 'proc-uptime', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_uptime.py b/jc/parsers/proc_uptime.py new file mode 100644 index 00000000..abea777f --- /dev/null +++ b/jc/parsers/proc_uptime.py @@ -0,0 +1,104 @@ +"""jc - JSON Convert `/proc/uptime` file parser + +Usage (cli): + + $ cat /proc/uptime | jc --proc + +or + + $ jc /proc/uptime + +or + + $ cat /proc/uptime | jc --proc-uptime + +Usage (module): + + import jc + result = jc.parse('proc', proc_uptime_file) + +or + + import jc + result = jc.parse('proc_uptime', proc_uptime_file) + +Schema: + + { + "up_time": float, + "idle_time": float + } + +Examples: + + $ cat /proc/uptime | jc --proc -p + { + "up_time": 46901.13, + "idle_time": 46856.66 + } +""" +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/uptime` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + + if jc.utils.has_data(data): + + uptime, idletime = data.split() + + raw_output = { + 'up_time': float(uptime), + 'idle_time': float(idletime) + } + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index c2545fb4..b59a8963 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -520,6 +520,11 @@ PLIST file parser \fB--proc-swaps\fP `/proc/swaps` file parser +.TP +.B +\fB--proc-uptime\fP +`/proc/uptime` file parser + .TP .B \fB--proc-pid-numa-maps\fP From 0bebb312dd208c61d508855e863a77b3ec88057d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 17 Sep 2022 12:32:15 -0700 Subject: [PATCH 53/71] add proc-version parser --- docs/parsers/proc_version.md | 79 ++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_version.py | 127 +++++++++++++++++++++++++++++++++++ man/jc.1 | 7 +- 4 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 docs/parsers/proc_version.md create mode 100644 jc/parsers/proc_version.py diff --git a/docs/parsers/proc_version.md b/docs/parsers/proc_version.md new file mode 100644 index 00000000..178ca9e1 --- /dev/null +++ b/docs/parsers/proc_version.md @@ -0,0 +1,79 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_version + +jc - JSON Convert `/proc/version` file parser + +> Note: This parser will parse `/proc/version` files that follow the +> common format used by most popular distributions. + +Usage (cli): + + $ cat /proc/version | jc --proc + +or + + $ jc /proc/version + +or + + $ cat /proc/version | jc --proc-version + +Usage (module): + + import jc + result = jc.parse('proc', proc_version_file) + +or + + import jc + result = jc.parse('proc_version', proc_version_file) + +Schema: + + { + "version": string, + "email": string, + "gcc": string, + "build": string, + "flags": string/null, + "date": string + } + +Examples: + + $ cat /proc/version | jc --proc -p + { + "version": "5.8.0-63-generic", + "email": "buildd@lcy01-amd64-028", + "gcc": "gcc (Ubuntu 10.3.0-1ubuntu1~20.10) 10.3.0, GNU ld (GNU Binutils for Ubuntu) 2.35.1", + "build": "#71-Ubuntu", + "flags": "SMP", + "date": "Tue Jul 13 15:59:12 UTC 2021" + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 535240de..cee0f3e6 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -108,6 +108,7 @@ parsers = [ 'proc-stat', 'proc-swaps', 'proc-uptime', + 'proc-version', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_version.py b/jc/parsers/proc_version.py new file mode 100644 index 00000000..89bab8a1 --- /dev/null +++ b/jc/parsers/proc_version.py @@ -0,0 +1,127 @@ +"""jc - JSON Convert `/proc/version` file parser + +> Note: This parser will parse `/proc/version` files that follow the +> common format used by most popular distributions. + +Usage (cli): + + $ cat /proc/version | jc --proc + +or + + $ jc /proc/version + +or + + $ cat /proc/version | jc --proc-version + +Usage (module): + + import jc + result = jc.parse('proc', proc_version_file) + +or + + import jc + result = jc.parse('proc_version', proc_version_file) + +Schema: + + { + "version": string, + "email": string, + "gcc": string, + "build": string, + "flags": string/null, + "date": string + } + +Examples: + + $ cat /proc/version | jc --proc -p + { + "version": "5.8.0-63-generic", + "email": "buildd@lcy01-amd64-028", + "gcc": "gcc (Ubuntu 10.3.0-1ubuntu1~20.10) 10.3.0, GNU ld (GNU Binutils for Ubuntu) 2.35.1", + "build": "#71-Ubuntu", + "flags": "SMP", + "date": "Tue Jul 13 15:59:12 UTC 2021" + } +""" +import re +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/version` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +# inspired by https://gist.github.com/ty2/ad61340e7a4067def2e3c709496bca9d +version_pattern = re.compile(r''' + Linux\ version\ (?P\S+)\s + \((?P\S+?)\)\s + \((?Pgcc.+)\)\s + (?P\#\d+(\S+)?)\s + (?P.*)? + (?P(Sun|Mon|Tue|Wed|Thu|Fri|Sat).+) + ''', re.VERBOSE +) + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + + if jc.utils.has_data(data): + version_match = version_pattern.match(data) + + if version_match: + + ver_dict = version_match.groupdict() + raw_output = {x: y.strip() or None for x, y in ver_dict.items()} + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index b59a8963..7d74df22 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-09-16 1.21.2 "JSON Convert" +.TH jc 1 2022-09-17 1.21.2 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS @@ -525,6 +525,11 @@ PLIST file parser \fB--proc-uptime\fP `/proc/uptime` file parser +.TP +.B +\fB--proc-version\fP +`/proc/version` file parser + .TP .B \fB--proc-pid-numa-maps\fP From a0ae19a8fdb901c637925a8c3bd7f3b7061868b0 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 17 Sep 2022 12:36:22 -0700 Subject: [PATCH 54/71] formatting --- docs/parsers/proc_version.md | 2 +- jc/parsers/proc_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parsers/proc_version.md b/docs/parsers/proc_version.md index 178ca9e1..5234e916 100644 --- a/docs/parsers/proc_version.md +++ b/docs/parsers/proc_version.md @@ -6,7 +6,7 @@ jc - JSON Convert `/proc/version` file parser > Note: This parser will parse `/proc/version` files that follow the -> common format used by most popular distributions. +> common format used by most popular linux distributions. Usage (cli): diff --git a/jc/parsers/proc_version.py b/jc/parsers/proc_version.py index 89bab8a1..8b888274 100644 --- a/jc/parsers/proc_version.py +++ b/jc/parsers/proc_version.py @@ -1,7 +1,7 @@ """jc - JSON Convert `/proc/version` file parser > Note: This parser will parse `/proc/version` files that follow the -> common format used by most popular distributions. +> common format used by most popular linux distributions. Usage (cli): From 28425cc493677bd0d198a382437875bea08ef200 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 17 Sep 2022 13:26:10 -0700 Subject: [PATCH 55/71] add proc-vmallocinfo parser --- docs/parsers/proc_vmallocinfo.md | 126 ++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_vmallocinfo.py | 195 +++++++++++++++++++++++++++++++ man/jc.1 | 5 + 4 files changed, 327 insertions(+) create mode 100644 docs/parsers/proc_vmallocinfo.md create mode 100644 jc/parsers/proc_vmallocinfo.py diff --git a/docs/parsers/proc_vmallocinfo.md b/docs/parsers/proc_vmallocinfo.md new file mode 100644 index 00000000..1eb11bf0 --- /dev/null +++ b/docs/parsers/proc_vmallocinfo.md @@ -0,0 +1,126 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_vmallocinfo + +jc - JSON Convert `/proc/vmallocinfo` file parser + +This parser will attempt to convert number values to integers. If that is +not desired, please use the `--raw` option (cli) or `raw=True` argument +(module). + +Usage (cli): + + $ cat /proc/vmallocinfo | jc --proc + +or + + $ jc /proc/vmallocinfo + +or + + $ cat /proc/vmallocinfo | jc --proc-vmallocinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_vmallocinfo_file) + +or + + import jc + result = jc.parse('proc_vmallocinfo', proc_vmallocinfo_file) + +Schema: + + [ + { + "start": string, + "end": string, + "size": integer, + "caller": string, + "options": [ + string + ], + "phys": string + "pages": integer, + "N": integer + } + ] + +Examples: + + $ cat /proc/vmallocinfo | jc --proc -p + [ + { + "start": "0xffffb3c1c0000000", + "end": "0xffffb3c1c0005000", + "size": 20480, + "caller": "map_irq_stack+0x93/0xe0", + "options": [ + "vmap" + ], + "phys": "0x00000000bfefe000" + }, + { + "start": "0xffffb3c1c0005000", + "end": "0xffffb3c1c0007000", + "size": 8192, + "caller": "acpi_os_map_iomem+0x1ac/0x1c0", + "options": [ + "ioremap" + ], + "phys": "0x00000000bfeff000" + }, + ... + ] + + $ cat /proc/vmallocinfo | jc --proc -p -r + [ + { + "start": "0xffffb3c1c0000000", + "end": "0xffffb3c1c0005000", + "size": "20480", + "caller": "map_irq_stack+0x93/0xe0", + "options": [ + "vmap" + ], + "phys": "0x00000000bfefe000" + }, + { + "start": "0xffffb3c1c0005000", + "end": "0xffffb3c1c0007000", + "size": "8192", + "caller": "acpi_os_map_iomem+0x1ac/0x1c0", + "options": [ + "ioremap" + ], + "phys": "0x00000000bfeff000" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index cee0f3e6..88fa425a 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -109,6 +109,7 @@ parsers = [ 'proc-swaps', 'proc-uptime', 'proc-version', + 'proc-vmallocinfo', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_vmallocinfo.py b/jc/parsers/proc_vmallocinfo.py new file mode 100644 index 00000000..b2044264 --- /dev/null +++ b/jc/parsers/proc_vmallocinfo.py @@ -0,0 +1,195 @@ +"""jc - JSON Convert `/proc/vmallocinfo` file parser + +This parser will attempt to convert number values to integers. If that is +not desired, please use the `--raw` option (cli) or `raw=True` argument +(module). + +Usage (cli): + + $ cat /proc/vmallocinfo | jc --proc + +or + + $ jc /proc/vmallocinfo + +or + + $ cat /proc/vmallocinfo | jc --proc-vmallocinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_vmallocinfo_file) + +or + + import jc + result = jc.parse('proc_vmallocinfo', proc_vmallocinfo_file) + +Schema: + + [ + { + "start": string, + "end": string, + "size": integer, + "caller": string, + "options": [ + string + ], + "phys": string + "pages": integer, + "N": integer + } + ] + +Examples: + + $ cat /proc/vmallocinfo | jc --proc -p + [ + { + "start": "0xffffb3c1c0000000", + "end": "0xffffb3c1c0005000", + "size": 20480, + "caller": "map_irq_stack+0x93/0xe0", + "options": [ + "vmap" + ], + "phys": "0x00000000bfefe000" + }, + { + "start": "0xffffb3c1c0005000", + "end": "0xffffb3c1c0007000", + "size": 8192, + "caller": "acpi_os_map_iomem+0x1ac/0x1c0", + "options": [ + "ioremap" + ], + "phys": "0x00000000bfeff000" + }, + ... + ] + + $ cat /proc/vmallocinfo | jc --proc -p -r + [ + { + "start": "0xffffb3c1c0000000", + "end": "0xffffb3c1c0005000", + "size": "20480", + "caller": "map_irq_stack+0x93/0xe0", + "options": [ + "vmap" + ], + "phys": "0x00000000bfefe000" + }, + { + "start": "0xffffb3c1c0005000", + "end": "0xffffb3c1c0007000", + "size": "8192", + "caller": "acpi_os_map_iomem+0x1ac/0x1c0", + "options": [ + "ioremap" + ], + "phys": "0x00000000bfeff000" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/vmallocinfo` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + for entry in proc_data: + for key in entry: + if isinstance(entry[key], str): + try: + entry[key] = int(entry[key]) + except Exception: + pass + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + output_line: Dict = {} + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + area, size, details = line.split(maxsplit=2) + start, end = area.split('-', maxsplit=1) + detail_split = details.split() + caller = '' + options: List = [] + + + if details == 'unpurged vm_area': + caller = 'unpurged vm_area' + + else: + caller = detail_split[0] + for item in detail_split[1:]: + if '=' in item: + key, val = item.split('=') + output_line.update({key: val}) + else: + options.append(item) + + output_line = { + 'start': start, + 'end': end, + 'size': size, + 'caller': caller or None, + 'options': options + } + + if output_line: + raw_output.append(output_line) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 7d74df22..d29f7b9a 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -530,6 +530,11 @@ PLIST file parser \fB--proc-version\fP `/proc/version` file parser +.TP +.B +\fB--proc-vmallocinfo\fP +`/proc/vmallocinfo` file parser + .TP .B \fB--proc-pid-numa-maps\fP From 52d252f1996d5c13691066ec1fb9605f0357bcd6 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 17 Sep 2022 13:27:50 -0700 Subject: [PATCH 56/71] formatting --- jc/parsers/proc_vmallocinfo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jc/parsers/proc_vmallocinfo.py b/jc/parsers/proc_vmallocinfo.py index b2044264..86f58eed 100644 --- a/jc/parsers/proc_vmallocinfo.py +++ b/jc/parsers/proc_vmallocinfo.py @@ -168,7 +168,6 @@ def parse( caller = '' options: List = [] - if details == 'unpurged vm_area': caller = 'unpurged vm_area' From e5913cd10d80ccb8147aec15fe1168404915e4a1 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 17 Sep 2022 13:33:31 -0700 Subject: [PATCH 57/71] add proc-vmstat parser --- jc/parsers/proc_vmstat.py | 117 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 jc/parsers/proc_vmstat.py diff --git a/jc/parsers/proc_vmstat.py b/jc/parsers/proc_vmstat.py new file mode 100644 index 00000000..128d0f70 --- /dev/null +++ b/jc/parsers/proc_vmstat.py @@ -0,0 +1,117 @@ +"""jc - JSON Convert `/proc/vmstat` file parser + +Usage (cli): + + $ cat /proc/vmstat | jc --proc + +or + + $ jc /proc/vmstat + +or + + $ cat /proc/vmstat | jc --proc-vmstat + +Usage (module): + + import jc + result = jc.parse('proc', proc_vmstat_file) + +or + + import jc + result = jc.parse('proc_vmstat', proc_vmstat_file) + +Schema: + +All values are integers. + + { + integer + } + +Examples: + + $ cat /proc/vmstat | jc --proc -p + { + "nr_free_pages": 615337, + "nr_zone_inactive_anon": 39, + "nr_zone_active_anon": 34838, + "nr_zone_inactive_file": 104036, + "nr_zone_active_file": 130601, + "nr_zone_unevictable": 4897, + "nr_zone_write_pending": 45, + "nr_mlock": 4897, + "nr_page_table_pages": 548, + "nr_kernel_stack": 5984, + "nr_bounce": 0, + "nr_zspages": 0, + "nr_free_cma": 0, + "numa_hit": 1910597, + "numa_miss": 0, + "numa_foreign": 0, + ... + } +""" +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/vmstat` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + key, val = line.split(maxsplit=1) + raw_output[key] = int(val) + + return raw_output if raw else _process(raw_output) From 906eeefa522098794b6397bff1e5a28567cf0354 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 17 Sep 2022 13:34:40 -0700 Subject: [PATCH 58/71] add proc-vmstat parser --- docs/parsers/proc_vmstat.md | 84 +++++++++++++++++++++++++++++++++++++ jc/lib.py | 1 + man/jc.1 | 5 +++ 3 files changed, 90 insertions(+) create mode 100644 docs/parsers/proc_vmstat.md diff --git a/docs/parsers/proc_vmstat.md b/docs/parsers/proc_vmstat.md new file mode 100644 index 00000000..d1e6a098 --- /dev/null +++ b/docs/parsers/proc_vmstat.md @@ -0,0 +1,84 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_vmstat + +jc - JSON Convert `/proc/vmstat` file parser + +Usage (cli): + + $ cat /proc/vmstat | jc --proc + +or + + $ jc /proc/vmstat + +or + + $ cat /proc/vmstat | jc --proc-vmstat + +Usage (module): + + import jc + result = jc.parse('proc', proc_vmstat_file) + +or + + import jc + result = jc.parse('proc_vmstat', proc_vmstat_file) + +Schema: + +All values are integers. + + { + integer + } + +Examples: + + $ cat /proc/vmstat | jc --proc -p + { + "nr_free_pages": 615337, + "nr_zone_inactive_anon": 39, + "nr_zone_active_anon": 34838, + "nr_zone_inactive_file": 104036, + "nr_zone_active_file": 130601, + "nr_zone_unevictable": 4897, + "nr_zone_write_pending": 45, + "nr_mlock": 4897, + "nr_page_table_pages": 548, + "nr_kernel_stack": 5984, + "nr_bounce": 0, + "nr_zspages": 0, + "nr_free_cma": 0, + "numa_hit": 1910597, + "numa_miss": 0, + "numa_foreign": 0, + ... + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 88fa425a..a8777f9c 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -110,6 +110,7 @@ parsers = [ 'proc-uptime', 'proc-version', 'proc-vmallocinfo', + 'proc-vmstat', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/man/jc.1 b/man/jc.1 index d29f7b9a..d679395c 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -535,6 +535,11 @@ PLIST file parser \fB--proc-vmallocinfo\fP `/proc/vmallocinfo` file parser +.TP +.B +\fB--proc-vmstat\fP +`/proc/vmstat` file parser + .TP .B \fB--proc-pid-numa-maps\fP From 92a044ba9b9163a1e2ce7c8db359210964ed2965 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 17 Sep 2022 17:39:16 -0700 Subject: [PATCH 59/71] add proc-zoneinfo parser --- docs/parsers/proc_zoneinfo.md | 334 +++++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_zoneinfo.py | 448 ++++++++++++++++++++++++++++++++++ man/jc.1 | 5 + 4 files changed, 788 insertions(+) create mode 100644 docs/parsers/proc_zoneinfo.md create mode 100644 jc/parsers/proc_zoneinfo.py diff --git a/docs/parsers/proc_zoneinfo.md b/docs/parsers/proc_zoneinfo.md new file mode 100644 index 00000000..e21faf8a --- /dev/null +++ b/docs/parsers/proc_zoneinfo.md @@ -0,0 +1,334 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_zoneinfo + +jc - JSON Convert `/proc/zoneinfo` file parser + +Usage (cli): + + $ cat /proc/zoneinfo | jc --proc + +or + + $ jc /proc/zoneinfo + +or + + $ cat /proc/zoneinfo | jc --proc-zoneinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_zoneinfo_file) + +or + + import jc + result = jc.parse('proc_zoneinfo', proc_zoneinfo_file) + +Schema: + +All values are integers. + + [ + { + "node": integer, + "": { + "pages": { + "free": integer, + "min": integer, + "low": integer, + "high": integer, + "spanned": integer, + "present": integer, + "managed": integer, + "protection": [ + integer + ], + "": integer + }, + "pagesets": [ + { + "cpu": integer, + "count": integer, + "high": integer, + "batch": integer, + "vm stats threshold": integer, + "": integer + } + ] + }, + "": integer, # [0] + } + ] + + [0] per-node stats + +Examples: + + $ cat /proc/zoneinfo | jc --proc -p + [ + { + "node": 0, + "DMA": { + "pages": { + "free": 3832, + "min": 68, + "low": 85, + "high": 102, + "spanned": 4095, + "present": 3997, + "managed": 3976, + "protection": [ + 0, + 2871, + 3795, + 3795, + 3795 + ], + "nr_free_pages": 3832, + "nr_zone_inactive_anon": 0, + "nr_zone_active_anon": 0, + "nr_zone_inactive_file": 0, + "nr_zone_active_file": 0, + "nr_zone_unevictable": 0, + "nr_zone_write_pending": 0, + "nr_mlock": 0, + "nr_page_table_pages": 0, + "nr_kernel_stack": 0, + "nr_bounce": 0, + "nr_zspages": 0, + "nr_free_cma": 0, + "numa_hit": 3, + "numa_miss": 0, + "numa_foreign": 0, + "numa_interleave": 1, + "numa_local": 3, + "numa_other": 0 + }, + "pagesets": [ + { + "cpu": 0, + "count": 0, + "high": 0, + "batch": 1, + "vm stats threshold": 4 + }, + { + "cpu": 1, + "count": 0, + "high": 0, + "batch": 1, + "vm stats threshold": 4, + "node_unreclaimable": 0, + "start_pfn": 1 + } + ] + }, + "nr_inactive_anon": 39, + "nr_active_anon": 34839, + "nr_inactive_file": 104172, + "nr_active_file": 130748, + "nr_unevictable": 4897, + "nr_slab_reclaimable": 49017, + "nr_slab_unreclaimable": 26177, + "nr_isolated_anon": 0, + "nr_isolated_file": 0, + "workingset_nodes": 0, + "workingset_refault": 0, + "workingset_activate": 0, + "workingset_restore": 0, + "workingset_nodereclaim": 0, + "nr_anon_pages": 40299, + "nr_mapped": 25140, + "nr_file_pages": 234396, + "nr_dirty": 0, + "nr_writeback": 0, + "nr_writeback_temp": 0, + "nr_shmem": 395, + "nr_shmem_hugepages": 0, + "nr_shmem_pmdmapped": 0, + "nr_file_hugepages": 0, + "nr_file_pmdmapped": 0, + "nr_anon_transparent_hugepages": 0, + "nr_vmscan_write": 0, + "nr_vmscan_immediate_reclaim": 0, + "nr_dirtied": 168223, + "nr_written": 144616, + "nr_kernel_misc_reclaimable": 0, + "nr_foll_pin_acquired": 0, + "nr_foll_pin_released": 0, + "DMA32": { + "pages": { + "free": 606010, + "min": 12729, + "low": 15911, + "high": 19093, + "spanned": 1044480, + "present": 782288, + "managed": 758708, + "protection": [ + 0, + 0, + 924, + 924, + 924 + ], + "nr_free_pages": 606010, + "nr_zone_inactive_anon": 4, + "nr_zone_active_anon": 17380, + "nr_zone_inactive_file": 41785, + "nr_zone_active_file": 64545, + "nr_zone_unevictable": 5, + "nr_zone_write_pending": 0, + "nr_mlock": 5, + "nr_page_table_pages": 101, + "nr_kernel_stack": 224, + "nr_bounce": 0, + "nr_zspages": 0, + "nr_free_cma": 0, + "numa_hit": 576595, + "numa_miss": 0, + "numa_foreign": 0, + "numa_interleave": 2, + "numa_local": 576595, + "numa_other": 0 + }, + "pagesets": [ + { + "cpu": 0, + "count": 253, + "high": 378, + "batch": 63, + "vm stats threshold": 24 + }, + { + "cpu": 1, + "count": 243, + "high": 378, + "batch": 63, + "vm stats threshold": 24, + "node_unreclaimable": 0, + "start_pfn": 4096 + } + ] + }, + "Normal": { + "pages": { + "free": 5113, + "min": 4097, + "low": 5121, + "high": 6145, + "spanned": 262144, + "present": 262144, + "managed": 236634, + "protection": [ + 0, + 0, + 0, + 0, + 0 + ], + "nr_free_pages": 5113, + "nr_zone_inactive_anon": 35, + "nr_zone_active_anon": 17459, + "nr_zone_inactive_file": 62387, + "nr_zone_active_file": 66203, + "nr_zone_unevictable": 4892, + "nr_zone_write_pending": 0, + "nr_mlock": 4892, + "nr_page_table_pages": 447, + "nr_kernel_stack": 5760, + "nr_bounce": 0, + "nr_zspages": 0, + "nr_free_cma": 0, + "numa_hit": 1338441, + "numa_miss": 0, + "numa_foreign": 0, + "numa_interleave": 66037, + "numa_local": 1338441, + "numa_other": 0 + }, + "pagesets": [ + { + "cpu": 0, + "count": 340, + "high": 378, + "batch": 63, + "vm stats threshold": 16 + }, + { + "cpu": 1, + "count": 174, + "high": 378, + "batch": 63, + "vm stats threshold": 16, + "node_unreclaimable": 0, + "start_pfn": 1048576 + } + ] + }, + "Movable": { + "pages": { + "free": 0, + "min": 0, + "low": 0, + "high": 0, + "spanned": 0, + "present": 0, + "managed": 0, + "protection": [ + 0, + 0, + 0, + 0, + 0 + ] + } + }, + "Device": { + "pages": { + "free": 0, + "min": 0, + "low": 0, + "high": 0, + "spanned": 0, + "present": 0, + "managed": 0, + "protection": [ + 0, + 0, + 0, + 0, + 0 + ] + } + } + } + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index a8777f9c..0a01105a 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -111,6 +111,7 @@ parsers = [ 'proc-version', 'proc-vmallocinfo', 'proc-vmstat', + 'proc-zoneinfo', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_zoneinfo.py b/jc/parsers/proc_zoneinfo.py new file mode 100644 index 00000000..d2f92be7 --- /dev/null +++ b/jc/parsers/proc_zoneinfo.py @@ -0,0 +1,448 @@ +"""jc - JSON Convert `/proc/zoneinfo` file parser + +Usage (cli): + + $ cat /proc/zoneinfo | jc --proc + +or + + $ jc /proc/zoneinfo + +or + + $ cat /proc/zoneinfo | jc --proc-zoneinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_zoneinfo_file) + +or + + import jc + result = jc.parse('proc_zoneinfo', proc_zoneinfo_file) + +Schema: + +All values are integers. + + [ + { + "node": integer, + "": { + "pages": { + "free": integer, + "min": integer, + "low": integer, + "high": integer, + "spanned": integer, + "present": integer, + "managed": integer, + "protection": [ + integer + ], + "": integer + }, + "pagesets": [ + { + "cpu": integer, + "count": integer, + "high": integer, + "batch": integer, + "vm stats threshold": integer, + "": integer + } + ] + }, + "": integer, # [0] + } + ] + + [0] per-node stats + +Examples: + + $ cat /proc/zoneinfo | jc --proc -p + [ + { + "node": 0, + "DMA": { + "pages": { + "free": 3832, + "min": 68, + "low": 85, + "high": 102, + "spanned": 4095, + "present": 3997, + "managed": 3976, + "protection": [ + 0, + 2871, + 3795, + 3795, + 3795 + ], + "nr_free_pages": 3832, + "nr_zone_inactive_anon": 0, + "nr_zone_active_anon": 0, + "nr_zone_inactive_file": 0, + "nr_zone_active_file": 0, + "nr_zone_unevictable": 0, + "nr_zone_write_pending": 0, + "nr_mlock": 0, + "nr_page_table_pages": 0, + "nr_kernel_stack": 0, + "nr_bounce": 0, + "nr_zspages": 0, + "nr_free_cma": 0, + "numa_hit": 3, + "numa_miss": 0, + "numa_foreign": 0, + "numa_interleave": 1, + "numa_local": 3, + "numa_other": 0 + }, + "pagesets": [ + { + "cpu": 0, + "count": 0, + "high": 0, + "batch": 1, + "vm stats threshold": 4 + }, + { + "cpu": 1, + "count": 0, + "high": 0, + "batch": 1, + "vm stats threshold": 4, + "node_unreclaimable": 0, + "start_pfn": 1 + } + ] + }, + "nr_inactive_anon": 39, + "nr_active_anon": 34839, + "nr_inactive_file": 104172, + "nr_active_file": 130748, + "nr_unevictable": 4897, + "nr_slab_reclaimable": 49017, + "nr_slab_unreclaimable": 26177, + "nr_isolated_anon": 0, + "nr_isolated_file": 0, + "workingset_nodes": 0, + "workingset_refault": 0, + "workingset_activate": 0, + "workingset_restore": 0, + "workingset_nodereclaim": 0, + "nr_anon_pages": 40299, + "nr_mapped": 25140, + "nr_file_pages": 234396, + "nr_dirty": 0, + "nr_writeback": 0, + "nr_writeback_temp": 0, + "nr_shmem": 395, + "nr_shmem_hugepages": 0, + "nr_shmem_pmdmapped": 0, + "nr_file_hugepages": 0, + "nr_file_pmdmapped": 0, + "nr_anon_transparent_hugepages": 0, + "nr_vmscan_write": 0, + "nr_vmscan_immediate_reclaim": 0, + "nr_dirtied": 168223, + "nr_written": 144616, + "nr_kernel_misc_reclaimable": 0, + "nr_foll_pin_acquired": 0, + "nr_foll_pin_released": 0, + "DMA32": { + "pages": { + "free": 606010, + "min": 12729, + "low": 15911, + "high": 19093, + "spanned": 1044480, + "present": 782288, + "managed": 758708, + "protection": [ + 0, + 0, + 924, + 924, + 924 + ], + "nr_free_pages": 606010, + "nr_zone_inactive_anon": 4, + "nr_zone_active_anon": 17380, + "nr_zone_inactive_file": 41785, + "nr_zone_active_file": 64545, + "nr_zone_unevictable": 5, + "nr_zone_write_pending": 0, + "nr_mlock": 5, + "nr_page_table_pages": 101, + "nr_kernel_stack": 224, + "nr_bounce": 0, + "nr_zspages": 0, + "nr_free_cma": 0, + "numa_hit": 576595, + "numa_miss": 0, + "numa_foreign": 0, + "numa_interleave": 2, + "numa_local": 576595, + "numa_other": 0 + }, + "pagesets": [ + { + "cpu": 0, + "count": 253, + "high": 378, + "batch": 63, + "vm stats threshold": 24 + }, + { + "cpu": 1, + "count": 243, + "high": 378, + "batch": 63, + "vm stats threshold": 24, + "node_unreclaimable": 0, + "start_pfn": 4096 + } + ] + }, + "Normal": { + "pages": { + "free": 5113, + "min": 4097, + "low": 5121, + "high": 6145, + "spanned": 262144, + "present": 262144, + "managed": 236634, + "protection": [ + 0, + 0, + 0, + 0, + 0 + ], + "nr_free_pages": 5113, + "nr_zone_inactive_anon": 35, + "nr_zone_active_anon": 17459, + "nr_zone_inactive_file": 62387, + "nr_zone_active_file": 66203, + "nr_zone_unevictable": 4892, + "nr_zone_write_pending": 0, + "nr_mlock": 4892, + "nr_page_table_pages": 447, + "nr_kernel_stack": 5760, + "nr_bounce": 0, + "nr_zspages": 0, + "nr_free_cma": 0, + "numa_hit": 1338441, + "numa_miss": 0, + "numa_foreign": 0, + "numa_interleave": 66037, + "numa_local": 1338441, + "numa_other": 0 + }, + "pagesets": [ + { + "cpu": 0, + "count": 340, + "high": 378, + "batch": 63, + "vm stats threshold": 16 + }, + { + "cpu": 1, + "count": 174, + "high": 378, + "batch": 63, + "vm stats threshold": 16, + "node_unreclaimable": 0, + "start_pfn": 1048576 + } + ] + }, + "Movable": { + "pages": { + "free": 0, + "min": 0, + "low": 0, + "high": 0, + "spanned": 0, + "present": 0, + "managed": 0, + "protection": [ + 0, + 0, + 0, + 0, + 0 + ] + } + }, + "Device": { + "pages": { + "free": 0, + "min": 0, + "low": 0, + "high": 0, + "spanned": 0, + "present": 0, + "managed": 0, + "protection": [ + 0, + 0, + 0, + 0, + 0 + ] + } + } + } + ] +""" +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/zoneinfo` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'size', 'used'} + + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + ouptput_line: Dict = {} + node = None + section = 'stats' # stats, pages, pagesets + pageset = None + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + if line == ' per-node stats': + continue + + if line.startswith('Node ') and line.endswith('DMA'): + if ouptput_line: + raw_output.append(ouptput_line) + ouptput_line = {} + + section = 'stats' + _, node, _, zone = line.replace(',', '').split() + ouptput_line['node'] = int(node) + ouptput_line[zone] = {} + continue + + if line.startswith('Node '): + section = 'stats' + + if pageset: + ouptput_line[zone]['pagesets'].append(pageset) + pageset = {} + + _, node, _, zone = line.replace(',', '').split() + ouptput_line['node'] = int(node) + ouptput_line[zone] = {} + continue + + if line.startswith(' pages free '): + section = 'pages' + ouptput_line[zone]['pages'] = {} + ouptput_line[zone]['pages']['free'] = int(line.split()[-1]) + continue + + if line.startswith(' pagesets'): + section = 'pagesets' + ouptput_line[zone]['pagesets'] = [] + pageset = {} # type: ignore + continue + + if section == 'stats': + key, val = line.split(maxsplit=1) + ouptput_line[key] = int(val) + continue + + if section == 'pages' and line.startswith(' protection: '): + protection = line.replace('(', '').replace(')', '').replace(',', '').split()[1:] + ouptput_line[zone]['pages']['protection'] = [int(x) for x in protection] + continue + + if section == 'pages': + key, val = line.split(maxsplit=1) + ouptput_line[zone]['pages'][key] = int(val) + continue + + if section == 'pagesets' and line.startswith(' cpu: '): + if pageset: + ouptput_line[zone]['pagesets'].append(pageset) + + split_line = line.replace(':', '').split(maxsplit=1) + pageset = {"cpu": int(split_line[1])} + continue + + if section == 'pagesets': + key, val = line.split(':', maxsplit=1) + pageset[key.strip()] = int(val) + continue + + if ouptput_line: + if pageset: + ouptput_line[zone]['pagesets'].append(pageset) + + raw_output.append(ouptput_line) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index d679395c..614b5b62 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -540,6 +540,11 @@ PLIST file parser \fB--proc-vmstat\fP `/proc/vmstat` file parser +.TP +.B +\fB--proc-zoneinfo\fP +`/proc/zoneinfo` file parser + .TP .B \fB--proc-pid-numa-maps\fP From e623ceacc8cd282742103f81505efe2afd1660eb Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 17 Sep 2022 17:43:21 -0700 Subject: [PATCH 60/71] remove unused _process code --- jc/parsers/proc_zoneinfo.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/jc/parsers/proc_zoneinfo.py b/jc/parsers/proc_zoneinfo.py index d2f92be7..ec3db99b 100644 --- a/jc/parsers/proc_zoneinfo.py +++ b/jc/parsers/proc_zoneinfo.py @@ -332,13 +332,6 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = {'size', 'used'} - - for entry in proc_data: - for key in entry: - if key in int_list: - entry[key] = jc.utils.convert_to_int(entry[key]) - return proc_data From c88bf3e94b1a028d4df17fa82e162e21d9670faf Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 17 Sep 2022 17:50:43 -0700 Subject: [PATCH 61/71] fix signature order --- jc/parsers/proc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jc/parsers/proc.py b/jc/parsers/proc.py index 872647de..0c6bdd5c 100644 --- a/jc/parsers/proc.py +++ b/jc/parsers/proc.py @@ -233,8 +233,8 @@ def parse( uptime_p: 'proc_uptime', version_p: 'proc_version', vmallocinfo_p: 'proc_vmallocinfo', - vmstat_p: 'proc_vmstat', - zoneinfo_p: 'proc_zoneinfo', + zoneinfo_p: 'proc_zoneinfo', # before vmstat + vmstat_p: 'proc_vmstat', # after zoneinfo driver_rtc_p: 'proc_driver_rtc', From 8ffde41fa42ae9806f319fd6b1cca3a2d6337f43 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 19 Sep 2022 16:50:01 -0700 Subject: [PATCH 62/71] add proc-pid-fdinfo parser --- docs/parsers/proc_pid_fdinfo.md | 127 +++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc.py | 16 +-- jc/parsers/proc_pid_fdinfo.py | 217 ++++++++++++++++++++++++++++++++ man/jc.1 | 7 +- 5 files changed, 359 insertions(+), 9 deletions(-) create mode 100644 docs/parsers/proc_pid_fdinfo.md create mode 100644 jc/parsers/proc_pid_fdinfo.py diff --git a/docs/parsers/proc_pid_fdinfo.md b/docs/parsers/proc_pid_fdinfo.md new file mode 100644 index 00000000..4f266668 --- /dev/null +++ b/docs/parsers/proc_pid_fdinfo.md @@ -0,0 +1,127 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_pid\_fdinfo + +jc - JSON Convert `/proc//fdinfo` file parser + +Usage (cli): + + $ cat /proc/1/fdinfo/5 | jc --proc + +or + + $ jc /proc/1/fdinfo/5 + +or + + $ cat /proc/1/fdinfo/5 | jc --proc-pid-fdinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_pid_fdinfo_file) + +or + + import jc + result = jc.parse('proc_pid_fdinfo', proc_pid_fdinfo_file) + +Schema: + +Any unspecified fields are strings. + + { + "pos": integer, + "flags": integer, + "mnt_id": integer, + "scm_fds": string, + "ino": integer, + "lock": string, + "epoll": { + "tfd": integer, + "events": string, + "data": string, + "pos": integer, + "ino": string, + "sdev": string + }, + "inotify": { + "wd": integer, + "ino": string, + "sdev": string, + "mask": string, + "ignored_mask": string, + "fhandle-bytes": string, + "fhandle-type": string, + "f_handle": string + }, + "fanotify": { + "flags": string, + "event-flags": string, + "mnt_id": string, + "mflags": string, + "mask": string, + "ignored_mask": string, + "ino": string, + "sdev": string, + "fhandle-bytes": string, + "fhandle-type": string, + "f_handle": string + }, + "clockid": integer, + "ticks": integer, + "settime flags": integer, + "it_value": [ + integer + ], + "it_interval": [ + integer + ] + } + +Examples: + + $ cat /proc/1/fdinfo/5 | jc --proc -p + { + "pos": 0, + "flags": 2, + "mnt_id": 9, + "ino": 63107, + "clockid": 0, + "ticks": 0, + "settime flags": 1, + "it_value": [ + 0, + 49406829 + ], + "it_interval": [ + 1, + 0 + ] + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 0a01105a..f2506197 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -112,6 +112,7 @@ parsers = [ 'proc-vmallocinfo', 'proc-vmstat', 'proc-zoneinfo', + 'proc-pid-fdinfo', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc.py b/jc/parsers/proc.py index 0c6bdd5c..cfe59399 100644 --- a/jc/parsers/proc.py +++ b/jc/parsers/proc.py @@ -195,15 +195,15 @@ def parse( net_route_p = re.compile(r'^Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\s+\n') net_unix_p = re.compile(r'^Num RefCount Protocol Flags Type St Inode Path\n') - pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n') - pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$') - pid_stat_p = re.compile(r'^\d+ \(.{1,16}\) \w \d+ \d+ \d+ \d+ -?\d+ (?:\d+ ){43}\d+$') - pid_smaps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ [^\n]+\nSize:\s+\d+ \S\S\n') - pid_maps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ ') - pid_numa_maps_p = re.compile(r'^[a-f0-9]{12} default [^\n]+\n') - pid_io_p = re.compile(r'^rchar: \d+\nwchar: \d+\nsyscr: \d+\n') - pid_mountinfo_p = re.compile(r'^\d+ \d+ \d+:\d+ /.+\n') pid_fdinfo_p = re.compile(r'^pos:\t\d+\nflags:\t\d+\nmnt_id:\t\d+\n') + pid_io_p = re.compile(r'^rchar: \d+\nwchar: \d+\nsyscr: \d+\n') + pid_maps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ ') + pid_mountinfo_p = re.compile(r'^\d+ \d+ \d+:\d+ /.+\n') + pid_numa_maps_p = re.compile(r'^[a-f0-9]{12} default [^\n]+\n') + pid_smaps_p = re.compile(r'^[0-9a-f]{12}-[0-9a-f]{12} [rwxsp\-]{4} [0-9a-f]{8} [0-9a-f]{2}:[0-9a-f]{2} \d+ [^\n]+\nSize:\s+\d+ \S\S\n') + pid_statm_p = re.compile(r'^\d+ \d+ \d+\s\d+\s\d+\s\d+\s\d+$') + pid_status_p = re.compile(r'^Name:\t.+\nUmask:\t\d+\nState:\t.+\nTgid:\t\d+\n') + pid_stat_p = re.compile(r'^\d+ \(.{1,16}\) \w \d+ \d+ \d+ \d+ -?\d+ (?:\d+ ){43}\d+$') scsi_device_info = re.compile(r"^'\w+' '.+' 0x\d+") scsi_scsi_p = re.compile(r'^Attached devices:\nHost: \w+ ') diff --git a/jc/parsers/proc_pid_fdinfo.py b/jc/parsers/proc_pid_fdinfo.py new file mode 100644 index 00000000..b752c30f --- /dev/null +++ b/jc/parsers/proc_pid_fdinfo.py @@ -0,0 +1,217 @@ +"""jc - JSON Convert `/proc//fdinfo` file parser + +Usage (cli): + + $ cat /proc/1/fdinfo/5 | jc --proc + +or + + $ jc /proc/1/fdinfo/5 + +or + + $ cat /proc/1/fdinfo/5 | jc --proc-pid-fdinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_pid_fdinfo_file) + +or + + import jc + result = jc.parse('proc_pid_fdinfo', proc_pid_fdinfo_file) + +Schema: + +Any unspecified fields are strings. + + { + "pos": integer, + "flags": integer, + "mnt_id": integer, + "scm_fds": string, + "ino": integer, + "lock": string, + "epoll": { + "tfd": integer, + "events": string, + "data": string, + "pos": integer, + "ino": string, + "sdev": string + }, + "inotify": { + "wd": integer, + "ino": string, + "sdev": string, + "mask": string, + "ignored_mask": string, + "fhandle-bytes": string, + "fhandle-type": string, + "f_handle": string + }, + "fanotify": { + "flags": string, + "event-flags": string, + "mnt_id": string, + "mflags": string, + "mask": string, + "ignored_mask": string, + "ino": string, + "sdev": string, + "fhandle-bytes": string, + "fhandle-type": string, + "f_handle": string + }, + "clockid": integer, + "ticks": integer, + "settime flags": integer, + "it_value": [ + integer + ], + "it_interval": [ + integer + ] + } + +Examples: + + $ cat /proc/1/fdinfo/5 | jc --proc -p + { + "pos": 0, + "flags": 2, + "mnt_id": 9, + "ino": 63107, + "clockid": 0, + "ticks": 0, + "settime flags": 1, + "it_value": [ + 0, + 49406829 + ], + "it_interval": [ + 1, + 0 + ] + } +""" +import re +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/pid-fdinfo` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + root_int_list = {'pos', 'flags', 'mnt_id', 'ino', 'clockid', 'ticks', + 'settime flags', 'size', 'count'} + epoll_int_list = {'tfd', 'pos'} + inotify_int_list = {'wd'} + + for key, val in proc_data.items(): + if key in root_int_list: + proc_data[key] = int(val) + + if 'epoll' in proc_data: + for key, val in proc_data['epoll'].items(): + if key in epoll_int_list: + proc_data['epoll'][key] = int(val) + + if 'inotify' in proc_data: + for key, val in proc_data['inotify'].items(): + if key in inotify_int_list: + proc_data['inotify'][key] = int(val) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + split_me = {'it_value:', 'it_interval:'} + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + + # epoll files + if line.startswith('tfd:'): + line_match = re.findall(r'(?P\S+):(?:\s+)?(?P\S+s*)', line) + if line_match: + raw_output.update({'epoll': {k.strip(): v.strip() for k, v in line_match}}) + continue + + # inotify files + if line.startswith('inotify'): + split_line = line[8:].split() + raw_output['inotify'] = {} + for item in split_line: + k, v = item.split(':', maxsplit=1) + raw_output['inotify'][k] = v + continue + + # fanotify files + if line.startswith('fanotify'): + split_line = line[9:].split() + + if not 'fanotify' in raw_output: + raw_output['fanotify'] = {} + + for item in split_line: + k, v = item.split(':', maxsplit=1) + raw_output['fanotify'][k] = v + continue + + # timerfd files + if line.split()[0] in split_me: + split_line = line.replace(':', '').replace('(', '').replace(')', '').replace(',', '').split() + raw_output[split_line[0]] = [int(x) for x in split_line[1:]] + continue + + key, val = line.split(':', maxsplit=1) + raw_output[key.strip()] = val.strip() + continue + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 614b5b62..2c073c3a 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-09-17 1.21.2 "JSON Convert" +.TH jc 1 2022-09-19 1.21.2 "JSON Convert" .SH NAME \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, and strings .SH SYNOPSIS @@ -545,6 +545,11 @@ PLIST file parser \fB--proc-zoneinfo\fP `/proc/zoneinfo` file parser +.TP +.B +\fB--proc-pid-fdinfo\fP +`/proc/pid-fdinfo` file parser + .TP .B \fB--proc-pid-numa-maps\fP From 859bece92124c1a54047e5e6c5a205281d85e302 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 19 Sep 2022 17:49:18 -0700 Subject: [PATCH 63/71] add proc-pid-io parser --- docs/parsers/proc_pid_io.md | 74 +++++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_pid_io.py | 107 ++++++++++++++++++++++++++++++++++++ man/jc.1 | 5 ++ 4 files changed, 187 insertions(+) create mode 100644 docs/parsers/proc_pid_io.md create mode 100644 jc/parsers/proc_pid_io.py diff --git a/docs/parsers/proc_pid_io.md b/docs/parsers/proc_pid_io.md new file mode 100644 index 00000000..471420ee --- /dev/null +++ b/docs/parsers/proc_pid_io.md @@ -0,0 +1,74 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_pid\_io + +jc - JSON Convert `/proc//io` file parser + +Usage (cli): + + $ cat /proc//io | jc --proc + +or + + $ jc /proc//io + +or + + $ cat /proc//io | jc --proc-pid-io + +Usage (module): + + import jc + result = jc.parse('proc', proc_pid_io_file) + +or + + import jc + result = jc.parse('proc_pid_io', proc_pid_io_file) + +Schema: + +All values are integers. + + { + integer + } + +Examples: + + $ cat /proc/1/io | jc --proc -p + { + "rchar": 4699288382, + "wchar": 2931802997, + "syscr": 661897, + "syscw": 890910, + "read_bytes": 168468480, + "write_bytes": 27357184, + "cancelled_write_bytes": 16883712 + } + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + Dictionary. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index f2506197..c7e21bd2 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -113,6 +113,7 @@ parsers = [ 'proc-vmstat', 'proc-zoneinfo', 'proc-pid-fdinfo', + 'proc-pid-io', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_pid_io.py b/jc/parsers/proc_pid_io.py new file mode 100644 index 00000000..c34c4d8d --- /dev/null +++ b/jc/parsers/proc_pid_io.py @@ -0,0 +1,107 @@ +"""jc - JSON Convert `/proc//io` file parser + +Usage (cli): + + $ cat /proc//io | jc --proc + +or + + $ jc /proc//io + +or + + $ cat /proc//io | jc --proc-pid-io + +Usage (module): + + import jc + result = jc.parse('proc', proc_pid_io_file) + +or + + import jc + result = jc.parse('proc_pid_io', proc_pid_io_file) + +Schema: + +All values are integers. + + { + integer + } + +Examples: + + $ cat /proc/1/io | jc --proc -p + { + "rchar": 4699288382, + "wchar": 2931802997, + "syscr": 661897, + "syscw": 890910, + "read_bytes": 168468480, + "write_bytes": 27357184, + "cancelled_write_bytes": 16883712 + } +""" +from typing import Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc/pid-io` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: Dict) -> Dict: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (Dictionary) raw structured data to process + + Returns: + + Dictionary. Structured to conform to the schema. + """ + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> Dict: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + Dictionary. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: Dict = {} + + if jc.utils.has_data(data): + + for line in filter(None, data.splitlines()): + key, val = line.split(':', maxsplit=1) + raw_output[key] = int(val) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 2c073c3a..1dbffce2 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -550,6 +550,11 @@ PLIST file parser \fB--proc-pid-fdinfo\fP `/proc/pid-fdinfo` file parser +.TP +.B +\fB--proc-pid-io\fP +`/proc/pid-io` file parser + .TP .B \fB--proc-pid-numa-maps\fP From e49e6ad1793bf553699938b1f0b53db0c544ff58 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 19 Sep 2022 17:54:04 -0700 Subject: [PATCH 64/71] simplify parse code --- jc/parsers/proc_meminfo.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jc/parsers/proc_meminfo.py b/jc/parsers/proc_meminfo.py index 7b069f12..bf378374 100644 --- a/jc/parsers/proc_meminfo.py +++ b/jc/parsers/proc_meminfo.py @@ -145,9 +145,7 @@ def parse( if jc.utils.has_data(data): for line in filter(None, data.splitlines()): - split_line = line.split(':', maxsplit=1) - key = split_line[0] - val = int(split_line[1].rsplit(maxsplit=1)[0]) - raw_output[key] = val + key, val, *_ = line.replace(':', '').split() + raw_output[key] = int(val) return raw_output if raw else _process(raw_output) From 76e7347ecfebfda702e193b4de52130bacb1078d Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 19 Sep 2022 18:33:30 -0700 Subject: [PATCH 65/71] add proc-pid-maps parser --- docs/parsers/proc_pid_maps.md | 125 ++++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_pid_maps.py | 191 ++++++++++++++++++++++++++++++++++ man/jc.1 | 5 + 4 files changed, 322 insertions(+) create mode 100644 docs/parsers/proc_pid_maps.md create mode 100644 jc/parsers/proc_pid_maps.py diff --git a/docs/parsers/proc_pid_maps.md b/docs/parsers/proc_pid_maps.md new file mode 100644 index 00000000..7ca1557b --- /dev/null +++ b/docs/parsers/proc_pid_maps.md @@ -0,0 +1,125 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_pid\_maps + +jc - JSON Convert `/proc//maps` file parser + +Usage (cli): + + $ cat /proc/1/maps | jc --proc + +or + + $ jc /proc/1/maps + +or + + $ cat /proc/1/maps | jc --proc-pid-maps + +Usage (module): + + import jc + result = jc.parse('proc', proc_pid_maps_file) + +or + + import jc + result = jc.parse('proc_pid_maps', proc_pid_maps_file) + +Schema: + + [ + { + "start": string, + "end": string, + "perms": [ + string + ], + "offset": string, + "inode": integer, + "pathname": string, + "maj": string, + "min": string + } + ] + +Examples: + + $ cat /proc/1/maps | jc --proc -p + [ + { + "perms": [ + "read", + "private" + ], + "offset": "00000000", + "inode": 798126, + "pathname": "/usr/lib/systemd/systemd", + "start": "55a9e753c000", + "end": "55a9e7570000", + "maj": "fd", + "min": "00" + }, + { + "perms": [ + "read", + "execute", + "private" + ], + "offset": "00034000", + "inode": 798126, + "pathname": "/usr/lib/systemd/systemd", + "start": "55a9e7570000", + "end": "55a9e763a000", + "maj": "fd", + "min": "00" + }, + ... + ] + + $ cat /proc/1/maps | jc --proc-pid-maps -p -r + [ + { + "address": "55a9e753c000-55a9e7570000", + "perms": "r--p", + "offset": "00000000", + "dev": "fd:00", + "inode": "798126", + "pathname": "/usr/lib/systemd/systemd" + }, + { + "address": "55a9e7570000-55a9e763a000", + "perms": "r-xp", + "offset": "00034000", + "dev": "fd:00", + "inode": "798126", + "pathname": "/usr/lib/systemd/systemd" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index c7e21bd2..4fbb91a7 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -114,6 +114,7 @@ parsers = [ 'proc-zoneinfo', 'proc-pid-fdinfo', 'proc-pid-io', + 'proc-pid-maps', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_pid_maps.py b/jc/parsers/proc_pid_maps.py new file mode 100644 index 00000000..22c8765b --- /dev/null +++ b/jc/parsers/proc_pid_maps.py @@ -0,0 +1,191 @@ +"""jc - JSON Convert `/proc//maps` file parser + +Usage (cli): + + $ cat /proc/1/maps | jc --proc + +or + + $ jc /proc/1/maps + +or + + $ cat /proc/1/maps | jc --proc-pid-maps + +Usage (module): + + import jc + result = jc.parse('proc', proc_pid_maps_file) + +or + + import jc + result = jc.parse('proc_pid_maps', proc_pid_maps_file) + +Schema: + + [ + { + "start": string, + "end": string, + "perms": [ + string + ], + "offset": string, + "inode": integer, + "pathname": string, + "maj": string, + "min": string + } + ] + +Examples: + + $ cat /proc/1/maps | jc --proc -p + [ + { + "perms": [ + "read", + "private" + ], + "offset": "00000000", + "inode": 798126, + "pathname": "/usr/lib/systemd/systemd", + "start": "55a9e753c000", + "end": "55a9e7570000", + "maj": "fd", + "min": "00" + }, + { + "perms": [ + "read", + "execute", + "private" + ], + "offset": "00034000", + "inode": 798126, + "pathname": "/usr/lib/systemd/systemd", + "start": "55a9e7570000", + "end": "55a9e763a000", + "maj": "fd", + "min": "00" + }, + ... + ] + + $ cat /proc/1/maps | jc --proc-pid-maps -p -r + [ + { + "address": "55a9e753c000-55a9e7570000", + "perms": "r--p", + "offset": "00000000", + "dev": "fd:00", + "inode": "798126", + "pathname": "/usr/lib/systemd/systemd" + }, + { + "address": "55a9e7570000-55a9e763a000", + "perms": "r-xp", + "offset": "00034000", + "dev": "fd:00", + "inode": "798126", + "pathname": "/usr/lib/systemd/systemd" + }, + ... + ] +""" +from typing import List, Dict +import jc.utils +from jc.parsers.universal import simple_table_parse + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc//maps` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'inode'} + + perms_map = { + 'r': 'read', + 'w': 'write', + 'x': 'execute', + 's': 'shared', + 'p': 'private', + '-': None + } + + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = int(entry[key]) + + if 'address' in entry: + start, end = entry['address'].split('-') + entry['start'] = start + entry['end'] = end + del entry['address'] + + if 'perms' in entry: + perms_list = [perms_map[x] for x in entry['perms'] if perms_map[x]] + entry['perms'] = perms_list + + if 'dev' in entry: + maj, min = entry['dev'].split(':', maxsplit=1) + entry['maj'] = maj + entry['min'] = min + del entry['dev'] + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + header = 'address perms offset dev inode pathname\n' + data = header + data + raw_output = simple_table_parse(data.splitlines()) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 1dbffce2..641d3a6f 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -555,6 +555,11 @@ PLIST file parser \fB--proc-pid-io\fP `/proc/pid-io` file parser +.TP +.B +\fB--proc-pid-maps\fP +`/proc//maps` file parser + .TP .B \fB--proc-pid-numa-maps\fP From a583ecba7b1c46cf38564a493c3bcf531656b462 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 19 Sep 2022 19:26:08 -0700 Subject: [PATCH 66/71] add proc-pid-mountinfo parser (initial) --- docs/parsers/proc_pid_io.md | 6 +- docs/parsers/proc_pid_mountinfo.md | 132 ++++++++++++++++++++ jc/lib.py | 1 + jc/parsers/proc_pid_io.py | 8 +- jc/parsers/proc_pid_mountinfo.py | 190 +++++++++++++++++++++++++++++ man/jc.1 | 7 +- 6 files changed, 336 insertions(+), 8 deletions(-) create mode 100644 docs/parsers/proc_pid_mountinfo.md create mode 100644 jc/parsers/proc_pid_mountinfo.py diff --git a/docs/parsers/proc_pid_io.md b/docs/parsers/proc_pid_io.md index 471420ee..63884858 100644 --- a/docs/parsers/proc_pid_io.md +++ b/docs/parsers/proc_pid_io.md @@ -7,15 +7,15 @@ jc - JSON Convert `/proc//io` file parser Usage (cli): - $ cat /proc//io | jc --proc + $ cat /proc/1/io | jc --proc or - $ jc /proc//io + $ jc /proc/1/io or - $ cat /proc//io | jc --proc-pid-io + $ cat /proc/1/io | jc --proc-pid-io Usage (module): diff --git a/docs/parsers/proc_pid_mountinfo.md b/docs/parsers/proc_pid_mountinfo.md new file mode 100644 index 00000000..58ecc4d3 --- /dev/null +++ b/docs/parsers/proc_pid_mountinfo.md @@ -0,0 +1,132 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + + +# jc.parsers.proc\_pid\_mountinfo + +jc - JSON Convert `/proc//mountinfo` file parser + +Usage (cli): + + $ cat /proc/1/mountinfo | jc --proc + +or + + $ jc /proc/1/mountinfo + +or + + $ cat /proc/1/mountinfo | jc --proc-pid-mountinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_pid_mountinfo_file) + +or + + import jc + result = jc.parse('proc_pid_mountinfo', proc_pid_mountinfo_file) + +Schema: + + [ + { + "module": string, + "size": integer, + "used": integer, + "used_by": [ + string + ], + "status": string, + "location": string + } + ] + +Examples: + + $ cat /proc/1/mountinfo | jc --proc -p + [ + { + "module": "binfmt_misc", + "size": 24576, + "used": 1, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": 16384, + "used": 0, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": 36864, + "used": 1, + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] + + $ cat /proc/1/mountinfo | jc --proc_pid-mountinfo -p -r + [ + { + "module": "binfmt_misc", + "size": "24576", + "used": "1", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": "16384", + "used": "0", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": "36864", + "used": "1", + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] + + + +### parse + +```python +def parse(data: str, raw: bool = False, quiet: bool = False) -> List[Dict] +``` + +Main text parsing function + +Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +### Parser Information +Compatibility: linux + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/jc/lib.py b/jc/lib.py index 4fbb91a7..da8bf7a9 100644 --- a/jc/lib.py +++ b/jc/lib.py @@ -115,6 +115,7 @@ parsers = [ 'proc-pid-fdinfo', 'proc-pid-io', 'proc-pid-maps', + 'proc-pid-mountinfo', 'proc-pid-numa-maps', 'ps', 'route', diff --git a/jc/parsers/proc_pid_io.py b/jc/parsers/proc_pid_io.py index c34c4d8d..8bfe71c1 100644 --- a/jc/parsers/proc_pid_io.py +++ b/jc/parsers/proc_pid_io.py @@ -2,15 +2,15 @@ Usage (cli): - $ cat /proc//io | jc --proc + $ cat /proc/1/io | jc --proc or - $ jc /proc//io + $ jc /proc/1/io or - $ cat /proc//io | jc --proc-pid-io + $ cat /proc/1/io | jc --proc-pid-io Usage (module): @@ -50,7 +50,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" version = '1.0' - description = '`/proc/pid-io` file parser' + description = '`/proc//io` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux'] diff --git a/jc/parsers/proc_pid_mountinfo.py b/jc/parsers/proc_pid_mountinfo.py new file mode 100644 index 00000000..f64018f7 --- /dev/null +++ b/jc/parsers/proc_pid_mountinfo.py @@ -0,0 +1,190 @@ +"""jc - JSON Convert `/proc//mountinfo` file parser + +Usage (cli): + + $ cat /proc/1/mountinfo | jc --proc + +or + + $ jc /proc/1/mountinfo + +or + + $ cat /proc/1/mountinfo | jc --proc-pid-mountinfo + +Usage (module): + + import jc + result = jc.parse('proc', proc_pid_mountinfo_file) + +or + + import jc + result = jc.parse('proc_pid_mountinfo', proc_pid_mountinfo_file) + +Schema: + + [ + { + "module": string, + "size": integer, + "used": integer, + "used_by": [ + string + ], + "status": string, + "location": string + } + ] + +Examples: + + $ cat /proc/1/mountinfo | jc --proc -p + [ + { + "module": "binfmt_misc", + "size": 24576, + "used": 1, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": 16384, + "used": 0, + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": 36864, + "used": 1, + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] + + $ cat /proc/1/mountinfo | jc --proc_pid-mountinfo -p -r + [ + { + "module": "binfmt_misc", + "size": "24576", + "used": "1", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0ab4000" + }, + { + "module": "vsock_loopback", + "size": "16384", + "used": "0", + "used_by": [], + "status": "Live", + "location": "0xffffffffc0a14000" + }, + { + "module": "vmw_vsock_virtio_transport_common", + "size": "36864", + "used": "1", + "used_by": [ + "vsock_loopback" + ], + "status": "Live", + "location": "0xffffffffc0a03000" + }, + ... + ] +""" +import re +from typing import List, Dict +import jc.utils + + +class info(): + """Provides parser metadata (version, author, etc.)""" + version = '1.0' + description = '`/proc//mountinfo` file parser' + author = 'Kelly Brazil' + author_email = 'kellyjonbrazil@gmail.com' + compatible = ['linux'] + hidden = True + + +__version__ = info.version + + +def _process(proc_data: List[Dict]) -> List[Dict]: + """ + Final processing to conform to the schema. + + Parameters: + + proc_data: (List of Dictionaries) raw structured data to process + + Returns: + + List of Dictionaries. Structured to conform to the schema. + """ + int_list = {'size', 'used'} + + for entry in proc_data: + for key in entry: + if key in int_list: + entry[key] = jc.utils.convert_to_int(entry[key]) + + return proc_data + + +def parse( + data: str, + raw: bool = False, + quiet: bool = False +) -> List[Dict]: + """ + Main text parsing function + + Parameters: + + data: (string) text data to parse + raw: (boolean) unprocessed output if True + quiet: (boolean) suppress warning messages if True + + Returns: + + List of Dictionaries. Raw or processed structured data. + """ + jc.utils.compatibility(__name__, info.compatible, quiet) + jc.utils.input_type_check(data) + + raw_output: List = [] + + if jc.utils.has_data(data): + + line_pattern = re.compile(r''' + ^(?P\d+)\s + (?P\d+)\s + (?P\d+): + (?P\d+)\s + (?P\S+)\s + (?P\S+)\s + (?P\S+)\s? + (?P(?:\s?\S+:\S+\s?)*)\s?-\s + (?P\S+)\s + (?P\S+)\s + (?P\S+) + ''', re.VERBOSE + ) + + for line in filter(None, data.splitlines()): + + line_match = line_pattern.search(line) + if line_match: + raw_output.append(line_match.groupdict()) + + return raw_output if raw else _process(raw_output) diff --git a/man/jc.1 b/man/jc.1 index 641d3a6f..4a72b238 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -553,13 +553,18 @@ PLIST file parser .TP .B \fB--proc-pid-io\fP -`/proc/pid-io` file parser +`/proc//io` file parser .TP .B \fB--proc-pid-maps\fP `/proc//maps` file parser +.TP +.B +\fB--proc-pid-mountinfo\fP +`/proc//mountinfo` file parser + .TP .B \fB--proc-pid-numa-maps\fP From 7c772d3a5a91a3ba99c9aee7c9a8ef275b5b9a5a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 19 Sep 2022 19:58:40 -0700 Subject: [PATCH 67/71] split super_options --- jc/parsers/proc_pid_mountinfo.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/jc/parsers/proc_pid_mountinfo.py b/jc/parsers/proc_pid_mountinfo.py index f64018f7..0e11ee94 100644 --- a/jc/parsers/proc_pid_mountinfo.py +++ b/jc/parsers/proc_pid_mountinfo.py @@ -138,6 +138,31 @@ def _process(proc_data: List[Dict]) -> List[Dict]: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) + if 'mount_options' in entry: + entry['mount_options'] = entry['mount_options'].split(',') + + if 'optional_fields' in entry: + entry['optional_fields'] = {x.split(':')[0]: x.split(':')[1] for x in entry['optional_fields'].split()} + + if 'super_options' in entry: + if entry['super_options']: + super_options_split = entry['super_options'].split(',') + s_options = [x for x in super_options_split if '=' not in x] + s_options_fields = [x for x in super_options_split if '=' in x] + + if s_options: + entry['super_options'] = s_options + else: + del entry['super_options'] + + if s_options_fields: + if not 'super_options_fields' in entry: + entry['super_options_fields'] = {} + + for field in s_options_fields: + key, val = field.split('=') + entry['super_options_fields'][key] = val + return proc_data @@ -177,7 +202,7 @@ def parse( (?P(?:\s?\S+:\S+\s?)*)\s?-\s (?P\S+)\s (?P\S+)\s - (?P\S+) + (?P\S+)? ''', re.VERBOSE ) From 52d98a1157ba99392ee059addc8ce6997923c4de Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 19 Sep 2022 20:32:26 -0700 Subject: [PATCH 68/71] doc update and schema update --- docs/parsers/proc_pid_mountinfo.md | 137 +++++++++++++++++---------- jc/parsers/proc_pid_mountinfo.py | 144 ++++++++++++++++++----------- 2 files changed, 177 insertions(+), 104 deletions(-) diff --git a/docs/parsers/proc_pid_mountinfo.md b/docs/parsers/proc_pid_mountinfo.md index 58ecc4d3..33463209 100644 --- a/docs/parsers/proc_pid_mountinfo.md +++ b/docs/parsers/proc_pid_mountinfo.md @@ -31,46 +31,81 @@ Schema: [ { - "module": string, - "size": integer, - "used": integer, - "used_by": [ + "mount_id": integer, + "parent_id": integer, + "maj": integer, + "min": integer, + "root": string, + "mount_point": string, + "mount_options": [ string ], - "status": string, - "location": string + "optional_fields": { # [0] + "": integer + }, + "fs_type": string, + "mount_source": string, + "super_options": [ + string + ], + "super_options_fields": { + "": string + } } ] + [0] if empty, then unbindable + Examples: $ cat /proc/1/mountinfo | jc --proc -p [ { - "module": "binfmt_misc", - "size": 24576, - "used": 1, - "used_by": [], - "status": "Live", - "location": "0xffffffffc0ab4000" - }, - { - "module": "vsock_loopback", - "size": 16384, - "used": 0, - "used_by": [], - "status": "Live", - "location": "0xffffffffc0a14000" - }, - { - "module": "vmw_vsock_virtio_transport_common", - "size": 36864, - "used": 1, - "used_by": [ - "vsock_loopback" + "mount_id": 24, + "parent_id": 30, + "maj": 0, + "min": 22, + "root": "/", + "mount_point": "/sys", + "mount_options": [ + "rw", + "nosuid", + "nodev", + "noexec", + "relatime" ], - "status": "Live", - "location": "0xffffffffc0a03000" + "optional_fields": { + "master": 1, + "shared": 7 + }, + "fs_type": "sysfs", + "mount_source": "sysfs", + "super_options": [ + "rw" + ] + }, + { + "mount_id": 25, + "parent_id": 30, + "maj": 0, + "min": 23, + "root": "/", + "mount_point": "/proc", + "mount_options": [ + "rw", + "nosuid", + "nodev", + "noexec", + "relatime" + ], + "optional_fields": { + "shared": 14 + }, + "fs_type": "proc", + "mount_source": "proc", + "super_options": [ + "rw" + ] }, ... ] @@ -78,30 +113,30 @@ Examples: $ cat /proc/1/mountinfo | jc --proc_pid-mountinfo -p -r [ { - "module": "binfmt_misc", - "size": "24576", - "used": "1", - "used_by": [], - "status": "Live", - "location": "0xffffffffc0ab4000" + "mount_id": "24", + "parent_id": "30", + "maj": "0", + "min": "22", + "root": "/", + "mount_point": "/sys", + "mount_options": "rw,nosuid,nodev,noexec,relatime", + "optional_fields": "master:1 shared:7 ", + "fs_type": "sysfs", + "mount_source": "sysfs", + "super_options": "rw" }, { - "module": "vsock_loopback", - "size": "16384", - "used": "0", - "used_by": [], - "status": "Live", - "location": "0xffffffffc0a14000" - }, - { - "module": "vmw_vsock_virtio_transport_common", - "size": "36864", - "used": "1", - "used_by": [ - "vsock_loopback" - ], - "status": "Live", - "location": "0xffffffffc0a03000" + "mount_id": "25", + "parent_id": "30", + "maj": "0", + "min": "23", + "root": "/", + "mount_point": "/proc", + "mount_options": "rw,nosuid,nodev,noexec,relatime", + "optional_fields": "shared:14 ", + "fs_type": "proc", + "mount_source": "proc", + "super_options": "rw" }, ... ] diff --git a/jc/parsers/proc_pid_mountinfo.py b/jc/parsers/proc_pid_mountinfo.py index 0e11ee94..55329b7a 100644 --- a/jc/parsers/proc_pid_mountinfo.py +++ b/jc/parsers/proc_pid_mountinfo.py @@ -26,46 +26,81 @@ Schema: [ { - "module": string, - "size": integer, - "used": integer, - "used_by": [ + "mount_id": integer, + "parent_id": integer, + "maj": integer, + "min": integer, + "root": string, + "mount_point": string, + "mount_options": [ string ], - "status": string, - "location": string + "optional_fields": { # [0] + "": integer + }, + "fs_type": string, + "mount_source": string, + "super_options": [ + string + ], + "super_options_fields": { + "": string + } } ] + [0] if empty, then unbindable + Examples: $ cat /proc/1/mountinfo | jc --proc -p [ { - "module": "binfmt_misc", - "size": 24576, - "used": 1, - "used_by": [], - "status": "Live", - "location": "0xffffffffc0ab4000" - }, - { - "module": "vsock_loopback", - "size": 16384, - "used": 0, - "used_by": [], - "status": "Live", - "location": "0xffffffffc0a14000" - }, - { - "module": "vmw_vsock_virtio_transport_common", - "size": 36864, - "used": 1, - "used_by": [ - "vsock_loopback" + "mount_id": 24, + "parent_id": 30, + "maj": 0, + "min": 22, + "root": "/", + "mount_point": "/sys", + "mount_options": [ + "rw", + "nosuid", + "nodev", + "noexec", + "relatime" ], - "status": "Live", - "location": "0xffffffffc0a03000" + "optional_fields": { + "master": 1, + "shared": 7 + }, + "fs_type": "sysfs", + "mount_source": "sysfs", + "super_options": [ + "rw" + ] + }, + { + "mount_id": 25, + "parent_id": 30, + "maj": 0, + "min": 23, + "root": "/", + "mount_point": "/proc", + "mount_options": [ + "rw", + "nosuid", + "nodev", + "noexec", + "relatime" + ], + "optional_fields": { + "shared": 14 + }, + "fs_type": "proc", + "mount_source": "proc", + "super_options": [ + "rw" + ] }, ... ] @@ -73,30 +108,30 @@ Examples: $ cat /proc/1/mountinfo | jc --proc_pid-mountinfo -p -r [ { - "module": "binfmt_misc", - "size": "24576", - "used": "1", - "used_by": [], - "status": "Live", - "location": "0xffffffffc0ab4000" + "mount_id": "24", + "parent_id": "30", + "maj": "0", + "min": "22", + "root": "/", + "mount_point": "/sys", + "mount_options": "rw,nosuid,nodev,noexec,relatime", + "optional_fields": "master:1 shared:7 ", + "fs_type": "sysfs", + "mount_source": "sysfs", + "super_options": "rw" }, { - "module": "vsock_loopback", - "size": "16384", - "used": "0", - "used_by": [], - "status": "Live", - "location": "0xffffffffc0a14000" - }, - { - "module": "vmw_vsock_virtio_transport_common", - "size": "36864", - "used": "1", - "used_by": [ - "vsock_loopback" - ], - "status": "Live", - "location": "0xffffffffc0a03000" + "mount_id": "25", + "parent_id": "30", + "maj": "0", + "min": "23", + "root": "/", + "mount_point": "/proc", + "mount_options": "rw,nosuid,nodev,noexec,relatime", + "optional_fields": "shared:14 ", + "fs_type": "proc", + "mount_source": "proc", + "super_options": "rw" }, ... ] @@ -131,7 +166,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = {'size', 'used'} + int_list = {'mount_id', 'parent_id', 'maj', 'min'} for entry in proc_data: for key in entry: @@ -142,7 +177,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]: entry['mount_options'] = entry['mount_options'].split(',') if 'optional_fields' in entry: - entry['optional_fields'] = {x.split(':')[0]: x.split(':')[1] for x in entry['optional_fields'].split()} + entry['optional_fields'] = {x.split(':')[0]: int(x.split(':')[1]) for x in entry['optional_fields'].split()} if 'super_options' in entry: if entry['super_options']: @@ -163,6 +198,9 @@ def _process(proc_data: List[Dict]) -> List[Dict]: key, val = field.split('=') entry['super_options_fields'][key] = val + else: + del entry['super_options'] + return proc_data From 3df006fb9760922226a865226c387a4068b23bf3 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 19 Sep 2022 20:35:18 -0700 Subject: [PATCH 69/71] use stdlib int conversion --- jc/parsers/proc_pid_mountinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jc/parsers/proc_pid_mountinfo.py b/jc/parsers/proc_pid_mountinfo.py index 55329b7a..8f197a2d 100644 --- a/jc/parsers/proc_pid_mountinfo.py +++ b/jc/parsers/proc_pid_mountinfo.py @@ -171,7 +171,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]: for entry in proc_data: for key in entry: if key in int_list: - entry[key] = jc.utils.convert_to_int(entry[key]) + entry[key] = int(entry[key]) if 'mount_options' in entry: entry['mount_options'] = entry['mount_options'].split(',') From df3f94b017b4c3b0d37724855171334857be543b Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 19 Sep 2022 20:54:46 -0700 Subject: [PATCH 70/71] doc update and unbindable fix --- docs/parsers/proc_pid_mountinfo.md | 5 +++-- jc/parsers/proc_pid_mountinfo.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/parsers/proc_pid_mountinfo.md b/docs/parsers/proc_pid_mountinfo.md index 33463209..9e0a4e2e 100644 --- a/docs/parsers/proc_pid_mountinfo.md +++ b/docs/parsers/proc_pid_mountinfo.md @@ -41,7 +41,7 @@ Schema: string ], "optional_fields": { # [0] - "": integer + "": integer # [1] }, "fs_type": string, "mount_source": string, @@ -54,7 +54,8 @@ Schema: } ] - [0] if empty, then unbindable + [0] if empty, then private mount + [1] unbindable will always have a value of 0 Examples: diff --git a/jc/parsers/proc_pid_mountinfo.py b/jc/parsers/proc_pid_mountinfo.py index 8f197a2d..72399911 100644 --- a/jc/parsers/proc_pid_mountinfo.py +++ b/jc/parsers/proc_pid_mountinfo.py @@ -36,7 +36,7 @@ Schema: string ], "optional_fields": { # [0] - "": integer + "": integer # [1] }, "fs_type": string, "mount_source": string, @@ -49,7 +49,8 @@ Schema: } ] - [0] if empty, then unbindable + [0] if empty, then private mount + [1] unbindable will always have a value of 0 Examples: @@ -177,7 +178,10 @@ def _process(proc_data: List[Dict]) -> List[Dict]: entry['mount_options'] = entry['mount_options'].split(',') if 'optional_fields' in entry: - entry['optional_fields'] = {x.split(':')[0]: int(x.split(':')[1]) for x in entry['optional_fields'].split()} + if 'unbindable' in entry['optional_fields']: + entry['optional_fields'] = {'unbindable': 0} + else: + entry['optional_fields'] = {x.split(':')[0]: int(x.split(':')[1]) for x in entry['optional_fields'].split()} if 'super_options' in entry: if entry['super_options']: @@ -237,7 +241,8 @@ def parse( (?P\S+)\s (?P\S+)\s (?P\S+)\s? - (?P(?:\s?\S+:\S+\s?)*)\s?-\s + # (?P(?:\s?\S+:\S+\s?)*)\s?-\s + (?P(?:\s?(?:\S+:\S+|unbindable)\s?)*)\s?-\s (?P\S+)\s (?P\S+)\s (?P\S+)? From 075e2301e4aa3efffa3083901e061b772ba2e36e Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 19 Sep 2022 21:00:00 -0700 Subject: [PATCH 71/71] doc update --- docs/parsers/proc_pid_fdinfo.md | 2 +- jc/parsers/proc_pid_fdinfo.py | 4 ++-- man/jc.1 | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/parsers/proc_pid_fdinfo.md b/docs/parsers/proc_pid_fdinfo.md index 4f266668..2d946fbc 100644 --- a/docs/parsers/proc_pid_fdinfo.md +++ b/docs/parsers/proc_pid_fdinfo.md @@ -3,7 +3,7 @@ # jc.parsers.proc\_pid\_fdinfo -jc - JSON Convert `/proc//fdinfo` file parser +jc - JSON Convert `/proc//fdinfo/` file parser Usage (cli): diff --git a/jc/parsers/proc_pid_fdinfo.py b/jc/parsers/proc_pid_fdinfo.py index b752c30f..e8e99460 100644 --- a/jc/parsers/proc_pid_fdinfo.py +++ b/jc/parsers/proc_pid_fdinfo.py @@ -1,4 +1,4 @@ -"""jc - JSON Convert `/proc//fdinfo` file parser +"""jc - JSON Convert `/proc//fdinfo/` file parser Usage (cli): @@ -104,7 +104,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" version = '1.0' - description = '`/proc/pid-fdinfo` file parser' + description = '`/proc//fdinfo/` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' compatible = ['linux'] diff --git a/man/jc.1 b/man/jc.1 index 4a72b238..b177429e 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -548,7 +548,7 @@ PLIST file parser .TP .B \fB--proc-pid-fdinfo\fP -`/proc/pid-fdinfo` file parser +`/proc//fdinfo/` file parser .TP .B