From 33b996f6be90ce236d4e009690ef4ded5be8f40a Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Sat, 16 Jul 2022 20:51:18 -0700 Subject: [PATCH] process optimizations --- jc/parsers/m3u.py | 3 --- jc/parsers/ntpq.py | 7 ++++--- jc/parsers/passwd.py | 3 ++- jc/parsers/pidstat.py | 11 +++++++---- jc/parsers/pidstat_s.py | 10 ++++++---- jc/parsers/ping.py | 12 +++++++----- jc/parsers/ping_s.py | 11 ++++++----- jc/parsers/ps.py | 8 +++++--- jc/parsers/route.py | 29 +++++++++++++++-------------- jc/parsers/rpm_qi.py | 6 +++--- jc/parsers/rsync.py | 10 ++++++---- jc/parsers/rsync_s.py | 10 ++++++---- jc/parsers/sfdisk.py | 10 ++++++---- jc/parsers/shadow.py | 5 +++-- jc/parsers/ss.py | 5 +++-- jc/parsers/stat.py | 18 +++++++++--------- jc/parsers/stat_s.py | 17 +++++++++-------- jc/parsers/systemctl_lj.py | 5 +++-- jc/parsers/systeminfo.py | 36 ++++++++++++++++++++---------------- jc/parsers/time.py | 25 ++++++++++++++----------- jc/parsers/timedatectl.py | 7 ++++--- jc/parsers/top.py | 12 ++++++------ jc/parsers/top_s.py | 12 ++++++------ jc/parsers/tracepath.py | 8 +++++--- jc/parsers/traceroute.py | 6 +++--- jc/parsers/ufw.py | 4 ++-- jc/parsers/ufw_appinfo.py | 6 +++--- jc/parsers/update_alt_q.py | 4 ++-- jc/parsers/uptime.py | 8 +++++--- jc/parsers/vmstat.py | 10 +++++----- jc/parsers/vmstat_s.py | 6 +++--- jc/parsers/w.py | 5 +++-- jc/parsers/wc.py | 4 ++-- jc/parsers/who.py | 5 +++-- jc/parsers/zipinfo.py | 10 ++++++---- 35 files changed, 192 insertions(+), 156 deletions(-) diff --git a/jc/parsers/m3u.py b/jc/parsers/m3u.py index 3405a0db..722826bf 100644 --- a/jc/parsers/m3u.py +++ b/jc/parsers/m3u.py @@ -5,9 +5,6 @@ the extended fields cannot be successfully parsed, then an `unparsed_info` field will be added to the object. If not using `--quiet`, then a warning message also will be printed to `STDERR`. -Parsing issues with extended field information will usually occur with lines -that include punctuation like single quotes. - Usage (cli): $ cat playlist.m3u | jc --m3u diff --git a/jc/parsers/ntpq.py b/jc/parsers/ntpq.py index 1c24731f..e4866243 100644 --- a/jc/parsers/ntpq.py +++ b/jc/parsers/ntpq.py @@ -207,7 +207,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.6' + version = '1.7' description = '`ntpq -p` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -230,6 +230,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'st', 'when', 'poll', 'reach'} + float_list = {'delay', 'offset', 'jitter'} + for entry in proc_data: if entry['s'] == '~': @@ -237,8 +240,6 @@ def _process(proc_data): entry['state'] = entry.pop('s') - int_list = ['st', 'when', 'poll', 'reach'] - float_list = ['delay', 'offset', 'jitter'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/passwd.py b/jc/parsers/passwd.py index 5a5e20a4..6c73f6a8 100644 --- a/jc/parsers/passwd.py +++ b/jc/parsers/passwd.py @@ -116,8 +116,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'uid', 'gid'} + for entry in proc_data: - int_list = ['uid', 'gid'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/pidstat.py b/jc/parsers/pidstat.py index c2cd3806..42e391cd 100644 --- a/jc/parsers/pidstat.py +++ b/jc/parsers/pidstat.py @@ -128,7 +128,7 @@ from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`pidstat -h` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -151,14 +151,17 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = ['time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'] - float_list = ['percent_usr', 'percent_system', 'percent_guest', 'percent_cpu', + int_list = {'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'} + + float_list = {'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu', 'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s', - 'kb_ccwr_s', 'cswch_s', 'nvcswch_s'] + 'kb_ccwr_s', 'cswch_s', 'nvcswch_s'} + for entry in proc_data: for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) + if key in float_list: entry[key] = jc.utils.convert_to_float(entry[key]) diff --git a/jc/parsers/pidstat_s.py b/jc/parsers/pidstat_s.py index ecaa46bb..9242d97a 100644 --- a/jc/parsers/pidstat_s.py +++ b/jc/parsers/pidstat_s.py @@ -83,7 +83,7 @@ from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`pidstat -h` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -106,14 +106,16 @@ def _process(proc_data: Dict) -> Dict: Dictionary. Structured data to conform to the schema. """ - int_list = ['time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'] - float_list = ['percent_usr', 'percent_system', 'percent_guest', 'percent_cpu', + int_list = {'time', 'uid', 'pid', 'cpu', 'vsz', 'rss', 'stksize', 'stkref'} + + float_list = {'percent_usr', 'percent_system', 'percent_guest', 'percent_cpu', 'minflt_s', 'majflt_s', 'percent_mem', 'kb_rd_s', 'kb_wr_s', - 'kb_ccwr_s', 'cswch_s', 'nvcswch_s'] + 'kb_ccwr_s', 'cswch_s', 'nvcswch_s'} for key in proc_data: if key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) + if key in float_list: proc_data[key] = jc.utils.convert_to_float(proc_data[key]) diff --git a/jc/parsers/ping.py b/jc/parsers/ping.py index 0f851c04..4d789728 100644 --- a/jc/parsers/ping.py +++ b/jc/parsers/ping.py @@ -164,7 +164,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.7' + version = '1.8' description = '`ping` and `ping6` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -187,14 +187,15 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - int_list = [ + int_list = { 'data_bytes', 'packets_transmitted', 'packets_received', 'bytes', 'icmp_seq', 'ttl', 'duplicates', 'vr', 'hl', 'tos', 'len', 'id', 'flg', 'off', 'pro', 'cks' - ] - float_list = [ + } + + float_list = { 'packet_loss_percent', 'round_trip_ms_min', 'round_trip_ms_avg', 'round_trip_ms_max', 'round_trip_ms_stddev', 'timestamp', 'time_ms' - ] + } for key in proc_data: if key in int_list: @@ -208,6 +209,7 @@ def _process(proc_data): for k in entry: if k in int_list: entry[k] = jc.utils.convert_to_int(entry[k]) + if k in float_list: entry[k] = jc.utils.convert_to_float(entry[k]) diff --git a/jc/parsers/ping_s.py b/jc/parsers/ping_s.py index 8cb9beb0..f9ca0507 100644 --- a/jc/parsers/ping_s.py +++ b/jc/parsers/ping_s.py @@ -85,7 +85,7 @@ from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`ping` and `ping6` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -108,14 +108,15 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - int_list = [ + int_list = { 'sent_bytes', 'packets_transmitted', 'packets_received', 'response_bytes', 'icmp_seq', 'ttl', 'duplicates', 'vr', 'hl', 'tos', 'len', 'id', 'flg', 'off', 'pro', 'cks' - ] - float_list = [ + } + + float_list = { 'packet_loss_percent', 'round_trip_ms_min', 'round_trip_ms_avg', 'round_trip_ms_max', 'round_trip_ms_stddev', 'timestamp', 'time_ms' - ] + } for key in proc_data: if key in int_list: diff --git a/jc/parsers/ps.py b/jc/parsers/ps.py index 3b529442..61720032 100644 --- a/jc/parsers/ps.py +++ b/jc/parsers/ps.py @@ -207,7 +207,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.6' + version = '1.7' description = '`ps` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -230,6 +230,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'pid', 'ppid', 'c', 'vsz', 'rss'} + float_list = {'cpu_percent', 'mem_percent'} + for entry in proc_data: # change key name '%cpu' to 'cpu_percent' if '%cpu' in entry: @@ -240,11 +243,10 @@ def _process(proc_data): entry['mem_percent'] = entry.pop('%mem') # convert ints and floats - int_list = ['pid', 'ppid', 'c', 'vsz', 'rss'] - float_list = ['cpu_percent', 'mem_percent'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) + if key in float_list: entry[key] = jc.utils.convert_to_float(entry[key]) diff --git a/jc/parsers/route.py b/jc/parsers/route.py index 4814d628..2e530c51 100644 --- a/jc/parsers/route.py +++ b/jc/parsers/route.py @@ -109,7 +109,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.7' + version = '1.8' description = '`route` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -132,8 +132,21 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'metric', 'ref', 'use', 'mss', 'window', 'irtt'} + + flag_map = { + 'U': 'UP', + 'H': 'HOST', + 'G': 'GATEWAY', + 'R': 'REINSTATE', + 'D': 'DYNAMIC', + 'M': 'MODIFIED', + 'A': 'ADDRCONF', + 'C': 'CACHE', + '!': 'REJECT' + } + for entry in proc_data: - int_list = ['metric', 'ref', 'use', 'mss', 'window', 'irtt'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) @@ -141,18 +154,6 @@ def _process(proc_data): # add flags_pretty # Flag mapping from https://www.man7.org/linux/man-pages/man8/route.8.html if 'flags' in entry: - flag_map = { - 'U': 'UP', - 'H': 'HOST', - 'G': 'GATEWAY', - 'R': 'REINSTATE', - 'D': 'DYNAMIC', - 'M': 'MODIFIED', - 'A': 'ADDRCONF', - 'C': 'CACHE', - '!': 'REJECT' - } - pretty_flags = [] for flag in entry['flags']: diff --git a/jc/parsers/rpm_qi.py b/jc/parsers/rpm_qi.py index 612fa31a..f0061a0d 100644 --- a/jc/parsers/rpm_qi.py +++ b/jc/parsers/rpm_qi.py @@ -161,7 +161,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`rpm -qi` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -184,9 +184,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ - for entry in proc_data: + int_list = {'epoch', 'size'} - int_list = ['epoch', 'size'] + for entry in proc_data: for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/rsync.py b/jc/parsers/rsync.py index f80bd451..c18e487c 100644 --- a/jc/parsers/rsync.py +++ b/jc/parsers/rsync.py @@ -137,7 +137,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`rsync` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -160,16 +160,18 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = [ + int_list = { 'process', 'sent', 'received', 'total_size', 'matches', 'hash_hits', 'false_alarms', 'data' - ] - float_list = ['bytes_sec', 'speedup'] + } + + float_list = {'bytes_sec', 'speedup'} for item in proc_data: for key in item['summary']: if key in int_list: item['summary'][key] = jc.utils.convert_to_int(item['summary'][key]) + if key in float_list: item['summary'][key] = jc.utils.convert_to_float(item['summary'][key]) diff --git a/jc/parsers/rsync_s.py b/jc/parsers/rsync_s.py index f8c4ce10..a8ad399b 100644 --- a/jc/parsers/rsync_s.py +++ b/jc/parsers/rsync_s.py @@ -88,7 +88,7 @@ from jc.streaming import ( class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`rsync` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -111,15 +111,17 @@ def _process(proc_data: Dict) -> Dict: Dictionary. Structured data to conform to the schema. """ - int_list = [ + int_list = { 'process', 'sent', 'received', 'total_size', 'matches', 'hash_hits', 'false_alarms', 'data' - ] - float_list = ['bytes_sec', 'speedup'] + } + + float_list = {'bytes_sec', 'speedup'} for key in proc_data.copy(): if key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) + if key in float_list: proc_data[key] = jc.utils.convert_to_float(proc_data[key]) diff --git a/jc/parsers/sfdisk.py b/jc/parsers/sfdisk.py index 9553de51..93d261cb 100644 --- a/jc/parsers/sfdisk.py +++ b/jc/parsers/sfdisk.py @@ -203,7 +203,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.2' + version = '1.3' description = '`sfdisk` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -226,12 +226,13 @@ def _process(proc_data): List of Dictionaries. Structured to conform to the schema. """ - int_list = [ + int_list = { 'cylinders', 'heads', 'sectors_per_track', 'start', 'end', 'cyls', 'mib', 'blocks', 'sectors', 'bytes', 'logical_sector_size', 'physical_sector_size', 'min_io_size', 'optimal_io_size', 'free_bytes', 'free_sectors' - ] - bool_list = ['boot'] + } + + bool_list = {'boot'} for entry in proc_data: for key in entry: @@ -249,6 +250,7 @@ def _process(proc_data): # normal conversions if key in int_list: p[key] = jc.utils.convert_to_int(p[key].replace('-', '')) + if key in bool_list: p[key] = jc.utils.convert_to_bool(p[key]) diff --git a/jc/parsers/shadow.py b/jc/parsers/shadow.py index 184c49bc..a57847ff 100644 --- a/jc/parsers/shadow.py +++ b/jc/parsers/shadow.py @@ -101,7 +101,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.4' + version = '1.5' description = '`/etc/shadow` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -123,8 +123,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'last_changed', 'minimum', 'maximum', 'warn', 'inactive', 'expire'} + for entry in proc_data: - int_list = ['last_changed', 'minimum', 'maximum', 'warn', 'inactive', 'expire'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/ss.py b/jc/parsers/ss.py index 4c4b3b45..9e53dd7d 100644 --- a/jc/parsers/ss.py +++ b/jc/parsers/ss.py @@ -281,7 +281,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`ss` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -304,8 +304,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'recv_q', 'send_q', 'pid'} + for entry in proc_data: - int_list = ['recv_q', 'send_q', 'pid'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/stat.py b/jc/parsers/stat.py index cf20f504..e49fc047 100644 --- a/jc/parsers/stat.py +++ b/jc/parsers/stat.py @@ -171,7 +171,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.11' + version = '1.12' description = '`stat` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -195,17 +195,17 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ for entry in proc_data: - int_list = ['size', 'blocks', 'io_blocks', 'inode', 'links', 'uid', 'gid', - 'unix_device', 'rdev', 'block_size'] - for key in entry: + int_list = {'size', 'blocks', 'io_blocks', 'inode', 'links', 'uid', 'gid', + 'unix_device', 'rdev', 'block_size'} + + null_list = {'access_time', 'modify_time', 'change_time', 'birth_time'} + + for key in entry.copy(): if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) - # turn - into null for time fields and add calculated timestamp fields - for entry in proc_data: - null_list = ['access_time', 'modify_time', 'change_time', 'birth_time'] - for key in null_list: - if key in entry: + # turn - into null for time fields and add calculated timestamp fields + if key in null_list: if entry[key] == '-': entry[key] = None ts = jc.utils.timestamp(entry[key], format_hint=(7100, 7200)) diff --git a/jc/parsers/stat_s.py b/jc/parsers/stat_s.py index 0c73241b..92f26d5c 100644 --- a/jc/parsers/stat_s.py +++ b/jc/parsers/stat_s.py @@ -82,7 +82,7 @@ from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`stat` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -105,16 +105,17 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - int_list = ['size', 'blocks', 'io_blocks', 'inode', 'links', 'uid', 'gid', - 'unix_device', 'rdev', 'block_size'] - for key in proc_data: + int_list = {'size', 'blocks', 'io_blocks', 'inode', 'links', 'uid', 'gid', + 'unix_device', 'rdev', 'block_size'} + + null_list = {'access_time', 'modify_time', 'change_time', 'birth_time'} + + for key in proc_data.copy(): if key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) - # turn - into null for time fields and add calculated timestamp fields - null_list = ['access_time', 'modify_time', 'change_time', 'birth_time'] - for key in null_list: - if key in proc_data: + # turn - into null for time fields and add calculated timestamp fields + if key in null_list: if proc_data[key] == '-': proc_data[key] = None ts = jc.utils.timestamp(proc_data[key], format_hint=(7100, 7200)) diff --git a/jc/parsers/systemctl_lj.py b/jc/parsers/systemctl_lj.py index 671e81e0..01e778f4 100644 --- a/jc/parsers/systemctl_lj.py +++ b/jc/parsers/systemctl_lj.py @@ -75,7 +75,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.6' + version = '1.7' description = '`systemctl list-jobs` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -98,8 +98,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'job'} + for entry in proc_data: - int_list = ['job'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/systeminfo.py b/jc/parsers/systeminfo.py index fde6ec94..8f2e7683 100644 --- a/jc/parsers/systeminfo.py +++ b/jc/parsers/systeminfo.py @@ -212,7 +212,7 @@ import jc.utils class info: """Provides parser metadata (version, author, etc.)""" - version = "1.2" + version = "1.3" description = "`systeminfo` command parser" author = "Jon Smith" author_email = "jon@rebelliondefense.com" @@ -240,6 +240,25 @@ def _process(proc_data): a system already running hyper-v will have an empty "hyperv_requirements" object. """ + int_list = { + "total_physical_memory_mb", + "available_physical_memory_mb", + "virtual_memory_max_size_mb", + "virtual_memory_available_mb", + "virtual_memory_in_use_mb", + } + + dt_list = {"original_install_date", "system_boot_time"} + + hyperv_key = "hyperv_requirements" + + hyperv_subkey_list = { + "vm_monitor_mode_extensions", + "virtualization_enabled_in_firmware", + "second_level_address_translation", + "data_execution_prevention_available", + } + # convert empty strings to None/null for item in proc_data: if isinstance(proc_data[item], str) and not proc_data[item]: @@ -255,17 +274,9 @@ def _process(proc_data): if isinstance(nic[item], str) and not nic[item]: proc_data["network_cards"][i][item] = None - int_list = [ - "total_physical_memory_mb", - "available_physical_memory_mb", - "virtual_memory_max_size_mb", - "virtual_memory_available_mb", - "virtual_memory_in_use_mb", - ] for key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) - dt_list = ["original_install_date", "system_boot_time"] for key in dt_list: tz = proc_data.get("time_zone", "") if tz: @@ -279,13 +290,6 @@ def _process(proc_data): proc_data[key + '_epoch'] = ts.naive proc_data[key + '_epoch_utc'] = ts.utc - hyperv_key = "hyperv_requirements" - hyperv_subkey_list = [ - "vm_monitor_mode_extensions", - "virtualization_enabled_in_firmware", - "second_level_address_translation", - "data_execution_prevention_available", - ] if hyperv_key in proc_data: for key in hyperv_subkey_list: if key in proc_data[hyperv_key]: diff --git a/jc/parsers/time.py b/jc/parsers/time.py index badeda02..b6cb9af5 100644 --- a/jc/parsers/time.py +++ b/jc/parsers/time.py @@ -132,7 +132,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.3' + version = '1.4' description = '`/usr/bin/time` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -154,6 +154,18 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ + int_list = { + 'cpu_percent', 'average_shared_text_size', 'average_unshared_data_size', + 'average_unshared_stack_size', 'average_shared_memory_size', 'maximum_resident_set_size', + 'block_input_operations', 'block_output_operations', 'major_pagefaults', 'minor_pagefaults', + 'swaps', 'page_reclaims', 'page_faults', 'messages_sent', 'messages_received', + 'signals_received', 'voluntary_context_switches', 'involuntary_context_switches', + 'average_stack_size', 'average_total_size', 'average_resident_set_size', + 'signals_delivered', 'page_size', 'exit_status' + } + + float_list = {'real_time', 'user_time', 'system_time'} + if 'command_being_timed' in proc_data: proc_data['command_being_timed'] = proc_data['command_being_timed'][1:-1] @@ -174,19 +186,10 @@ def _process(proc_data): (proc_data['elapsed_time_centiseconds'] / 100) # convert ints and floats - int_list = [ - 'cpu_percent', 'average_shared_text_size', 'average_unshared_data_size', - 'average_unshared_stack_size', 'average_shared_memory_size', 'maximum_resident_set_size', - 'block_input_operations', 'block_output_operations', 'major_pagefaults', 'minor_pagefaults', - 'swaps', 'page_reclaims', 'page_faults', 'messages_sent', 'messages_received', - 'signals_received', 'voluntary_context_switches', 'involuntary_context_switches', - 'average_stack_size', 'average_total_size', 'average_resident_set_size', - 'signals_delivered', 'page_size', 'exit_status' - ] - float_list = ['real_time', 'user_time', 'system_time'] for key in proc_data: if key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) + if key in float_list: proc_data[key] = jc.utils.convert_to_float(proc_data[key]) diff --git a/jc/parsers/timedatectl.py b/jc/parsers/timedatectl.py index 79752a2b..1944b2ce 100644 --- a/jc/parsers/timedatectl.py +++ b/jc/parsers/timedatectl.py @@ -64,7 +64,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.6' + version = '1.7' description = '`timedatectl status` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -87,8 +87,9 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - bool_list = ['ntp_enabled', 'ntp_synchronized', 'rtc_in_local_tz', 'dst_active', - 'system_clock_synchronized', 'systemd-timesyncd.service_active'] + bool_list = {'ntp_enabled', 'ntp_synchronized', 'rtc_in_local_tz', 'dst_active', + 'system_clock_synchronized', 'systemd-timesyncd.service_active'} + for key in proc_data: if key in bool_list: proc_data[key] = jc.utils.convert_to_bool(proc_data[key]) diff --git a/jc/parsers/top.py b/jc/parsers/top.py index 1af63254..c42e68d0 100644 --- a/jc/parsers/top.py +++ b/jc/parsers/top.py @@ -308,7 +308,7 @@ Examples: } ] """ -from typing import List, Dict +from typing import List, Dict, Set import jc.utils from jc.parsers.uptime import parse as parse_uptime from jc.parsers.universal import sparse_table_parse as parse_table @@ -316,7 +316,7 @@ from jc.parsers.universal import sparse_table_parse as parse_table class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`top -b` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -420,7 +420,7 @@ def _process(proc_data: List[Dict], quiet=False) -> List[Dict]: 'Z': 'zombie' } - int_list: List = [ + int_list: Set = { 'uptime', 'users', 'tasks_total', 'tasks_running', 'tasks_sleeping', 'tasks_stopped', 'tasks_zombie', 'pid', 'priority', 'nice', 'parent_pid', 'uid', 'real_uid', 'saved_uid', 'gid', 'pgrp', 'tty_process_gid', 'session_id', 'thread_count', 'last_used_processor', @@ -430,14 +430,14 @@ def _process(proc_data: List[Dict], quiet=False) -> List[Dict]: 'user_namespace_inode', 'nts_namespace_inode', 'numa_node', 'out_of_mem_adjustment', 'out_of_mem_score', 'resident_anon_mem', 'resident_file_backed_mem', 'resident_locked_mem', 'resident_shared_mem' - ] + } - float_list: List = [ + float_list: Set = { 'load_1m', 'load_5m', 'load_15m', 'cpu_user', 'cpu_sys', 'cpu_nice', 'cpu_idle', 'cpu_wait', 'cpu_hardware', 'cpu_software', 'cpu_steal', 'percent_cpu', 'percent_mem', 'mem_total', 'mem_free', 'mem_used', 'mem_buff_cache', 'swap_total', 'swap_free', 'swap_used', 'mem_available', 'virtual_mem', 'resident_mem', 'shared_mem', 'swap', 'code', 'data', 'used' - ] + } for idx, item in enumerate(proc_data): for key in item: diff --git a/jc/parsers/top_s.py b/jc/parsers/top_s.py index 4be47810..85e9dff3 100644 --- a/jc/parsers/top_s.py +++ b/jc/parsers/top_s.py @@ -141,7 +141,7 @@ Examples: {"time":"11:24:50","uptime":"2 min","users":"2","load_1m":"0.23","lo...} ... """ -from typing import List, Dict, Iterable, Union +from typing import List, Dict, Set, Iterable, Union import jc.utils from jc.streaming import ( add_jc_meta, streaming_input_type_check, streaming_line_input_type_check, raise_or_yield @@ -153,7 +153,7 @@ from jc.parsers.universal import sparse_table_parse as parse_table class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`top -b` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -257,7 +257,7 @@ def _process(proc_data: Dict, idx=0, quiet=False) -> Dict: 'Z': 'zombie' } - int_list: List = [ + int_list: Set = { 'uptime', 'users', 'tasks_total', 'tasks_running', 'tasks_sleeping', 'tasks_stopped', 'tasks_zombie', 'pid', 'priority', 'nice', 'parent_pid', 'uid', 'real_uid', 'saved_uid', 'gid', 'pgrp', 'tty_process_gid', 'session_id', 'thread_count', 'last_used_processor', @@ -267,14 +267,14 @@ def _process(proc_data: Dict, idx=0, quiet=False) -> Dict: 'user_namespace_inode', 'nts_namespace_inode', 'numa_node', 'out_of_mem_adjustment', 'out_of_mem_score', 'resident_anon_mem', 'resident_file_backed_mem', 'resident_locked_mem', 'resident_shared_mem' - ] + } - float_list: List = [ + float_list: Set = { 'load_1m', 'load_5m', 'load_15m', 'cpu_user', 'cpu_sys', 'cpu_nice', 'cpu_idle', 'cpu_wait', 'cpu_hardware', 'cpu_software', 'cpu_steal', 'percent_cpu', 'percent_mem', 'mem_total', 'mem_free', 'mem_used', 'mem_buff_cache', 'swap_total', 'swap_free', 'swap_used', 'mem_available', 'virtual_mem', 'resident_mem', 'shared_mem', 'swap', 'code', 'data', 'used' - ] + } for key in proc_data: # root truncation warnings diff --git a/jc/parsers/tracepath.py b/jc/parsers/tracepath.py index 63ebe2c5..dbf9f69f 100644 --- a/jc/parsers/tracepath.py +++ b/jc/parsers/tracepath.py @@ -132,7 +132,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.3' + version = '1.4' description = '`tracepath` and `tracepath6` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -156,11 +156,13 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ # convert ints and floats - int_list = ['pmtu', 'forward_hops', 'return_hops', 'ttl', 'asymmetric_difference'] - float_list = ['reply_ms'] + int_list = {'pmtu', 'forward_hops', 'return_hops', 'ttl', 'asymmetric_difference'} + float_list = {'reply_ms'} + for key in proc_data: if key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) + if key in float_list: proc_data[key] = jc.utils.convert_to_float(proc_data[key]) diff --git a/jc/parsers/traceroute.py b/jc/parsers/traceroute.py index f60c3583..e6d423aa 100644 --- a/jc/parsers/traceroute.py +++ b/jc/parsers/traceroute.py @@ -122,7 +122,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.4' + version = '1.5' description = '`traceroute` and `traceroute6` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -331,8 +331,8 @@ def _process(proc_data): Dictionary. Structured to conform to the schema. """ - int_list = ['hop', 'asn'] - float_list = ['rtt'] + int_list = {'hop', 'asn'} + float_list = {'rtt'} if 'hops' in proc_data: for entry in proc_data['hops']: diff --git a/jc/parsers/ufw.py b/jc/parsers/ufw.py index 35f7843e..53416353 100644 --- a/jc/parsers/ufw.py +++ b/jc/parsers/ufw.py @@ -202,7 +202,7 @@ import ipaddress class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.1' + version = '1.2' description = '`ufw status` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -225,7 +225,7 @@ def _process(proc_data): Dictionary. Structured to conform to the schema. """ - int_list = ['index', 'to_ip_prefix', 'from_ip_prefix'] + int_list = {'index', 'to_ip_prefix', 'from_ip_prefix'} if 'rules' in proc_data: for i, item in enumerate(proc_data['rules']): diff --git a/jc/parsers/ufw_appinfo.py b/jc/parsers/ufw_appinfo.py index 3327c3f5..beb4cd04 100644 --- a/jc/parsers/ufw_appinfo.py +++ b/jc/parsers/ufw_appinfo.py @@ -138,7 +138,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.1' + version = '1.2' description = '`ufw app info [application]` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -161,10 +161,10 @@ def _process(proc_data): List of Dictionaries. Structured to conform to the schema. """ + int_list = {'start', 'end'} + for profile in proc_data: # convert to ints - int_list = ['start', 'end'] - if 'tcp_list' in profile: profile['tcp_list'] = [int(p) for p in profile['tcp_list']] diff --git a/jc/parsers/update_alt_q.py b/jc/parsers/update_alt_q.py index 7833fc3e..a785c19c 100644 --- a/jc/parsers/update_alt_q.py +++ b/jc/parsers/update_alt_q.py @@ -132,7 +132,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.1' + version = '1.2' description = '`update-alternatives --query` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -155,7 +155,7 @@ def _process(proc_data: Dict) -> Dict: Dictionary. Structured to conform to the schema. """ - int_list = ['priority'] + int_list = {'priority'} if 'value' in proc_data: if proc_data['value'] == 'none': diff --git a/jc/parsers/uptime.py b/jc/parsers/uptime.py index 99e5f705..8a596e72 100644 --- a/jc/parsers/uptime.py +++ b/jc/parsers/uptime.py @@ -65,7 +65,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.6' + version = '1.7' description = '`uptime` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -88,6 +88,9 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ + int_list = {'users'} + float_list = {'load_1m', 'load_5m', 'load_15m'} + if 'time' in proc_data: time_list = proc_data['time'].split(':') proc_data['time_hour'] = jc.utils.convert_to_int(time_list[0]) @@ -128,11 +131,10 @@ def _process(proc_data): proc_data['uptime_total_seconds'] = uptime_total_seconds # integer and float conversions - int_list = ['users'] - float_list = ['load_1m', 'load_5m', 'load_15m'] for key in proc_data: if key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) + if key in float_list: proc_data[key] = jc.utils.convert_to_float(proc_data[key]) diff --git a/jc/parsers/vmstat.py b/jc/parsers/vmstat.py index 6b44c7eb..cf799466 100644 --- a/jc/parsers/vmstat.py +++ b/jc/parsers/vmstat.py @@ -126,7 +126,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.2' + version = '1.3' description = '`vmstat` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -149,14 +149,15 @@ def _process(proc_data): List of Dictionaries. Structured to conform to the schema. """ - - int_list = [ + int_list = { 'runnable_procs', 'uninterruptible_sleeping_procs', 'virtual_mem_used', 'free_mem', 'buffer_mem', 'cache_mem', 'inactive_mem', 'active_mem', 'swap_in', 'swap_out', 'blocks_in', 'blocks_out', 'interrupts', 'context_switches', 'user_time', 'system_time', 'idle_time', 'io_wait_time', 'stolen_time', 'total_reads', 'merged_reads', 'sectors_read', 'reading_ms', 'total_writes', 'merged_writes', 'sectors_written', 'writing_ms', 'current_io', 'io_seconds' - ] + } + + fmt_hint = (7250, 7255) for entry in proc_data: for key in entry: @@ -164,7 +165,6 @@ def _process(proc_data): entry[key] = jc.utils.convert_to_int(entry[key]) if entry['timestamp']: - fmt_hint = (7250, 7255) ts = jc.utils.timestamp(f'{entry["timestamp"]} {entry["timezone"]}', format_hint=fmt_hint) entry['epoch'] = ts.naive entry['epoch_utc'] = ts.utc diff --git a/jc/parsers/vmstat_s.py b/jc/parsers/vmstat_s.py index e7d12e34..ca47a9a4 100644 --- a/jc/parsers/vmstat_s.py +++ b/jc/parsers/vmstat_s.py @@ -100,7 +100,7 @@ from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`vmstat` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -123,13 +123,13 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - int_list = [ + int_list = { 'runnable_procs', 'uninterruptible_sleeping_procs', 'virtual_mem_used', 'free_mem', 'buffer_mem', 'cache_mem', 'inactive_mem', 'active_mem', 'swap_in', 'swap_out', 'blocks_in', 'blocks_out', 'interrupts', 'context_switches', 'user_time', 'system_time', 'idle_time', 'io_wait_time', 'stolen_time', 'total_reads', 'merged_reads', 'sectors_read', 'reading_ms', 'total_writes', 'merged_writes', 'sectors_written', 'writing_ms', 'current_io', 'io_seconds' - ] + } for key in proc_data: if key in int_list: diff --git a/jc/parsers/w.py b/jc/parsers/w.py index a40659d8..395f4d7d 100644 --- a/jc/parsers/w.py +++ b/jc/parsers/w.py @@ -104,7 +104,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`w` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -127,8 +127,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + null_list = {'user', 'tty', 'from', 'login_at', 'idle', 'what'} + for entry in proc_data: - null_list = ['user', 'tty', 'from', 'login_at', 'idle', 'what'] for key in entry: if key in null_list: if entry[key] == '-': diff --git a/jc/parsers/wc.py b/jc/parsers/wc.py index 9638a5b0..2d171a0d 100644 --- a/jc/parsers/wc.py +++ b/jc/parsers/wc.py @@ -54,7 +54,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.3' + version = '1.4' description = '`wc` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -77,9 +77,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'lines', 'words', 'characters'} for entry in proc_data: - int_list = ['lines', 'words', 'characters'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/who.py b/jc/parsers/who.py index 199c6498..80d8093b 100644 --- a/jc/parsers/who.py +++ b/jc/parsers/who.py @@ -136,7 +136,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.6' + version = '1.7' description = '`who` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -159,8 +159,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'pid'} + for entry in proc_data: - int_list = ['pid'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/zipinfo.py b/jc/parsers/zipinfo.py index 33599090..edb5f825 100644 --- a/jc/parsers/zipinfo.py +++ b/jc/parsers/zipinfo.py @@ -77,7 +77,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`zipinfo` command parser' author = 'Matt J' author_email = 'https://github.com/listuser' @@ -100,14 +100,16 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'bytes_compressed', 'bytes_uncompressed', 'number_entries', + 'number_files', 'size', 'filesize'} + + float_list = {'percent_compressed'} for entry in proc_data: - int_list = ['bytes_compressed', 'bytes_uncompressed', 'number_entries', - 'number_files', 'size', 'filesize'] - float_list = ['percent_compressed'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) + if key in float_list: entry[key] = jc.utils.convert_to_float(entry[key])