From bc15a636f1fcda61ecd2653493fd1a482dd8496c Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Fri, 15 Jul 2022 17:28:08 -0700 Subject: [PATCH] optimizations up to netstat --- CHANGELOG | 1 + jc/parsers/acpi.py | 8 ++--- jc/parsers/airport.py | 8 ++--- jc/parsers/airport_s.py | 6 ++-- jc/parsers/arp.py | 4 +-- jc/parsers/blkid.py | 14 +++++---- jc/parsers/chage.py | 6 ++-- jc/parsers/cksum.py | 4 +-- jc/parsers/date.py | 6 ++-- jc/parsers/df.py | 8 ++--- jc/parsers/dig.py | 7 +++-- jc/parsers/dir.py | 5 +-- jc/parsers/dmidecode.py | 5 +-- jc/parsers/dpkg_l.py | 69 ++++++++++++++++++----------------------- jc/parsers/du.py | 5 +-- jc/parsers/free.py | 4 +-- jc/parsers/fstab.py | 5 +-- jc/parsers/git_log.py | 5 +-- jc/parsers/git_log_s.py | 4 +-- jc/parsers/group.py | 5 +-- jc/parsers/hash.py | 5 +-- jc/parsers/hciconfig.py | 10 +++--- jc/parsers/ifconfig.py | 13 ++++---- jc/parsers/iostat.py | 14 ++++++--- jc/parsers/iostat_s.py | 12 ++++--- jc/parsers/jobs.py | 8 ++--- jc/parsers/ls.py | 5 +-- jc/parsers/ls_s.py | 5 +-- jc/parsers/lsblk.py | 12 ++++--- jc/parsers/lsmod.py | 5 +-- jc/parsers/lsof.py | 5 +-- jc/parsers/m3u.py | 3 +- jc/parsers/mpstat.py | 7 +++-- jc/parsers/mpstat_s.py | 7 +++-- jc/parsers/netstat.py | 25 ++++++++------- 35 files changed, 170 insertions(+), 145 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 7ccd884c..08a39992 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,7 @@ jc changelog 20220705 v1.20.3 - Add pager functionality to help (parser documentation only) - Add m3u/m3u8 file parser +- Minor parser performance optimizations 20220705 v1.20.2 - Add `gpg --with-colons` parser tested on linux diff --git a/jc/parsers/acpi.py b/jc/parsers/acpi.py index 32769e57..2f5d1f68 100644 --- a/jc/parsers/acpi.py +++ b/jc/parsers/acpi.py @@ -227,7 +227,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.3' + version = '1.4' description = '`acpi` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -250,9 +250,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ - int_list = ['id', 'charge_percent', 'design_capacity_mah', 'last_full_capacity', - 'last_full_capacity_percent'] - float_list = ['temperature'] + int_list = {'id', 'charge_percent', 'design_capacity_mah', 'last_full_capacity', + 'last_full_capacity_percent'} + float_list = {'temperature'} for entry in proc_data: for key in entry: diff --git a/jc/parsers/airport.py b/jc/parsers/airport.py index 045802b8..18f0431e 100644 --- a/jc/parsers/airport.py +++ b/jc/parsers/airport.py @@ -80,7 +80,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.4' + version = '1.5' description = '`airport -I` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -103,9 +103,9 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - # integer changes - int_list = ['agrctlrssi', 'agrextrssi', 'agrctlnoise', 'agrextnoise', - 'lasttxrate', 'maxrate', 'lastassocstatus', 'mcs'] + int_list = {'agrctlrssi', 'agrextrssi', 'agrctlnoise', 'agrextnoise', + 'lasttxrate', 'maxrate', 'lastassocstatus', 'mcs'} + for key in proc_data: if key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) diff --git a/jc/parsers/airport_s.py b/jc/parsers/airport_s.py index b635acaa..7089f14f 100644 --- a/jc/parsers/airport_s.py +++ b/jc/parsers/airport_s.py @@ -109,7 +109,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`airport -s` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -135,8 +135,8 @@ def _process(proc_data): for entry in proc_data: # convert integers and booleans - int_list = ['rssi'] - bool_list = ['ht'] + int_list = {'rssi'} + bool_list = {'ht'} for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/arp.py b/jc/parsers/arp.py index f0bc43a3..0012b891 100644 --- a/jc/parsers/arp.py +++ b/jc/parsers/arp.py @@ -119,7 +119,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.9' + version = '1.10' description = '`arp` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -142,13 +142,13 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured data to conform to the schema: """ + int_list = {'expires'} # in BSD style, change name to null if it is a question mark for entry in proc_data: if 'name' in entry and entry['name'] == '?': entry['name'] = None - int_list = ['expires'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/blkid.py b/jc/parsers/blkid.py index 808b903e..6cd36794 100644 --- a/jc/parsers/blkid.py +++ b/jc/parsers/blkid.py @@ -121,7 +121,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`blkid` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -144,15 +144,17 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = { + 'part_entry_number', 'part_entry_offset', 'part_entry_size', 'id_part_entry_number', + 'id_part_entry_offset', 'id_part_entry_size', 'minimum_io_size', 'physical_sector_size', + 'logical_sector_size', 'id_iolimit_minimum_io_size', 'id_iolimit_physical_sector_size', + 'id_iolimit_logical_sector_size' + } + for entry in proc_data: if 'devname' in entry: entry['device'] = entry.pop('devname') - int_list = ['part_entry_number', 'part_entry_offset', 'part_entry_size', - 'id_part_entry_number', 'id_part_entry_offset', 'id_part_entry_size', - 'minimum_io_size', 'physical_sector_size', 'logical_sector_size', - 'id_iolimit_minimum_io_size', 'id_iolimit_physical_sector_size', - 'id_iolimit_logical_sector_size'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/chage.py b/jc/parsers/chage.py index d23844f9..1d56908c 100644 --- a/jc/parsers/chage.py +++ b/jc/parsers/chage.py @@ -57,7 +57,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`chage --list` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -80,8 +80,8 @@ def _process(proc_data: Dict) -> Dict: Dictionary. Structured to conform to the schema. """ - int_list = ['min_days_between_password_change', 'max_days_between_password_change', - 'warning_days_before_password_expires'] + int_list = {'min_days_between_password_change', 'max_days_between_password_change', + 'warning_days_before_password_expires'} for key in proc_data: if key in int_list: diff --git a/jc/parsers/cksum.py b/jc/parsers/cksum.py index f0fd52ed..4368853c 100644 --- a/jc/parsers/cksum.py +++ b/jc/parsers/cksum.py @@ -54,7 +54,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.3' + version = '1.4' description = '`cksum` and `sum` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -77,9 +77,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'checksum', 'blocks'} for entry in proc_data: - int_list = ['checksum', 'blocks'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/date.py b/jc/parsers/date.py index 61180f57..4b6d9430 100644 --- a/jc/parsers/date.py +++ b/jc/parsers/date.py @@ -78,7 +78,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '2.4' + version = '2.5' description = '`date` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -128,7 +128,7 @@ def parse(data, raw=False, quiet=False): # find the timezone no matter where it is in the string # from https://www.timeanddate.com/time/zones/ - tz_abbr = [ + tz_abbr = { 'A', 'ACDT', 'ACST', 'ACT', 'ACWST', 'ADT', 'AEDT', 'AEST', 'AET', 'AFT', 'AKDT', 'AKST', 'ALMT', 'AMST', 'AMT', 'ANAST', 'ANAT', 'AQTT', 'ART', 'AST', 'AT', 'AWDT', 'AWST', 'AZOST', 'AZOT', 'AZST', 'AZT', 'AoE', 'B', 'BNT', 'BOT', 'BRST', 'BRT', 'BST', @@ -154,7 +154,7 @@ def parse(data, raw=False, quiet=False): 'UTC+0400', 'UTC+0430', 'UTC+0500', 'UTC+0530', 'UTC+0545', 'UTC+0600', 'UTC+0630', 'UTC+0700', 'UTC+0800', 'UTC+0845', 'UTC+0900', 'UTC+1000', 'UTC+1030', 'UTC+1100', 'UTC+1200', 'UTC+1300', 'UTC+1345', 'UTC+1400' - ] + } tz = None for term in data.replace('(', '').replace(')', '').split(): if term in tz_abbr: diff --git a/jc/parsers/df.py b/jc/parsers/df.py index 4907041c..16c1d288 100644 --- a/jc/parsers/df.py +++ b/jc/parsers/df.py @@ -99,7 +99,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.10' + version = '1.11' description = '`df` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -122,6 +122,8 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema: """ + int_list = {'used', 'available', 'use_percent', 'capacity_percent', 'ifree', + 'iused', 'iused_percent'} for entry in proc_data: # change 'avail' to 'available' @@ -155,9 +157,7 @@ def _process(proc_data): if 'iused_percent' in entry: entry['iused_percent'] = entry['iused_percent'].rstrip('%') - # change used, available, use_percent, capacity_percent, ifree, iused, iused_percent to int - int_list = ['used', 'available', 'use_percent', 'capacity_percent', 'ifree', - 'iused', 'iused_percent'] + # convert integers for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/dig.py b/jc/parsers/dig.py index 7119889f..fcb24b68 100644 --- a/jc/parsers/dig.py +++ b/jc/parsers/dig.py @@ -322,7 +322,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '2.3' + version = '2.4' description = '`dig` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -345,9 +345,10 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'id', 'query_num', 'answer_num', 'authority_num', 'additional_num', + 'rcvd', 'query_size', 'query_time'} + for entry in proc_data: - int_list = ['id', 'query_num', 'answer_num', 'authority_num', 'additional_num', - 'rcvd', 'query_size', 'query_time'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/dir.py b/jc/parsers/dir.py index 173475ff..68ea7070 100644 --- a/jc/parsers/dir.py +++ b/jc/parsers/dir.py @@ -121,7 +121,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`dir` command parser' author = 'Rasheed Elsaleh' author_email = 'rasheed@rebelliondefense.com' @@ -143,6 +143,8 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'size'} + for entry in proc_data: # add timestamps if 'date' in entry and 'time' in entry: @@ -151,7 +153,6 @@ def _process(proc_data): entry['epoch'] = timestamp.naive # add ints - int_list = ["size"] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/dmidecode.py b/jc/parsers/dmidecode.py index 23b7f392..e2f23ea1 100644 --- a/jc/parsers/dmidecode.py +++ b/jc/parsers/dmidecode.py @@ -125,7 +125,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.4' + version = '1.5' description = '`dmidecode` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -148,8 +148,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'type', 'bytes'} + for entry in proc_data: - int_list = ['type', 'bytes'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/dpkg_l.py b/jc/parsers/dpkg_l.py index 4e8e0e4c..0fccc422 100644 --- a/jc/parsers/dpkg_l.py +++ b/jc/parsers/dpkg_l.py @@ -132,7 +132,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.2' + version = '1.3' description = '`dpkg -l` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -155,48 +155,41 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema: """ + desired_map = { + 'u': 'unknown', + 'i': 'install', + 'r': 'remove', + 'p': 'purge', + 'h': 'hold' + } + + status_map = { + 'n': 'not installed', + 'i': 'installed', + 'c': 'config-files', + 'u': 'unpacked', + 'f': 'failed config', + 'h': 'half installed', + 'w': 'trigger await', + 't': 'trigger pending' + } + + err_map = { + 'r': 'reinstall required' + } + for entry in proc_data: if 'codes' in entry: - desired, status, *err = list(entry['codes']) + desired, status, *err = list(entry['codes'].lower()) - desired_map = { - 'u': 'unknown', - 'i': 'install', - 'r': 'remove', - 'p': 'purge', - 'h': 'hold' - } + if desired in desired_map: + entry['desired'] = desired_map[desired] - for key, value in desired_map.items(): - if desired.lower() == key: - entry['desired'] = value - break + if status in status_map: + entry['status'] = status_map[status] - status_map = { - 'n': 'not installed', - 'i': 'installed', - 'c': 'config-files', - 'u': 'unpacked', - 'f': 'failed config', - 'h': 'half installed', - 'w': 'trigger await', - 't': 'trigger pending' - } - - for key, value in status_map.items(): - if status.lower() == key: - entry['status'] = value - break - - if err: - err_map = { - 'r': 'reinstall required' - } - - for key, value in err_map.items(): - if err[0].lower() == key: - entry['error'] = value - break + if err and err[0] in err_map: + entry['error'] = err_map[err[0]] return proc_data diff --git a/jc/parsers/du.py b/jc/parsers/du.py index 60403345..fa13c157 100644 --- a/jc/parsers/du.py +++ b/jc/parsers/du.py @@ -88,7 +88,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`du` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -111,7 +111,8 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ - int_list = ['size'] + int_list = {'size'} + for entry in proc_data: for key in entry: if key in int_list: diff --git a/jc/parsers/free.py b/jc/parsers/free.py index c87d294e..4b460d32 100644 --- a/jc/parsers/free.py +++ b/jc/parsers/free.py @@ -73,7 +73,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`free` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -96,9 +96,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'total', 'used', 'free', 'shared', 'buff_cache', 'available'} for entry in proc_data: - int_list = ['total', 'used', 'free', 'shared', 'buff_cache', 'available'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/fstab.py b/jc/parsers/fstab.py index 3d136de2..f51afed8 100644 --- a/jc/parsers/fstab.py +++ b/jc/parsers/fstab.py @@ -85,7 +85,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.6' + version = '1.7' description = '`/etc/fstab` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -107,8 +107,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'fs_freq', 'fs_passno'} + for entry in proc_data: - int_list = ['fs_freq', 'fs_passno'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/git_log.py b/jc/parsers/git_log.py index 3e205a0d..5fcc49cb 100644 --- a/jc/parsers/git_log.py +++ b/jc/parsers/git_log.py @@ -153,7 +153,7 @@ changes_pattern = re.compile(r'\s(?P\d+)\s+(files? changed),\s+(?P List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = ['files_changed', 'insertions', 'deletions'] + int_list = {'files_changed', 'insertions', 'deletions'} for entry in proc_data: if 'date' in entry: @@ -188,6 +188,7 @@ def _process(proc_data: List[Dict]) -> List[Dict]: for key in entry['stats']: if key in int_list: entry['stats'][key] = jc.utils.convert_to_int(entry['stats'][key]) + return proc_data diff --git a/jc/parsers/git_log_s.py b/jc/parsers/git_log_s.py index 1bd3ef5a..63a27a1b 100644 --- a/jc/parsers/git_log_s.py +++ b/jc/parsers/git_log_s.py @@ -87,7 +87,7 @@ changes_pattern = re.compile(r'\s(?P\d+)\s+(files? changed),\s+(?P Dict: Dictionary. Structured data to conform to the schema. """ - int_list = ['files_changed', 'insertions', 'deletions'] + int_list = {'files_changed', 'insertions', 'deletions'} if 'date' in proc_data: ts = jc.utils.timestamp(proc_data['date'], format_hint=(1100,)) diff --git a/jc/parsers/group.py b/jc/parsers/group.py index 72a433ef..e24ec987 100644 --- a/jc/parsers/group.py +++ b/jc/parsers/group.py @@ -109,7 +109,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.4' + version = '1.5' description = '`/etc/group` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -131,8 +131,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'gid'} + for entry in proc_data: - int_list = ['gid'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/hash.py b/jc/parsers/hash.py index 2c25ba81..9de1b592 100644 --- a/jc/parsers/hash.py +++ b/jc/parsers/hash.py @@ -38,7 +38,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.3' + version = '1.4' description = '`hash` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -60,8 +60,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'hits'} + for entry in proc_data: - int_list = ['hits'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/hciconfig.py b/jc/parsers/hciconfig.py index 88d84d32..2e7d13a3 100644 --- a/jc/parsers/hciconfig.py +++ b/jc/parsers/hciconfig.py @@ -317,7 +317,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.3' + version = '1.4' description = '`hciconfig` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -340,12 +340,12 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = { + 'acl_mtu', 'acl_mtu_packets', 'sco_mtu', 'sco_mtu_packets', 'rx_bytes', 'rx_acl', 'rx_sco', + 'rx_events', 'rx_errors', 'tx_bytes', 'tx_acl', 'tx_sco', 'tx_commands', 'tx_errors' + } for entry in proc_data: - - int_list = ['acl_mtu', 'acl_mtu_packets', 'sco_mtu', 'sco_mtu_packets', - 'rx_bytes', 'rx_acl', 'rx_sco', 'rx_events', 'rx_errors', - 'tx_bytes', 'tx_acl', 'tx_sco', 'tx_commands', 'tx_errors'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/ifconfig.py b/jc/parsers/ifconfig.py index 65a95893..7c0d1b67 100644 --- a/jc/parsers/ifconfig.py +++ b/jc/parsers/ifconfig.py @@ -188,7 +188,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.11' + version = '1.12' description = '`ifconfig` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -430,12 +430,13 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = { + 'flags', 'mtu', 'ipv6_mask', 'rx_packets', 'rx_bytes', 'rx_errors', 'rx_dropped', + 'rx_overruns', 'rx_frame', 'tx_packets', 'tx_bytes', 'tx_errors', 'tx_dropped', + 'tx_overruns', 'tx_carrier', 'tx_collisions', 'metric' + } + for entry in proc_data: - int_list = [ - 'flags', 'mtu', 'ipv6_mask', 'rx_packets', 'rx_bytes', 'rx_errors', 'rx_dropped', - 'rx_overruns', 'rx_frame', 'tx_packets', 'tx_bytes', 'tx_errors', 'tx_dropped', - 'tx_overruns', 'tx_carrier', 'tx_collisions', 'metric' - ] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/iostat.py b/jc/parsers/iostat.py index 8474ca0d..66982e06 100644 --- a/jc/parsers/iostat.py +++ b/jc/parsers/iostat.py @@ -160,7 +160,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`iostat` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -183,9 +183,8 @@ def _process(proc_data): List of Dictionaries. Structured to conform to the schema. """ - for entry in proc_data: - float_list = [ + float_list = { 'percent_user', 'percent_nice', 'percent_system', 'percent_iowait', 'percent_steal', 'percent_idle', 'tps', 'kb_read_s', 'mb_read_s', 'kb_wrtn_s', 'mb_wrtn_s', 'rrqm_s', 'wrqm_s', 'r_s', 'w_s', 'rmb_s', 'rkb_s', 'wmb_s', @@ -193,8 +192,10 @@ def _process(proc_data): 'percent_util', 'percent_rrqm', 'percent_wrqm', 'aqu_sz', 'rareq_sz', 'wareq_sz', 'd_s', 'dkb_s', 'dmb_s', 'drqm_s', 'percent_drqm', 'd_await', 'dareq_sz', 'f_s', 'f_await', 'kb_dscd_s', 'mb_dscd_s' - ] - int_list = ['kb_read', 'mb_read', 'kb_wrtn', 'mb_wrtn', 'kb_dscd', 'mb_dscd'] + } + + int_list = {'kb_read', 'mb_read', 'kb_wrtn', 'mb_wrtn', 'kb_dscd', 'mb_dscd'} + for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) @@ -204,15 +205,18 @@ def _process(proc_data): return proc_data + def _normalize_headers(line): return line.replace('%', 'percent_').replace('/', '_').replace('-', '_').lower() + def _create_obj_list(section_list, section_name): output_list = jc.parsers.universal.simple_table_parse(section_list) for item in output_list: item['type'] = section_name return output_list + def parse(data, raw=False, quiet=False): """ Main text parsing function diff --git a/jc/parsers/iostat_s.py b/jc/parsers/iostat_s.py index eb3a72c6..95850a71 100644 --- a/jc/parsers/iostat_s.py +++ b/jc/parsers/iostat_s.py @@ -108,7 +108,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.1' + version = '1.2' description = '`iostat` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -131,7 +131,7 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - float_list = [ + float_list = { 'percent_user', 'percent_nice', 'percent_system', 'percent_iowait', 'percent_steal', 'percent_idle', 'tps', 'kb_read_s', 'mb_read_s', 'kb_wrtn_s', 'mb_wrtn_s', 'rrqm_s', 'wrqm_s', 'r_s', 'w_s', 'rmb_s', 'rkb_s', 'wmb_s', @@ -139,8 +139,10 @@ def _process(proc_data): 'percent_util', 'percent_rrqm', 'percent_wrqm', 'aqu_sz', 'rareq_sz', 'wareq_sz', 'd_s', 'dkb_s', 'dmb_s', 'drqm_s', 'percent_drqm', 'd_await', 'dareq_sz', 'f_s', 'f_await', 'kb_dscd_s', 'mb_dscd_s' - ] - int_list = ['kb_read', 'mb_read', 'kb_wrtn', 'mb_wrtn', 'kb_dscd', 'mb_dscd'] + } + + int_list = {'kb_read', 'mb_read', 'kb_wrtn', 'mb_wrtn', 'kb_dscd', 'mb_dscd'} + for key in proc_data: if key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) @@ -150,9 +152,11 @@ def _process(proc_data): return proc_data + def _normalize_headers(line): return line.replace('%', 'percent_').replace('/', '_').replace('-', '_').lower() + def _create_obj_list(section_list, section_name): output_list = jc.parsers.universal.simple_table_parse(section_list) for item in output_list: diff --git a/jc/parsers/jobs.py b/jc/parsers/jobs.py index 7e39e7fb..8637d81b 100644 --- a/jc/parsers/jobs.py +++ b/jc/parsers/jobs.py @@ -88,13 +88,12 @@ Example: } ] """ -import string import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`jobs` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -117,8 +116,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'job_number', 'pid'} + for entry in proc_data: - int_list = ['job_number', 'pid'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) @@ -160,7 +160,7 @@ def parse(data, raw=False, quiet=False): parsed_line = entry.split(maxsplit=2) # check if -l was used - if parsed_line[1][0] in string.digits: + if parsed_line[1][0].isdecimal(): pid = parsed_line.pop(1) remainder = parsed_line.pop(1) job_number = parsed_line.pop(0) diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index 92de479d..3c919e78 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -118,7 +118,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.11' + version = '1.12' description = '`ls` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -141,8 +141,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'links', 'size'} + for entry in proc_data: - int_list = ['links', 'size'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/ls_s.py b/jc/parsers/ls_s.py index 650e30fc..b768ce76 100644 --- a/jc/parsers/ls_s.py +++ b/jc/parsers/ls_s.py @@ -77,7 +77,7 @@ from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`ls` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -100,7 +100,8 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - int_list = ['links', 'size'] + int_list = {'links', 'size'} + for key in proc_data: if key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) diff --git a/jc/parsers/lsblk.py b/jc/parsers/lsblk.py index 64326f92..95365c48 100644 --- a/jc/parsers/lsblk.py +++ b/jc/parsers/lsblk.py @@ -275,7 +275,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.8' + version = '1.9' description = '`lsblk` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -298,14 +298,16 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + bool_list = {'rm', 'ro', 'rota', 'disc_zero', 'rand'} + + int_list = {'ra', 'alignment', 'min_io', 'opt_io', 'phy_sec', 'log_sec', + 'rq_size', 'disc_aln'} + for entry in proc_data: - # boolean and integer changes - bool_list = ['rm', 'ro', 'rota', 'disc_zero', 'rand'] - int_list = ['ra', 'alignment', 'min_io', 'opt_io', 'phy_sec', 'log_sec', - 'rq_size', 'disc_aln'] for key in entry: if key in bool_list: entry[key] = jc.utils.convert_to_bool(entry[key]) + if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/lsmod.py b/jc/parsers/lsmod.py index 524d6960..594c9fd6 100644 --- a/jc/parsers/lsmod.py +++ b/jc/parsers/lsmod.py @@ -126,7 +126,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.6' + version = '1.7' description = '`lsmod` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -149,8 +149,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'size', 'used'} + for entry in proc_data: - int_list = ['size', 'used'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/lsof.py b/jc/parsers/lsof.py index e0865d99..9f48a7c7 100644 --- a/jc/parsers/lsof.py +++ b/jc/parsers/lsof.py @@ -120,7 +120,7 @@ import jc.parsers.universal class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.5' + version = '1.6' description = '`lsof` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -143,8 +143,9 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = {'pid', 'tid', 'size_off', 'node'} + for entry in proc_data: - int_list = ['pid', 'tid', 'size_off', 'node'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) diff --git a/jc/parsers/m3u.py b/jc/parsers/m3u.py index 3eb4e71d..22c6558c 100644 --- a/jc/parsers/m3u.py +++ b/jc/parsers/m3u.py @@ -79,7 +79,8 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - int_list = ['runtime'] + int_list = {'runtime'} + for entry in proc_data: for key in entry: if key in int_list: diff --git a/jc/parsers/mpstat.py b/jc/parsers/mpstat.py index 896328d1..368a64c5 100644 --- a/jc/parsers/mpstat.py +++ b/jc/parsers/mpstat.py @@ -116,7 +116,7 @@ from jc.parsers.universal import simple_table_parse class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`mpstat` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -139,14 +139,15 @@ def _process(proc_data: List[Dict]) -> List[Dict]: List of Dictionaries. Structured to conform to the schema. """ - float_list = [ + float_list = { "percent_usr", "percent_nice", "percent_sys", "percent_iowait", "percent_irq", "percent_soft", "percent_steal", "percent_guest", "percent_gnice", "percent_idle", "intr_s", "nmi_s", "loc_s", "spu_s", "pmi_s", "iwi_s", "rtr_s", "res_s", "cal_s", "tlb_s", "trm_s", "thr_s", "dfr_s", "mce_s", "mcp_s", "err_s", "mis_s", "pin_s", "npi_s", "piw_s", "hi_s", "timer_s", "net_tx_s", "net_rx_s", "block_s", "irq_poll_s", "block_iopoll_s", "tasklet_s", "sched_s", "hrtimer_s", "rcu_s" - ] + } + for entry in proc_data: for key in entry: if (key in float_list or (key[0].isdigit() and key.endswith('_s'))): diff --git a/jc/parsers/mpstat_s.py b/jc/parsers/mpstat_s.py index 4e8e5769..cecd6164 100644 --- a/jc/parsers/mpstat_s.py +++ b/jc/parsers/mpstat_s.py @@ -101,7 +101,7 @@ from jc.exceptions import ParseError class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.0' + version = '1.1' description = '`mpstat` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -124,14 +124,15 @@ def _process(proc_data: Dict) -> Dict: Dictionary. Structured data to conform to the schema. """ - float_list = [ + float_list = { "percent_usr", "percent_nice", "percent_sys", "percent_iowait", "percent_irq", "percent_soft", "percent_steal", "percent_guest", "percent_gnice", "percent_idle", "intr_s", "nmi_s", "loc_s", "spu_s", "pmi_s", "iwi_s", "rtr_s", "res_s", "cal_s", "tlb_s", "trm_s", "thr_s", "dfr_s", "mce_s", "mcp_s", "err_s", "mis_s", "pin_s", "npi_s", "piw_s", "hi_s", "timer_s", "net_tx_s", "net_rx_s", "block_s", "irq_poll_s", "block_iopoll_s", "tasklet_s", "sched_s", "hrtimer_s", "rcu_s" - ] + } + for key in proc_data: if (key in float_list or (key[0].isdigit() and key.endswith('_s'))): proc_data[key] = jc.utils.convert_to_float(proc_data[key]) diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index 9b0a48f0..4e30de77 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -355,7 +355,7 @@ import jc.utils class info(): """Provides parser metadata (version, author, etc.)""" - version = '1.12' + version = '1.13' description = '`netstat` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -378,20 +378,23 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ + int_list = { + 'recv_q', 'send_q', 'pid', 'refcnt', 'inode', 'unit', 'vendor', 'class', 'osx_flags', + 'subcla', 'pcbcount', 'rcvbuf', 'sndbuf', 'rxbytes', 'txbytes', 'route_refs', 'use', + 'mtu', 'mss', 'window', 'irtt', 'metric', 'ipkts', 'ierrs', 'opkts', 'oerrs', 'coll', + 'rx_ok', 'rx_err', 'rx_drp', 'rx_ovr', 'tx_ok', 'tx_err', 'tx_drp', 'tx_ovr', 'idrop', + 'ibytes', 'obytes', 'r_mbuf', 's_mbuf', 'r_clus', 's_clus', 'r_hiwa', 's_hiwa', + 'r_lowa', 's_lowa', 'r_bcnt', 's_bcnt', 'r_bmax', 's_bmax', 'rexmit', 'ooorcv', '0_win' + } + + float_list = {'rexmt', 'persist', 'keep', '2msl', 'delack', 'rcvtime'} + for entry in proc_data: - # integer and float conversions - int_list = [ - 'recv_q', 'send_q', 'pid', 'refcnt', 'inode', 'unit', 'vendor', 'class', 'osx_flags', - 'subcla', 'pcbcount', 'rcvbuf', 'sndbuf', 'rxbytes', 'txbytes', 'route_refs', 'use', - 'mtu', 'mss', 'window', 'irtt', 'metric', 'ipkts', 'ierrs', 'opkts', 'oerrs', 'coll', - 'rx_ok', 'rx_err', 'rx_drp', 'rx_ovr', 'tx_ok', 'tx_err', 'tx_drp', 'tx_ovr', 'idrop', - 'ibytes', 'obytes', 'r_mbuf', 's_mbuf', 'r_clus', 's_clus', 'r_hiwa', 's_hiwa', - 'r_lowa', 's_lowa', 'r_bcnt', 's_bcnt', 'r_bmax', 's_bmax', 'rexmit', 'ooorcv', '0_win' - ] - float_list = ['rexmt', 'persist', 'keep', '2msl', 'delack', 'rcvtime'] + # ints and floats for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) + if key in float_list: entry[key] = jc.utils.convert_to_float(entry[key])