Merge pull request #180 from kellyjonbrazil/dev

Dev v1.17.2

CHANGELOG

@@ -1,5 +1,10 @@
 jc changelog
 
+20211117 v1.17.2
+- Fix ping parser to add Alpine linux support
+- Fix netstat parser for older versions of netstat on linux
+- Fix df parser for cases where the filesystem field overflows the column length
+
 20211030 v1.17.1
 - Fix file parser for gzip files
 - Fix uname parser for cases where the 'processor' and/or 'hardware_platform' fields are missing on linux

docs/parsers/df.md

@@ -122,4 +122,4 @@ Returns:
 ## Parser Information
 Compatibility:  linux, darwin, freebsd
 
-Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.8 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/netstat.md

@@ -379,4 +379,4 @@ Returns:
 ## Parser Information
 Compatibility:  linux, darwin, freebsd
 
-Version 1.10 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.11 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/ping.md

@@ -181,4 +181,4 @@ Returns:
 ## Parser Information
 Compatibility:  linux, darwin, freebsd
 
-Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com)

jc/__init__.py

@@ -73,4 +73,4 @@ Module Example:
 """
 
 name = 'jc'
-__version__ = '1.17.1'
+__version__ = '1.17.2'

jc/parsers/df.py

@@ -92,13 +92,14 @@ Examples:
       ...
     ]
 """
+import hashlib
 import jc.utils
 import jc.parsers.universal
 
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.7'
+    version = '1.8'
     description = '`df` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -165,6 +166,29 @@ def _process(proc_data):
     return proc_data
 
 
+def _long_filesystem_hash(header, line):
+    """Returns truncated hash and value of the filesystem field if it is too long for the column"""
+    filesystem_field = line.split()[0]
+
+    # get length of filesystem column
+    space_count = 0
+    for char in header[10:]:
+        if char == ' ':
+            space_count += 1
+            continue
+
+        break
+
+    filesystem_col_len = space_count + 9
+
+    # return the hash and value if the field data is longer than the column length
+    if len(filesystem_field) > filesystem_col_len:
+        truncated_hash = hashlib.sha256(filesystem_field.encode('utf-8')).hexdigest()[:filesystem_col_len]
+        return truncated_hash, filesystem_field
+
+    return None, None
+
+
 def parse(data, raw=False, quiet=False):
     """
     Main text parsing function
@@ -184,7 +208,9 @@ def parse(data, raw=False, quiet=False):
         jc.utils.compatibility(__name__, info.compatible)
 
     cleandata = data.splitlines()
+    fix_data = []
     raw_output = []
+    filesystem_map = {}
 
     if jc.utils.has_data(data):
 
@@ -193,8 +219,26 @@ def parse(data, raw=False, quiet=False):
         cleandata[0] = cleandata[0].replace('-', '_')
         cleandata[0] = cleandata[0].replace('mounted on', 'mounted_on')
 
+        # fix long filesystem data in some older versions of df
+        header = cleandata[0]
+        fix_data.append(header)
+        for line in cleandata[1:]:
+            field_hash, field_value = _long_filesystem_hash(header, line)
+            if field_hash:
+                filesystem_map.update({field_hash: field_value})
+                newline = line.replace(field_value, field_hash)
+                fix_data.append(newline)
+            else:
+                fix_data.append(line)
+
         # parse the data
-        raw_output = jc.parsers.universal.sparse_table_parse(cleandata)
+        raw_output = jc.parsers.universal.sparse_table_parse(fix_data)
+
+        # replace hash values with real values to fix long filesystem data in some older versions of df
+        for item in raw_output:
+            if 'filesystem' in item:
+                if item['filesystem'] in filesystem_map:
+                    item['filesystem'] = filesystem_map[item['filesystem']]
 
     if raw:
         return raw_output

jc/parsers/netstat.py

@@ -354,7 +354,7 @@ import jc.utils
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.10'
+    version = '1.11'
     description = '`netstat` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'

jc/parsers/netstat_linux.py

@@ -8,7 +8,7 @@ def normalize_headers(header):
     header = header.replace('foreign address', 'foreign_address')
     header = header.replace('pid/program name', 'program_name')
     header = header.replace('security context', 'security_context')
-    header = header.replace('i-node', 'inode')
+    header = header.replace('i-node', ' inode')
     header = header.replace('-', '_')
 
     return header

jc/parsers/ping.py

@@ -158,7 +158,7 @@ import jc.utils
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.5'
+    version = '1.6'
     description = '`ping` and `ping6` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -480,7 +480,7 @@ def _bsd_parse(data):
                         'round_trip_ms_min': split_line[0],
                         'round_trip_ms_avg': split_line[1],
                         'round_trip_ms_max': split_line[2],
-                        'round_trip_ms_stddev': split_line[3].replace(' ms', '')
+                        'round_trip_ms_stddev': split_line[3].replace(' ms', '') if len(split_line) == 4 else None
                     }
                 )
 

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2021-10-30 1.17.1 "JSON CLI output utility"
+.TH jc 1 2021-11-18 1.17.2 "JSON CLI output utility"
 .SH NAME
 jc \- JSONifies the output of many CLI tools and file-types
 .SH SYNOPSIS

setup.py

@@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
 
 setuptools.setup(
     name='jc',
-    version='1.17.1',
+    version='1.17.2',
     author='Kelly Brazil',
     author_email='kellyjonbrazil@gmail.com',
     description='Converts the output of popular command-line tools and file-types to JSON.',

tests/fixtures/alpine-linux-3.13/ping-hostname.json

@@ -0,0 +1 @@
+{"destination_ip":"142.250.125.102","data_bytes":56,"pattern":null,"destination":"google.com","packets_transmitted":4,"packets_received":4,"packet_loss_percent":0.0,"duplicates":0,"round_trip_ms_min":1.281,"round_trip_ms_avg":1.345,"round_trip_ms_max":1.408,"round_trip_ms_stddev":null,"responses":[{"type":"reply","bytes":64,"response_ip":"142.250.125.102","icmp_seq":0,"ttl":42,"time_ms":1.331,"duplicate":false},{"type":"reply","bytes":64,"response_ip":"142.250.125.102","icmp_seq":1,"ttl":42,"time_ms":1.281,"duplicate":false},{"type":"reply","bytes":64,"response_ip":"142.250.125.102","icmp_seq":2,"ttl":42,"time_ms":1.408,"duplicate":false},{"type":"reply","bytes":64,"response_ip":"142.250.125.102","icmp_seq":3,"ttl":42,"time_ms":1.36,"duplicate":false}]}

tests/fixtures/alpine-linux-3.13/ping-hostname.out

@@ -0,0 +1,9 @@
+PING google.com (142.250.125.102): 56 data bytes
+64 bytes from 142.250.125.102: seq=0 ttl=42 time=1.331 ms
+64 bytes from 142.250.125.102: seq=1 ttl=42 time=1.281 ms
+64 bytes from 142.250.125.102: seq=2 ttl=42 time=1.408 ms
+64 bytes from 142.250.125.102: seq=3 ttl=42 time=1.360 ms
+
+--- google.com ping statistics ---
+4 packets transmitted, 4 packets received, 0% packet loss
+round-trip min/avg/max = 1.281/1.345/1.408 ms

tests/fixtures/alpine-linux-3.13/ping-ip.json

@@ -0,0 +1 @@
+{"destination_ip":"8.8.8.8","data_bytes":56,"pattern":null,"destination":"8.8.8.8","packets_transmitted":1,"packets_received":1,"packet_loss_percent":0.0,"duplicates":0,"round_trip_ms_min":1.637,"round_trip_ms_avg":1.637,"round_trip_ms_max":1.637,"round_trip_ms_stddev":null,"responses":[{"type":"reply","bytes":64,"response_ip":"8.8.8.8","icmp_seq":0,"ttl":42,"time_ms":1.637,"duplicate":false}]}

tests/fixtures/alpine-linux-3.13/ping-ip.out

@@ -0,0 +1,6 @@
+PING 8.8.8.8 (8.8.8.8): 56 data bytes
+64 bytes from 8.8.8.8: seq=0 ttl=42 time=1.637 ms
+
+--- 8.8.8.8 ping statistics ---
+1 packets transmitted, 1 packets received, 0% packet loss
+round-trip min/avg/max = 1.637/1.637/1.637 ms

tests/fixtures/generic/df-long-filesystem.json

@@ -0,0 +1 @@
+[{"filesystem":"/dev/mapper/VolGroup00-LogVol00","type":"ext3","1024_blocks":6030784,"used":1147932,"available":4571556,"mounted_on":"/","capacity_percent":21},{"filesystem":"proc","type":"proc","1024_blocks":0,"used":0,"available":0,"mounted_on":"/proc","capacity_percent":null},{"filesystem":"sysfs","type":"sysfs","1024_blocks":0,"used":0,"available":0,"mounted_on":"/sys","capacity_percent":null}]

tests/fixtures/generic/df-long-filesystem.out

@@ -0,0 +1,4 @@
+Filesystem    Type 1024-blocks      Used Available Capacity Mounted on
+/dev/mapper/VolGroup00-LogVol00 ext3     6030784   1147932   4571556      21% /
+proc          proc           0         0         0       -  /proc
+sysfs        sysfs           0         0         0       -  /sys

tests/fixtures/generic/netstat-old.json

@@ -0,0 +1 @@
+[{"proto":"tcp","recv_q":0,"send_q":0,"local_address":"0.0.0.0","foreign_address":"0.0.0.0","state":"LISTEN","program_name":"systemd","kind":"network","pid":1,"local_port":"111","foreign_port":"*","transport_protocol":"tcp","network_protocol":"ipv4","local_port_num":111},{"proto":"udp","recv_q":0,"send_q":0,"local_address":"200.4.30.128","foreign_address":"0.0.0.0","state":null,"program_name":"NetworkManager","kind":"network","pid":903,"local_port":"68","foreign_port":"*","transport_protocol":"udp","network_protocol":"ipv4","local_port_num":68},{"proto":"unix","refcnt":2,"flags":"ACC","type":"STREAM","state":"LISTENING","inode":18438,"program_name":"systemd","path":"/run/lvm/lvmpolld.socket","kind":"socket","pid":1},{"proto":"unix","refcnt":2,"flags":null,"type":"DGRAM","state":null,"inode":23569,"program_name":"chronyd","path":"/var/run/chrony/chronyd.sock","kind":"socket","pid":871},{"proto":"unix","refcnt":2,"flags":"ACC","type":"STREAM","state":"LISTENING","inode":23584,"program_name":"gssproxy","path":"/run/gssproxy.sock","kind":"socket","pid":872},{"proto":"unix","refcnt":2,"flags":"ACC","type":"STREAM","state":"LISTENING","inode":18511,"program_name":"systemd","path":"/run/rpcbind.sock","kind":"socket","pid":1},{"proto":"unix","refcnt":2,"flags":"ACC","type":"SEQPACKET","state":"LISTENING","inode":18521,"program_name":"systemd","path":"/run/udev/control","kind":"socket","pid":1},{"proto":"unix","refcnt":3,"flags":null,"type":"DGRAM","state":null,"inode":11098,"program_name":"systemd","path":"/run/systemd/notify","kind":"socket","pid":1},{"proto":"unix","refcnt":2,"flags":null,"type":"DGRAM","state":null,"inode":11100,"program_name":"systemd","path":"/run/systemd/cgroups-agent","kind":"socket","pid":1},{"proto":"unix","refcnt":2,"flags":"ACC","type":"STREAM","state":"LISTENING","inode":24236,"program_name":"sssd","path":"/var/lib/sss/pipes/private/sbus-monitor","kind":"socket","pid":863}]

tests/fixtures/generic/netstat-old.out

@@ -0,0 +1,14 @@
+Active Internet connections (servers and established)
+Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
+tcp        0      0 0.0.0.0:111             0.0.0.0:*               LISTEN      1/systemd
+udp        0      0 200.4.30.128:68         0.0.0.0:*                           903/NetworkManager
+Active UNIX domain sockets (servers and established)
+Proto RefCnt Flags       Type       State         I-Node PID/Program name    Path
+unix  2      [ ACC ]     STREAM     LISTENING      18438 1/systemd           /run/lvm/lvmpolld.socket
+unix  2      [ ]         DGRAM                     23569 871/chronyd         /var/run/chrony/chronyd.sock
+unix  2      [ ACC ]     STREAM     LISTENING      23584 872/gssproxy        /run/gssproxy.sock
+unix  2      [ ACC ]     STREAM     LISTENING      18511 1/systemd           /run/rpcbind.sock
+unix  2      [ ACC ]     SEQPACKET  LISTENING      18521 1/systemd           /run/udev/control
+unix  3      [ ]         DGRAM                     11098 1/systemd           /run/systemd/notify
+unix  2      [ ]         DGRAM                     11100 1/systemd           /run/systemd/cgroups-agent
+unix  2      [ ACC ]     STREAM     LISTENING      24236 863/sssd            /var/lib/sss/pipes/private/sbus-monitor

tests/test_df.py

@@ -34,6 +34,9 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/df-h.out'), 'r', encoding='utf-8') as f:
             self.osx_10_14_6_df_h = f.read()
 
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/df-long-filesystem.out'), 'r', encoding='utf-8') as f:
+            self.generic_df_long_filesystem = f.read()
+
         # output
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/df.json'), 'r', encoding='utf-8') as f:
             self.centos_7_7_df_json = json.loads(f.read())
@@ -59,6 +62,9 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/df-h.json'), 'r', encoding='utf-8') as f:
             self.osx_10_14_6_df_h_json = json.loads(f.read())
 
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/df-long-filesystem.json'), 'r', encoding='utf-8') as f:
+            self.generic_df_long_filesystem_json = json.loads(f.read())
+
     def test_df_nodata(self):
         """
         Test plain 'df' with no data
@@ -113,6 +119,12 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(jc.parsers.df.parse(self.osx_10_14_6_df_h, quiet=True), self.osx_10_14_6_df_h_json)
 
+    def test_df_long_filesystem(self):
+        """
+        Test older version of 'df' with long filesystem data
+        """
+        self.assertEqual(jc.parsers.df.parse(self.generic_df_long_filesystem, quiet=True), self.generic_df_long_filesystem_json)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_netstat.py

@@ -74,6 +74,9 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/freebsd12/netstat-aT.out'), 'r', encoding='utf-8') as f:
             self.freebsd12_netstat_aT = f.read()
 
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/netstat-old.out'), 'r', encoding='utf-8') as f:
+            self.generic_netstat_old = f.read()
+
         # netstat -r
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/netstat-r.out'), 'r', encoding='utf-8') as f:
             self.centos_7_7_netstat_r = f.read()
@@ -183,6 +186,9 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/freebsd12/netstat-an.json'), 'r', encoding='utf-8') as f:
             self.freebsd12_netstat_an_json = json.loads(f.read())
 
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/netstat-old.json'), 'r', encoding='utf-8') as f:
+            self.generic_netstat_old_json = json.loads(f.read())
+
         # netsat -r
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/netstat-r.json'), 'r', encoding='utf-8') as f:
             self.centos_7_7_netstat_r_json = json.loads(f.read())
@@ -353,6 +359,12 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(jc.parsers.netstat.parse(self.freebsd12_netstat_an, quiet=True), self.freebsd12_netstat_an_json)
 
+    def test_netstat_old_generic(self):
+        """
+        Test 'netstat' with older version of netstat on linux
+        """
+        self.assertEqual(jc.parsers.netstat.parse(self.generic_netstat_old, quiet=True), self.generic_netstat_old_json)
+
     def test_netstat_r_centos_7_7(self):
         """
         Test 'netstat -r' on Centos 7.7

tests/test_ping.py

@@ -203,6 +203,13 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/pi/ping-ip-O-D.out'), 'r', encoding='utf-8') as f:
             self.pi_ping_ip_O_D = f.read()
 
+        # alpine-linux
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/alpine-linux-3.13/ping-ip.out'), 'r', encoding='utf-8') as f:
+            self.alpine_linux_3_13_ping_ip = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/alpine-linux-3.13/ping-hostname.out'), 'r', encoding='utf-8') as f:
+            self.alpine_linux_3_13_ping_hostname = f.read()
+
         # output
 
         # centos
@@ -397,6 +404,14 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/pi/ping-ip-O-D.json'), 'r', encoding='utf-8') as f:
             self.pi_ping_ip_O_D_json = json.loads(f.read())
 
+        # alpine-linux
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/alpine-linux-3.13/ping-ip.json'), 'r', encoding='utf-8') as f:
+            self.alpine_linux_3_13_ping_ip_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/alpine-linux-3.13/ping-hostname.json'), 'r', encoding='utf-8') as f:
+            self.alpine_linux_3_13_ping_hostname_json = json.loads(f.read())
+
+
     def test_ping_nodata(self):
         """
         Test 'ping' with no data
@@ -775,6 +790,18 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(jc.parsers.ping.parse(self.pi_ping_ip_O_D, quiet=True), self.pi_ping_ip_O_D_json)
 
+    def test_ping_ip_alpine_linux(self):
+        """
+        Test 'ping <ip> -O' on alpine linux
+        """
+        self.assertEqual(jc.parsers.ping.parse(self.alpine_linux_3_13_ping_ip, quiet=True), self.alpine_linux_3_13_ping_ip_json)
+
+    def test_ping_hostname_alpine_linux(self):
+        """
+        Test 'ping <hostname>' on alpine linux
+        """
+        self.assertEqual(jc.parsers.ping.parse(self.alpine_linux_3_13_ping_hostname, quiet=True), self.alpine_linux_3_13_ping_hostname_json)
+
 
 if __name__ == '__main__':
     unittest.main()