python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-19 00:17:51 +02:00
Code Issues Releases Activity

Merge pull request #180 from kellyjonbrazil/dev

Browse Source 
Dev v1.17.2
This commit is contained in:
Kelly Brazil
2021-11-18 06:52:01 -10:00
committed by GitHub
parent b59e38cfd2 eb888dcbbc
commit 597d39c28e
22 changed files with 149 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG
View File

@ -1,5 +1,10 @@
jc changelog
20211117 v1.17.2
- Fix ping parser to add Alpine linux support
- Fix netstat parser for older versions of netstat on linux
- Fix df parser for cases where the filesystem field overflows the column length
20211030 v1.17.1
- Fix file parser for gzip files
- Fix uname parser for cases where the 'processor' and/or 'hardware_platform' fields are missing on linux

2
docs/parsers/df.md
View File

@ -122,4 +122,4 @@ Returns:
## Parser Information
Compatibility: linux, darwin, freebsd
Version 1.7 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.8 by Kelly Brazil (kellyjonbrazil@gmail.com)

2
docs/parsers/netstat.md
View File

@ -379,4 +379,4 @@ Returns:
## Parser Information
Compatibility: linux, darwin, freebsd
Version 1.10 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.11 by Kelly Brazil (kellyjonbrazil@gmail.com)

2
docs/parsers/ping.md
View File

@ -181,4 +181,4 @@ Returns:
## Parser Information
Compatibility: linux, darwin, freebsd
Version 1.5 by Kelly Brazil (kellyjonbrazil@gmail.com)
Version 1.6 by Kelly Brazil (kellyjonbrazil@gmail.com)

2
jc/__init__.py
View File

@ -73,4 +73,4 @@ Module Example:
"""
name = 'jc'
__version__ = '1.17.1'
__version__ = '1.17.2'

48
jc/parsers/df.py
View File

@ -92,13 +92,14 @@ Examples:
...
]
"""
import hashlib
import jc.utils
import jc.parsers.universal
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.7'
version = '1.8'
description = '`df` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -165,6 +166,29 @@ def _process(proc_data):
return proc_data
def _long_filesystem_hash(header, line):
"""Returns truncated hash and value of the filesystem field if it is too long for the column"""
filesystem_field = line.split()[0]
# get length of filesystem column
space_count = 0
for char in header[10:]:
if char == ' ':
space_count += 1
continue
break
filesystem_col_len = space_count + 9
# return the hash and value if the field data is longer than the column length
if len(filesystem_field) > filesystem_col_len:
truncated_hash = hashlib.sha256(filesystem_field.encode('utf-8')).hexdigest()[:filesystem_col_len]
return truncated_hash, filesystem_field
return None, None
def parse(data, raw=False, quiet=False):
"""
Main text parsing function
@ -184,7 +208,9 @@ def parse(data, raw=False, quiet=False):
jc.utils.compatibility(__name__, info.compatible)
cleandata = data.splitlines()
fix_data = []
raw_output = []
filesystem_map = {}
if jc.utils.has_data(data):
@ -193,8 +219,26 @@ def parse(data, raw=False, quiet=False):
cleandata[0] = cleandata[0].replace('-', '_')
cleandata[0] = cleandata[0].replace('mounted on', 'mounted_on')
# fix long filesystem data in some older versions of df
header = cleandata[0]
fix_data.append(header)
for line in cleandata[1:]:
field_hash, field_value = _long_filesystem_hash(header, line)
if field_hash:
filesystem_map.update({field_hash: field_value})
newline = line.replace(field_value, field_hash)
fix_data.append(newline)
else:
fix_data.append(line)
# parse the data
raw_output = jc.parsers.universal.sparse_table_parse(cleandata)
raw_output = jc.parsers.universal.sparse_table_parse(fix_data)
# replace hash values with real values to fix long filesystem data in some older versions of df
for item in raw_output:
if 'filesystem' in item:
if item['filesystem'] in filesystem_map:
item['filesystem'] = filesystem_map[item['filesystem']]
if raw:
return raw_output

2
jc/parsers/netstat.py
View File

@ -354,7 +354,7 @@ import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.10'
version = '1.11'
description = '`netstat` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'

2
jc/parsers/netstat_linux.py
View File

@ -8,7 +8,7 @@ def normalize_headers(header):
header = header.replace('foreign address', 'foreign_address')
header = header.replace('pid/program name', 'program_name')
header = header.replace('security context', 'security_context')
header = header.replace('i-node', 'inode')
header = header.replace('i-node', ' inode')
header = header.replace('-', '_')
return header

4
jc/parsers/ping.py
View File

@ -158,7 +158,7 @@ import jc.utils
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '1.5'
version = '1.6'
description = '`ping` and `ping6` command parser'
author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com'
@ -480,7 +480,7 @@ def _bsd_parse(data):
'round_trip_ms_min': split_line[0],
'round_trip_ms_avg': split_line[1],
'round_trip_ms_max': split_line[2],
'round_trip_ms_stddev': split_line[3].replace(' ms', '')
'round_trip_ms_stddev': split_line[3].replace(' ms', '') if len(split_line) == 4 else None
}
)

2
man/jc.1
View File

@ -1,4 +1,4 @@
.TH jc 1 2021-10-30 1.17.1 "JSON CLI output utility"
.TH jc 1 2021-11-18 1.17.2 "JSON CLI output utility"
.SH NAME
jc \- JSONifies the output of many CLI tools and file-types
.SH SYNOPSIS

2
setup.py
View File

@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
setuptools.setup(
name='jc',
version='1.17.1',
version='1.17.2',
author='Kelly Brazil',
author_email='kellyjonbrazil@gmail.com',
description='Converts the output of popular command-line tools and file-types to JSON.',

1
tests/fixtures/alpine-linux-3.13/ping-hostname.json vendored Normal file
View File

@ -0,0 +1 @@
{"destination_ip":"142.250.125.102","data_bytes":56,"pattern":null,"destination":"google.com","packets_transmitted":4,"packets_received":4,"packet_loss_percent":0.0,"duplicates":0,"round_trip_ms_min":1.281,"round_trip_ms_avg":1.345,"round_trip_ms_max":1.408,"round_trip_ms_stddev":null,"responses":[{"type":"reply","bytes":64,"response_ip":"142.250.125.102","icmp_seq":0,"ttl":42,"time_ms":1.331,"duplicate":false},{"type":"reply","bytes":64,"response_ip":"142.250.125.102","icmp_seq":1,"ttl":42,"time_ms":1.281,"duplicate":false},{"type":"reply","bytes":64,"response_ip":"142.250.125.102","icmp_seq":2,"ttl":42,"time_ms":1.408,"duplicate":false},{"type":"reply","bytes":64,"response_ip":"142.250.125.102","icmp_seq":3,"ttl":42,"time_ms":1.36,"duplicate":false}]}

9
tests/fixtures/alpine-linux-3.13/ping-hostname.out vendored Normal file
View File

@ -0,0 +1,9 @@
PING google.com (142.250.125.102): 56 data bytes
64 bytes from 142.250.125.102: seq=0 ttl=42 time=1.331 ms
64 bytes from 142.250.125.102: seq=1 ttl=42 time=1.281 ms
64 bytes from 142.250.125.102: seq=2 ttl=42 time=1.408 ms
64 bytes from 142.250.125.102: seq=3 ttl=42 time=1.360 ms
--- google.com ping statistics ---
4 packets transmitted, 4 packets received, 0% packet loss
round-trip min/avg/max = 1.281/1.345/1.408 ms

1
tests/fixtures/alpine-linux-3.13/ping-ip.json vendored Normal file
View File

@ -0,0 +1 @@
{"destination_ip":"8.8.8.8","data_bytes":56,"pattern":null,"destination":"8.8.8.8","packets_transmitted":1,"packets_received":1,"packet_loss_percent":0.0,"duplicates":0,"round_trip_ms_min":1.637,"round_trip_ms_avg":1.637,"round_trip_ms_max":1.637,"round_trip_ms_stddev":null,"responses":[{"type":"reply","bytes":64,"response_ip":"8.8.8.8","icmp_seq":0,"ttl":42,"time_ms":1.637,"duplicate":false}]}

6
tests/fixtures/alpine-linux-3.13/ping-ip.out vendored Normal file
View File

@ -0,0 +1,6 @@
PING 8.8.8.8 (8.8.8.8): 56 data bytes
64 bytes from 8.8.8.8: seq=0 ttl=42 time=1.637 ms
--- 8.8.8.8 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 1.637/1.637/1.637 ms

1
tests/fixtures/generic/df-long-filesystem.json vendored Normal file
View File

@ -0,0 +1 @@
[{"filesystem":"/dev/mapper/VolGroup00-LogVol00","type":"ext3","1024_blocks":6030784,"used":1147932,"available":4571556,"mounted_on":"/","capacity_percent":21},{"filesystem":"proc","type":"proc","1024_blocks":0,"used":0,"available":0,"mounted_on":"/proc","capacity_percent":null},{"filesystem":"sysfs","type":"sysfs","1024_blocks":0,"used":0,"available":0,"mounted_on":"/sys","capacity_percent":null}]

4
tests/fixtures/generic/df-long-filesystem.out vendored Normal file
View File

@ -0,0 +1,4 @@
Filesystem Type 1024-blocks Used Available Capacity Mounted on
/dev/mapper/VolGroup00-LogVol00 ext3 6030784 1147932 4571556 21% /
proc proc 0 0 0 - /proc
sysfs sysfs 0 0 0 - /sys

1
tests/fixtures/generic/netstat-old.json vendored Normal file
View File

@ -0,0 +1 @@
[{"proto":"tcp","recv_q":0,"send_q":0,"local_address":"0.0.0.0","foreign_address":"0.0.0.0","state":"LISTEN","program_name":"systemd","kind":"network","pid":1,"local_port":"111","foreign_port":"*","transport_protocol":"tcp","network_protocol":"ipv4","local_port_num":111},{"proto":"udp","recv_q":0,"send_q":0,"local_address":"200.4.30.128","foreign_address":"0.0.0.0","state":null,"program_name":"NetworkManager","kind":"network","pid":903,"local_port":"68","foreign_port":"*","transport_protocol":"udp","network_protocol":"ipv4","local_port_num":68},{"proto":"unix","refcnt":2,"flags":"ACC","type":"STREAM","state":"LISTENING","inode":18438,"program_name":"systemd","path":"/run/lvm/lvmpolld.socket","kind":"socket","pid":1},{"proto":"unix","refcnt":2,"flags":null,"type":"DGRAM","state":null,"inode":23569,"program_name":"chronyd","path":"/var/run/chrony/chronyd.sock","kind":"socket","pid":871},{"proto":"unix","refcnt":2,"flags":"ACC","type":"STREAM","state":"LISTENING","inode":23584,"program_name":"gssproxy","path":"/run/gssproxy.sock","kind":"socket","pid":872},{"proto":"unix","refcnt":2,"flags":"ACC","type":"STREAM","state":"LISTENING","inode":18511,"program_name":"systemd","path":"/run/rpcbind.sock","kind":"socket","pid":1},{"proto":"unix","refcnt":2,"flags":"ACC","type":"SEQPACKET","state":"LISTENING","inode":18521,"program_name":"systemd","path":"/run/udev/control","kind":"socket","pid":1},{"proto":"unix","refcnt":3,"flags":null,"type":"DGRAM","state":null,"inode":11098,"program_name":"systemd","path":"/run/systemd/notify","kind":"socket","pid":1},{"proto":"unix","refcnt":2,"flags":null,"type":"DGRAM","state":null,"inode":11100,"program_name":"systemd","path":"/run/systemd/cgroups-agent","kind":"socket","pid":1},{"proto":"unix","refcnt":2,"flags":"ACC","type":"STREAM","state":"LISTENING","inode":24236,"program_name":"sssd","path":"/var/lib/sss/pipes/private/sbus-monitor","kind":"socket","pid":863}]

14
tests/fixtures/generic/netstat-old.out vendored Normal file
View File

@ -0,0 +1,14 @@
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 1/systemd
udp 0 0 200.4.30.128:68 0.0.0.0:* 903/NetworkManager
Active UNIX domain sockets (servers and established)
Proto RefCnt Flags Type State I-Node PID/Program name Path
unix 2 [ ACC ] STREAM LISTENING 18438 1/systemd /run/lvm/lvmpolld.socket
unix 2 [ ] DGRAM 23569 871/chronyd /var/run/chrony/chronyd.sock
unix 2 [ ACC ] STREAM LISTENING 23584 872/gssproxy /run/gssproxy.sock
unix 2 [ ACC ] STREAM LISTENING 18511 1/systemd /run/rpcbind.sock
unix 2 [ ACC ] SEQPACKET LISTENING 18521 1/systemd /run/udev/control
unix 3 [ ] DGRAM 11098 1/systemd /run/systemd/notify
unix 2 [ ] DGRAM 11100 1/systemd /run/systemd/cgroups-agent
unix 2 [ ACC ] STREAM LISTENING 24236 863/sssd /var/lib/sss/pipes/private/sbus-monitor

12
tests/test_df.py
View File

@ -34,6 +34,9 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/df-h.out'), 'r', encoding='utf-8') as f:
self.osx_10_14_6_df_h = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/df-long-filesystem.out'), 'r', encoding='utf-8') as f:
self.generic_df_long_filesystem = f.read()
# output
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/df.json'), 'r', encoding='utf-8') as f:
self.centos_7_7_df_json = json.loads(f.read())
@ -59,6 +62,9 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/df-h.json'), 'r', encoding='utf-8') as f:
self.osx_10_14_6_df_h_json = json.loads(f.read())
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/df-long-filesystem.json'), 'r', encoding='utf-8') as f:
self.generic_df_long_filesystem_json = json.loads(f.read())
def test_df_nodata(self):
"""
Test plain 'df' with no data
@ -113,6 +119,12 @@ class MyTests(unittest.TestCase):
"""
self.assertEqual(jc.parsers.df.parse(self.osx_10_14_6_df_h, quiet=True), self.osx_10_14_6_df_h_json)
def test_df_long_filesystem(self):
"""
Test older version of 'df' with long filesystem data
"""
self.assertEqual(jc.parsers.df.parse(self.generic_df_long_filesystem, quiet=True), self.generic_df_long_filesystem_json)
if __name__ == '__main__':
unittest.main()

12
tests/test_netstat.py
View File

@ -74,6 +74,9 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/freebsd12/netstat-aT.out'), 'r', encoding='utf-8') as f:
self.freebsd12_netstat_aT = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/netstat-old.out'), 'r', encoding='utf-8') as f:
self.generic_netstat_old = f.read()
# netstat -r
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/netstat-r.out'), 'r', encoding='utf-8') as f:
self.centos_7_7_netstat_r = f.read()
@ -183,6 +186,9 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/freebsd12/netstat-an.json'), 'r', encoding='utf-8') as f:
self.freebsd12_netstat_an_json = json.loads(f.read())
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/netstat-old.json'), 'r', encoding='utf-8') as f:
self.generic_netstat_old_json = json.loads(f.read())
# netsat -r
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/netstat-r.json'), 'r', encoding='utf-8') as f:
self.centos_7_7_netstat_r_json = json.loads(f.read())
@ -353,6 +359,12 @@ class MyTests(unittest.TestCase):
"""
self.assertEqual(jc.parsers.netstat.parse(self.freebsd12_netstat_an, quiet=True), self.freebsd12_netstat_an_json)
def test_netstat_old_generic(self):
"""
Test 'netstat' with older version of netstat on linux
"""
self.assertEqual(jc.parsers.netstat.parse(self.generic_netstat_old, quiet=True), self.generic_netstat_old_json)
def test_netstat_r_centos_7_7(self):
"""
Test 'netstat -r' on Centos 7.7

27
tests/test_ping.py
View File

@ -203,6 +203,13 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/pi/ping-ip-O-D.out'), 'r', encoding='utf-8') as f:
self.pi_ping_ip_O_D = f.read()
# alpine-linux
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/alpine-linux-3.13/ping-ip.out'), 'r', encoding='utf-8') as f:
self.alpine_linux_3_13_ping_ip = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/alpine-linux-3.13/ping-hostname.out'), 'r', encoding='utf-8') as f:
self.alpine_linux_3_13_ping_hostname = f.read()
# output
# centos
@ -397,6 +404,14 @@ class MyTests(unittest.TestCase):
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/pi/ping-ip-O-D.json'), 'r', encoding='utf-8') as f:
self.pi_ping_ip_O_D_json = json.loads(f.read())
# alpine-linux
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/alpine-linux-3.13/ping-ip.json'), 'r', encoding='utf-8') as f:
self.alpine_linux_3_13_ping_ip_json = json.loads(f.read())
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/alpine-linux-3.13/ping-hostname.json'), 'r', encoding='utf-8') as f:
self.alpine_linux_3_13_ping_hostname_json = json.loads(f.read())
def test_ping_nodata(self):
"""
Test 'ping' with no data
@ -775,6 +790,18 @@ class MyTests(unittest.TestCase):
"""
self.assertEqual(jc.parsers.ping.parse(self.pi_ping_ip_O_D, quiet=True), self.pi_ping_ip_O_D_json)
def test_ping_ip_alpine_linux(self):
"""
Test 'ping <ip> -O' on alpine linux
"""
self.assertEqual(jc.parsers.ping.parse(self.alpine_linux_3_13_ping_ip, quiet=True), self.alpine_linux_3_13_ping_ip_json)
def test_ping_hostname_alpine_linux(self):
"""
Test 'ping <hostname>' on alpine linux
"""
self.assertEqual(jc.parsers.ping.parse(self.alpine_linux_3_13_ping_hostname, quiet=True), self.alpine_linux_3_13_ping_hostname_json)
if __name__ == '__main__':
unittest.main()