python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-17 00:07:37 +02:00
Code Issues Releases Activity

change schema to a list of dictionaries to support ufw app info all use case

Browse Source
This commit is contained in:
Kelly Brazil
2021-04-26 10:00:44 -07:00
parent 8d2d3db3fa
commit a683f68003
9 changed files with 278 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
EXAMPLES.md
View File

@ -3199,39 +3199,41 @@ ufw status verbose | jc --ufw -p # or jc -p ufw status verbose
ufw app info MSN | jc --ufw-appinfo -p # or: jc -p ufw app info MSN ufw app info MSN | jc --ufw-appinfo -p # or: jc -p ufw app info MSN
``` ```
```json ```json
{ [
"profile": "MSN", {
"title": "MSN Chat", "profile": "MSN",
"description": "MSN chat protocol (with file transfer and voice)", "title": "MSN Chat",
"tcp_list": [ "description": "MSN chat protocol (with file transfer and voice)",
1863, "tcp_list": [
6901 1863,
], 6901
"udp_list": [ ],
1863, "udp_list": [
6901 1863,
], 6901
"tcp_ranges": [ ],
{ "tcp_ranges": [
"start": 6891, {
"end": 6900 "start": 6891,
} "end": 6900
], }
"normalized_tcp_list": [ ],
1863, "normalized_tcp_list": [
6901 1863,
], 6901
"normalized_tcp_ranges": [ ],
{ "normalized_tcp_ranges": [
"start": 6891, {
"end": 6900 "start": 6891,
} "end": 6900
], }
"normalized_udp_list": [ ],
1863, "normalized_udp_list": [
6901 1863,
] 6901
} ]
}
]
``` ```
### uname -a ### uname -a
```bash ```bash

331
jc/parsers/ufw_appinfo.py
View File

@ -1,5 +1,7 @@
"""jc - JSON CLI output utility `ufw app info [application]` command output parser """jc - JSON CLI output utility `ufw app info [application]` command output parser
Supports individual apps via `ufw app info [application]` and all apps list via `ufw app info all`.
Because `ufw` application definitions allow overlapping ports and port ranges, this parser preserves that behavior, but also provides `normalized` lists and ranges that remove duplicate ports and merge overlapping ranges. Because `ufw` application definitions allow overlapping ports and port ranges, this parser preserves that behavior, but also provides `normalized` lists and ranges that remove duplicate ports and merge overlapping ranges.
Usage (cli): Usage (cli):
@ -17,105 +19,111 @@ Usage (module):
Schema: Schema:
{ [
"profile": string, {
"title": string, "profile": string,
"description": string, "title": string,
"tcp_list": [ "description": string,
integer "tcp_list": [
], integer
"tcp_ranges": [ ],
{ "tcp_ranges": [
"start": integer, # 'any' is converted to start/end: 0/65535 {
"end": integer "start": integer, # 'any' is converted to start/end: 0/65535
} "end": integer
], }
"udp_list": [ ],
integer "udp_list": [
], integer
"udp_ranges": [ ],
{ "udp_ranges": [
"start": integer, # 'any' is converted to start/end: 0/65535 {
"end": integer "start": integer, # 'any' is converted to start/end: 0/65535
} "end": integer
], }
"normalized_tcp_list": [ ],
integers # duplicates and overlapping are removed "normalized_tcp_list": [
], integers # duplicates and overlapping are removed
"normalized_tcp_ranges": [ ],
{ "normalized_tcp_ranges": [
"start": integer, # 'any' is converted to start/end: 0/65535 {
"end": integers # overlapping are merged "start": integer, # 'any' is converted to start/end: 0/65535
} "end": integers # overlapping are merged
], }
"normalized_udp_list": [ ],
integers # duplicates and overlapping are removed "normalized_udp_list": [
], integers # duplicates and overlapping are removed
"normalized_udp_ranges": [ ],
{ "normalized_udp_ranges": [
"start": integer, # 'any' is converted to start/end: 0/65535 {
"end": integers # overlapping are merged "start": integer, # 'any' is converted to start/end: 0/65535
} "end": integers # overlapping are merged
] }
} ]
}
]
Examples: Examples:
$ ufw app info MSN | jc --ufw-appinfo -p $ ufw app info MSN | jc --ufw-appinfo -p
{ [
"profile": "MSN", {
"title": "MSN Chat", "profile": "MSN",
"description": "MSN chat protocol (with file transfer and voice)", "title": "MSN Chat",
"tcp_list": [ "description": "MSN chat protocol (with file transfer and voice)",
1863, "tcp_list": [
6901 1863,
], 6901
"udp_list": [ ],
1863, "udp_list": [
6901 1863,
], 6901
"tcp_ranges": [ ],
{ "tcp_ranges": [
"start": 6891, {
"end": 6900 "start": 6891,
} "end": 6900
], }
"normalized_tcp_list": [ ],
1863, "normalized_tcp_list": [
6901 1863,
], 6901
"normalized_tcp_ranges": [ ],
{ "normalized_tcp_ranges": [
"start": 6891, {
"end": 6900 "start": 6891,
} "end": 6900
], }
"normalized_udp_list": [ ],
1863, "normalized_udp_list": [
6901 1863,
] 6901
} ]
}
]
$ ufw app info MSN | jc --ufw-appinfo -p -r $ ufw app info MSN | jc --ufw-appinfo -p -r
{ [
"profile": "MSN", {
"title": "MSN Chat", "profile": "MSN",
"description": "MSN chat protocol (with file transfer and voice)", "title": "MSN Chat",
"tcp_list": [ "description": "MSN chat protocol (with file transfer and voice)",
"1863", "tcp_list": [
"6901" "1863",
], "6901"
"udp_list": [ ],
"1863", "udp_list": [
"6901" "1863",
], "6901"
"tcp_ranges": [ ],
{ "tcp_ranges": [
"start": "6891", {
"end": "6900" "start": "6891",
} "end": "6900"
] }
} ]
}
]
""" """
import jc.utils import jc.utils
@ -143,56 +151,57 @@ def _process(proc_data):
Returns: Returns:
Dictionary. Structured to conform to the schema. List of Dictionaries. Structured to conform to the schema.
""" """
# convert to ints for profile in proc_data:
int_list = ['start', 'end'] # convert to ints
int_list = ['start', 'end']
if 'tcp_list' in proc_data: if 'tcp_list' in profile:
proc_data['tcp_list'] = [int(p) for p in proc_data['tcp_list']] profile['tcp_list'] = [int(p) for p in profile['tcp_list']]
if 'udp_list' in proc_data: if 'udp_list' in profile:
proc_data['udp_list'] = [int(p) for p in proc_data['udp_list']] profile['udp_list'] = [int(p) for p in profile['udp_list']]
for protocol in ['tcp', 'udp']: for protocol in ['tcp', 'udp']:
if protocol + '_ranges' in proc_data: if protocol + '_ranges' in profile:
for i, item in enumerate(proc_data[protocol + '_ranges']): for i, item in enumerate(profile[protocol + '_ranges']):
for key in item: for key in item:
if key in int_list: if key in int_list:
proc_data[protocol + '_ranges'][i][key] = int(proc_data[protocol + '_ranges'][i][key]) profile[protocol + '_ranges'][i][key] = int(profile[protocol + '_ranges'][i][key])
# create normalized port lists and port ranges (remove duplicates and merge ranges) # create normalized port lists and port ranges (remove duplicates and merge ranges)
# dump ranges into a set of 0 - 65535 # dump ranges into a set of 0 - 65535
# if items in the port list are in the set, then remove them # if items in the port list are in the set, then remove them
# iterate through the set to find gaps and create new ranges based on them # iterate through the set to find gaps and create new ranges based on them
for protocol in ['tcp', 'udp']: for protocol in ['tcp', 'udp']:
port_set = set() port_set = set()
if protocol + '_ranges' in proc_data: if protocol + '_ranges' in profile:
for item in proc_data[protocol + '_ranges']: for item in profile[protocol + '_ranges']:
port_set.update(range(item['start'], item['end'] + 1)) port_set.update(range(item['start'], item['end'] + 1))
if protocol + '_list' in proc_data: if protocol + '_list' in profile:
new_port_list = sorted(set([p for p in proc_data[protocol + '_list'] if p not in port_set])) new_port_list = sorted(set([p for p in profile[protocol + '_list'] if p not in port_set]))
if new_port_list: if new_port_list:
proc_data['normalized_' + protocol + '_list'] = new_port_list profile['normalized_' + protocol + '_list'] = new_port_list
new_port_ranges = [] new_port_ranges = []
state = 'findstart' # 'findstart' or 'findend' state = 'findstart' # 'findstart' or 'findend'
for port in range(0, 65535 + 2): for port in range(0, 65535 + 2):
if state == 'findstart': if state == 'findstart':
port_range_obj = {} port_range_obj = {}
if port in port_set: if port in port_set:
port_range_obj['start'] = port port_range_obj['start'] = port
state = 'findend' state = 'findend'
continue continue
if state == 'findend': if state == 'findend':
if port not in port_set: if port not in port_set:
port_range_obj['end'] = port - 1 port_range_obj['end'] = port - 1
new_port_ranges.append(port_range_obj) new_port_ranges.append(port_range_obj)
state = 'findstart' state = 'findstart'
if new_port_ranges: if new_port_ranges:
proc_data['normalized_' + protocol + '_ranges'] = new_port_ranges profile['normalized_' + protocol + '_ranges'] = new_port_ranges
return proc_data return proc_data
@ -254,12 +263,13 @@ def parse(data, raw=False, quiet=False):
Returns: Returns:
Dictionary. Raw or processed structured data. List of Dictionaries. Raw or processed structured data.
""" """
if not quiet: if not quiet:
jc.utils.compatibility(__name__, info.compatible) jc.utils.compatibility(__name__, info.compatible)
raw_output = {} raw_output = []
item_obj = {}
if jc.utils.has_data(data): if jc.utils.has_data(data):
@ -267,16 +277,22 @@ def parse(data, raw=False, quiet=False):
for line in filter(None, data.splitlines()): for line in filter(None, data.splitlines()):
if line.startswith('--'):
if item_obj:
raw_output.append(item_obj)
item_obj = {}
continue
if line.startswith('Profile:'): if line.startswith('Profile:'):
raw_output['profile'] = line.split(': ')[1] item_obj['profile'] = line.split(': ')[1]
continue continue
if line.startswith('Title:'): if line.startswith('Title:'):
raw_output['title'] = line.split(': ')[1] item_obj['title'] = line.split(': ')[1]
continue continue
if line.startswith('Description:'): if line.startswith('Description:'):
raw_output['description'] = line.split(': ')[1] item_obj['description'] = line.split(': ')[1]
continue continue
if line.startswith('Port'): if line.startswith('Port'):
@ -289,20 +305,20 @@ def parse(data, raw=False, quiet=False):
if line_list[1] == 'tcp': if line_list[1] == 'tcp':
tcp_prot_list = _parse_port_list(line_list[0]) tcp_prot_list = _parse_port_list(line_list[0])
if tcp_prot_list: if tcp_prot_list:
raw_output['tcp_list'] = tcp_prot_list item_obj['tcp_list'] = tcp_prot_list
tcp_prot_range = _parse_port_range(line_list[0]) tcp_prot_range = _parse_port_range(line_list[0])
if tcp_prot_range: if tcp_prot_range:
raw_output['tcp_ranges'] = tcp_prot_range item_obj['tcp_ranges'] = tcp_prot_range
elif line_list[1] == 'udp': elif line_list[1] == 'udp':
udp_prot_list = _parse_port_list(line_list[0]) udp_prot_list = _parse_port_list(line_list[0])
if udp_prot_list: if udp_prot_list:
raw_output['udp_list'] = udp_prot_list item_obj['udp_list'] = udp_prot_list
udp_prot_range = _parse_port_range(line_list[0]) udp_prot_range = _parse_port_range(line_list[0])
if udp_prot_range: if udp_prot_range:
raw_output['udp_ranges'] = udp_prot_range item_obj['udp_ranges'] = udp_prot_range
# 'any' case # 'any' case
else: else:
@ -311,35 +327,36 @@ def parse(data, raw=False, quiet=False):
u_list = [] u_list = []
u_range = [] u_range = []
if 'tcp_list' in raw_output: if 'tcp_list' in item_obj:
t_list = raw_output['tcp_list'] t_list = item_obj['tcp_list']
if 'tcp_ranges' in raw_output: if 'tcp_ranges' in item_obj:
t_range = raw_output['tcp_ranges'] t_range = item_obj['tcp_ranges']
if 'udp_list' in raw_output: if 'udp_list' in item_obj:
u_list = raw_output['udp_list'] u_list = item_obj['udp_list']
if 'udp_ranges' in raw_output: if 'udp_ranges' in item_obj:
u_range = raw_output['udp_ranges'] u_range = item_obj['udp_ranges']
t_p_list = _parse_port_list(line, t_list) t_p_list = _parse_port_list(line, t_list)
if t_p_list: if t_p_list:
raw_output['tcp_list'] = t_p_list item_obj['tcp_list'] = t_p_list
t_r_list = _parse_port_range(line, t_range) t_r_list = _parse_port_range(line, t_range)
if t_r_list: if t_r_list:
raw_output['tcp_ranges'] = t_r_list item_obj['tcp_ranges'] = t_r_list
u_p_list = _parse_port_list(line, u_list) u_p_list = _parse_port_list(line, u_list)
if u_p_list: if u_p_list:
raw_output['udp_list'] = u_p_list item_obj['udp_list'] = u_p_list
u_r_list = _parse_port_range(line, u_range) u_r_list = _parse_port_range(line, u_range)
if u_r_list: if u_r_list:
raw_output['udp_ranges'] = u_r_list item_obj['udp_ranges'] = u_r_list
raw_output.update(raw_output) if item_obj:
raw_output.append(item_obj)
if raw: if raw:
return raw_output return raw_output

2
tests/fixtures/generic/ufw-appinfo-msn.json vendored
View File

@ -1 +1 @@
{"profile":"MSN","title":"MSN Chat","description":"MSN chat protocol (with file transfer and voice)","tcp_list":[1863,6901],"udp_list":[1863,6901],"tcp_ranges":[{"start":6891,"end":6900}],"normalized_tcp_list":[1863,6901],"normalized_tcp_ranges":[{"start":6891,"end":6900}],"normalized_udp_list":[1863,6901]} [{"profile":"MSN","title":"MSN Chat","description":"MSN chat protocol (with file transfer and voice)","tcp_list":[1863,6901],"udp_list":[1863,6901],"tcp_ranges":[{"start":6891,"end":6900}],"normalized_tcp_list":[1863,6901],"normalized_tcp_ranges":[{"start":6891,"end":6900}],"normalized_udp_list":[1863,6901]}]

2
tests/fixtures/generic/ufw-appinfo-test.json vendored
View File

@ -1 +1 @@
{"profile":"TEST","title":"My test app","description":"a longer description of the test app here.","tcp_list":[1,2,3,4,5,6,7,8,9,10,9,8,7,30,53],"tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"udp_ranges":[{"start":50,"end":51},{"start":40,"end":60}],"udp_list":[53],"normalized_tcp_list":[1,2,3,4,5,6,7,8,9,10,30,53],"normalized_tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"normalized_udp_ranges":[{"start":40,"end":60}]} [{"profile":"TEST","title":"My test app","description":"a longer description of the test app here.","tcp_list":[1,2,3,4,5,6,7,8,9,10,9,8,7,30,53],"tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"udp_ranges":[{"start":50,"end":51},{"start":40,"end":60}],"udp_list":[53],"normalized_tcp_list":[1,2,3,4,5,6,7,8,9,10,30,53],"normalized_tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"normalized_udp_ranges":[{"start":40,"end":60}]}]

2
tests/fixtures/generic/ufw-appinfo-test2.json vendored
View File

@ -1 +1 @@
{"profile":"TEST2","title":"My test app2","description":"a longer description of the test app here.","tcp_ranges":[{"start":0,"end":65535}],"udp_ranges":[{"start":50,"end":51}],"tcp_list":[53],"udp_list":[53],"normalized_tcp_ranges":[{"start":0,"end":65535}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]} [{"profile":"TEST2","title":"My test app2","description":"a longer description of the test app here.","tcp_ranges":[{"start":0,"end":65535}],"udp_ranges":[{"start":50,"end":51}],"tcp_list":[53],"udp_list":[53],"normalized_tcp_ranges":[{"start":0,"end":65535}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]}]

2
tests/fixtures/generic/ufw-appinfo-test3.json vendored
View File

@ -1 +1 @@
{"profile":"TEST3","title":"My test app3","description":"test overlapping ports","tcp_list":[80,83,80,53],"tcp_ranges":[{"start":70,"end":90}],"udp_ranges":[{"start":50,"end":51}],"udp_list":[53],"normalized_tcp_list":[53],"normalized_tcp_ranges":[{"start":70,"end":90}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]} [{"profile":"TEST3","title":"My test app3","description":"test overlapping ports","tcp_list":[80,83,80,53],"tcp_ranges":[{"start":70,"end":90}],"udp_ranges":[{"start":50,"end":51}],"udp_list":[53],"normalized_tcp_list":[53],"normalized_tcp_ranges":[{"start":70,"end":90}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]}]

1
tests/fixtures/ubuntu-18.04/ufw-appinfo-all.json vendored Normal file
View File

@ -0,0 +1 @@
[{"profile":"MSN","title":"MSN Chat","description":"MSN chat protocol (with file transfer and voice)","tcp_list":[1863,6901],"udp_list":[1863,6901],"tcp_ranges":[{"start":6891,"end":6900}],"normalized_tcp_list":[1863,6901],"normalized_tcp_ranges":[{"start":6891,"end":6900}],"normalized_udp_list":[1863,6901]},{"profile":"OpenSSH","title":"Secure shell server, an rshd replacement","description":"OpenSSH is a free implementation of the Secure Shell protocol.","tcp_list":[22],"normalized_tcp_list":[22]},{"profile":"TEST","title":"My test app","description":"a longer description of the test app here.","tcp_list":[1,2,3,4,5,6,7,8,9,10,30,53],"tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"udp_ranges":[{"start":50,"end":51}],"udp_list":[53],"normalized_tcp_list":[1,2,3,4,5,6,7,8,9,10,30,53],"normalized_tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]},{"profile":"TEST2","title":"My test app2","description":"a longer description of the test app here.","tcp_ranges":[{"start":0,"end":65535}],"udp_ranges":[{"start":50,"end":51}],"tcp_list":[53],"udp_list":[53],"normalized_tcp_ranges":[{"start":0,"end":65535}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]},{"profile":"TEST3","title":"My test app3","description":"test overlapping ports","tcp_list":[80,83,80,53],"tcp_ranges":[{"start":70,"end":90}],"udp_ranges":[{"start":50,"end":51}],"udp_list":[53],"normalized_tcp_list":[53],"normalized_tcp_ranges":[{"start":70,"end":90}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]}]

51
tests/fixtures/ubuntu-18.04/ufw-appinfo-all.out vendored Normal file
View File

@ -0,0 +1,51 @@
Profile: MSN
Title: MSN Chat
Description: MSN chat protocol (with file transfer and voice)
Ports:
1863
6891:6900/tcp
6901
--
Profile: OpenSSH
Title: Secure shell server, an rshd replacement
Description: OpenSSH is a free implementation of the Secure Shell protocol.
Port:
22/tcp
--
Profile: TEST
Title: My test app
Description: a longer description of the test app here.
Ports:
1,2,3,4,5,6,7,8,9,10,30,80:90,8080:8090/tcp
50:51/udp
53
--
Profile: TEST2
Title: My test app2
Description: a longer description of the test app here.
Ports:
any/tcp
50:51/udp
53
--
Profile: TEST3
Title: My test app3
Description: test overlapping ports
Ports:
80,83,80,70:90/tcp
50:51/udp
53

14
tests/test_ufw_appinfo.py
View File

@ -10,6 +10,9 @@ class MyTests(unittest.TestCase):
def setUp(self): def setUp(self):
# input # input
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ufw-appinfo-all.out'), 'r', encoding='utf-8') as f:
self.ubuntu_18_04_ufw_appinfo_all = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ufw-appinfo-test.out'), 'r', encoding='utf-8') as f: with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ufw-appinfo-test.out'), 'r', encoding='utf-8') as f:
self.generic_ufw_appinfo_test = f.read() self.generic_ufw_appinfo_test = f.read()
@ -23,6 +26,9 @@ class MyTests(unittest.TestCase):
self.generic_ufw_appinfo_msn = f.read() self.generic_ufw_appinfo_msn = f.read()
# output # output
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ufw-appinfo-all.json'), 'r', encoding='utf-8') as f:
self.ubuntu_18_04_ufw_appinfo_all_json = json.loads(f.read())
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ufw-appinfo-test.json'), 'r', encoding='utf-8') as f: with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ufw-appinfo-test.json'), 'r', encoding='utf-8') as f:
self.generic_ufw_appinfo_test_json = json.loads(f.read()) self.generic_ufw_appinfo_test_json = json.loads(f.read())
@ -39,7 +45,13 @@ class MyTests(unittest.TestCase):
""" """
Test 'ufw_appinfo' with no data Test 'ufw_appinfo' with no data
""" """
self.assertEqual(jc.parsers.ufw_appinfo.parse('', quiet=True), {}) self.assertEqual(jc.parsers.ufw_appinfo.parse('', quiet=True), [])
def test_ufw_appinfo_ubuntu_18_04_all(self):
"""
Test 'ufw app info all' on Ubuntu 18.04
"""
self.assertEqual(jc.parsers.ufw_appinfo.parse(self.ubuntu_18_04_ufw_appinfo_all, quiet=True), self.ubuntu_18_04_ufw_appinfo_all_json)
def test_ufw_appinfo_generic_test(self): def test_ufw_appinfo_generic_test(self):
""" """