From a683f68003ea64dd5e2c4f06b53a1561ad774f12 Mon Sep 17 00:00:00 2001 From: Kelly Brazil Date: Mon, 26 Apr 2021 10:00:44 -0700 Subject: [PATCH] change schema to a list of dictionaries to support `ufw app info all` use case --- EXAMPLES.md | 68 ++-- jc/parsers/ufw_appinfo.py | 331 +++++++++--------- tests/fixtures/generic/ufw-appinfo-msn.json | 2 +- tests/fixtures/generic/ufw-appinfo-test.json | 2 +- tests/fixtures/generic/ufw-appinfo-test2.json | 2 +- tests/fixtures/generic/ufw-appinfo-test3.json | 2 +- .../ubuntu-18.04/ufw-appinfo-all.json | 1 + .../fixtures/ubuntu-18.04/ufw-appinfo-all.out | 51 +++ tests/test_ufw_appinfo.py | 14 +- 9 files changed, 278 insertions(+), 195 deletions(-) create mode 100644 tests/fixtures/ubuntu-18.04/ufw-appinfo-all.json create mode 100644 tests/fixtures/ubuntu-18.04/ufw-appinfo-all.out diff --git a/EXAMPLES.md b/EXAMPLES.md index c221cbf9..25279f31 100644 --- a/EXAMPLES.md +++ b/EXAMPLES.md @@ -3199,39 +3199,41 @@ ufw status verbose | jc --ufw -p # or jc -p ufw status verbose ufw app info MSN | jc --ufw-appinfo -p # or: jc -p ufw app info MSN ``` ```json -{ - "profile": "MSN", - "title": "MSN Chat", - "description": "MSN chat protocol (with file transfer and voice)", - "tcp_list": [ - 1863, - 6901 - ], - "udp_list": [ - 1863, - 6901 - ], - "tcp_ranges": [ - { - "start": 6891, - "end": 6900 - } - ], - "normalized_tcp_list": [ - 1863, - 6901 - ], - "normalized_tcp_ranges": [ - { - "start": 6891, - "end": 6900 - } - ], - "normalized_udp_list": [ - 1863, - 6901 - ] -} +[ + { + "profile": "MSN", + "title": "MSN Chat", + "description": "MSN chat protocol (with file transfer and voice)", + "tcp_list": [ + 1863, + 6901 + ], + "udp_list": [ + 1863, + 6901 + ], + "tcp_ranges": [ + { + "start": 6891, + "end": 6900 + } + ], + "normalized_tcp_list": [ + 1863, + 6901 + ], + "normalized_tcp_ranges": [ + { + "start": 6891, + "end": 6900 + } + ], + "normalized_udp_list": [ + 1863, + 6901 + ] + } +] ``` ### uname -a ```bash diff --git a/jc/parsers/ufw_appinfo.py b/jc/parsers/ufw_appinfo.py index 57396f2c..ecde58c1 100644 --- a/jc/parsers/ufw_appinfo.py +++ b/jc/parsers/ufw_appinfo.py @@ -1,5 +1,7 @@ """jc - JSON CLI output utility `ufw app info [application]` command output parser +Supports individual apps via `ufw app info [application]` and all apps list via `ufw app info all`. + Because `ufw` application definitions allow overlapping ports and port ranges, this parser preserves that behavior, but also provides `normalized` lists and ranges that remove duplicate ports and merge overlapping ranges. Usage (cli): @@ -17,105 +19,111 @@ Usage (module): Schema: - { - "profile": string, - "title": string, - "description": string, - "tcp_list": [ - integer - ], - "tcp_ranges": [ - { - "start": integer, # 'any' is converted to start/end: 0/65535 - "end": integer - } - ], - "udp_list": [ - integer - ], - "udp_ranges": [ - { - "start": integer, # 'any' is converted to start/end: 0/65535 - "end": integer - } - ], - "normalized_tcp_list": [ - integers # duplicates and overlapping are removed - ], - "normalized_tcp_ranges": [ - { - "start": integer, # 'any' is converted to start/end: 0/65535 - "end": integers # overlapping are merged - } - ], - "normalized_udp_list": [ - integers # duplicates and overlapping are removed - ], - "normalized_udp_ranges": [ - { - "start": integer, # 'any' is converted to start/end: 0/65535 - "end": integers # overlapping are merged - } - ] - } + [ + { + "profile": string, + "title": string, + "description": string, + "tcp_list": [ + integer + ], + "tcp_ranges": [ + { + "start": integer, # 'any' is converted to start/end: 0/65535 + "end": integer + } + ], + "udp_list": [ + integer + ], + "udp_ranges": [ + { + "start": integer, # 'any' is converted to start/end: 0/65535 + "end": integer + } + ], + "normalized_tcp_list": [ + integers # duplicates and overlapping are removed + ], + "normalized_tcp_ranges": [ + { + "start": integer, # 'any' is converted to start/end: 0/65535 + "end": integers # overlapping are merged + } + ], + "normalized_udp_list": [ + integers # duplicates and overlapping are removed + ], + "normalized_udp_ranges": [ + { + "start": integer, # 'any' is converted to start/end: 0/65535 + "end": integers # overlapping are merged + } + ] + } + ] Examples: $ ufw app info MSN | jc --ufw-appinfo -p - { - "profile": "MSN", - "title": "MSN Chat", - "description": "MSN chat protocol (with file transfer and voice)", - "tcp_list": [ - 1863, - 6901 - ], - "udp_list": [ - 1863, - 6901 - ], - "tcp_ranges": [ - { - "start": 6891, - "end": 6900 - } - ], - "normalized_tcp_list": [ - 1863, - 6901 - ], - "normalized_tcp_ranges": [ - { - "start": 6891, - "end": 6900 - } - ], - "normalized_udp_list": [ - 1863, - 6901 - ] - } + [ + { + "profile": "MSN", + "title": "MSN Chat", + "description": "MSN chat protocol (with file transfer and voice)", + "tcp_list": [ + 1863, + 6901 + ], + "udp_list": [ + 1863, + 6901 + ], + "tcp_ranges": [ + { + "start": 6891, + "end": 6900 + } + ], + "normalized_tcp_list": [ + 1863, + 6901 + ], + "normalized_tcp_ranges": [ + { + "start": 6891, + "end": 6900 + } + ], + "normalized_udp_list": [ + 1863, + 6901 + ] + } + ] $ ufw app info MSN | jc --ufw-appinfo -p -r - { - "profile": "MSN", - "title": "MSN Chat", - "description": "MSN chat protocol (with file transfer and voice)", - "tcp_list": [ - "1863", - "6901" - ], - "udp_list": [ - "1863", - "6901" - ], - "tcp_ranges": [ - { - "start": "6891", - "end": "6900" - } - ] - } + [ + { + "profile": "MSN", + "title": "MSN Chat", + "description": "MSN chat protocol (with file transfer and voice)", + "tcp_list": [ + "1863", + "6901" + ], + "udp_list": [ + "1863", + "6901" + ], + "tcp_ranges": [ + { + "start": "6891", + "end": "6900" + } + ] + } + ] """ import jc.utils @@ -143,56 +151,57 @@ def _process(proc_data): Returns: - Dictionary. Structured to conform to the schema. + List of Dictionaries. Structured to conform to the schema. """ - # convert to ints - int_list = ['start', 'end'] + for profile in proc_data: + # convert to ints + int_list = ['start', 'end'] - if 'tcp_list' in proc_data: - proc_data['tcp_list'] = [int(p) for p in proc_data['tcp_list']] + if 'tcp_list' in profile: + profile['tcp_list'] = [int(p) for p in profile['tcp_list']] - if 'udp_list' in proc_data: - proc_data['udp_list'] = [int(p) for p in proc_data['udp_list']] + if 'udp_list' in profile: + profile['udp_list'] = [int(p) for p in profile['udp_list']] - for protocol in ['tcp', 'udp']: - if protocol + '_ranges' in proc_data: - for i, item in enumerate(proc_data[protocol + '_ranges']): - for key in item: - if key in int_list: - proc_data[protocol + '_ranges'][i][key] = int(proc_data[protocol + '_ranges'][i][key]) + for protocol in ['tcp', 'udp']: + if protocol + '_ranges' in profile: + for i, item in enumerate(profile[protocol + '_ranges']): + for key in item: + if key in int_list: + profile[protocol + '_ranges'][i][key] = int(profile[protocol + '_ranges'][i][key]) - # create normalized port lists and port ranges (remove duplicates and merge ranges) - # dump ranges into a set of 0 - 65535 - # if items in the port list are in the set, then remove them - # iterate through the set to find gaps and create new ranges based on them - for protocol in ['tcp', 'udp']: - port_set = set() - if protocol + '_ranges' in proc_data: - for item in proc_data[protocol + '_ranges']: - port_set.update(range(item['start'], item['end'] + 1)) + # create normalized port lists and port ranges (remove duplicates and merge ranges) + # dump ranges into a set of 0 - 65535 + # if items in the port list are in the set, then remove them + # iterate through the set to find gaps and create new ranges based on them + for protocol in ['tcp', 'udp']: + port_set = set() + if protocol + '_ranges' in profile: + for item in profile[protocol + '_ranges']: + port_set.update(range(item['start'], item['end'] + 1)) - if protocol + '_list' in proc_data: - new_port_list = sorted(set([p for p in proc_data[protocol + '_list'] if p not in port_set])) - if new_port_list: - proc_data['normalized_' + protocol + '_list'] = new_port_list + if protocol + '_list' in profile: + new_port_list = sorted(set([p for p in profile[protocol + '_list'] if p not in port_set])) + if new_port_list: + profile['normalized_' + protocol + '_list'] = new_port_list - new_port_ranges = [] - state = 'findstart' # 'findstart' or 'findend' - for port in range(0, 65535 + 2): - if state == 'findstart': - port_range_obj = {} - if port in port_set: - port_range_obj['start'] = port - state = 'findend' - continue - if state == 'findend': - if port not in port_set: - port_range_obj['end'] = port - 1 - new_port_ranges.append(port_range_obj) - state = 'findstart' + new_port_ranges = [] + state = 'findstart' # 'findstart' or 'findend' + for port in range(0, 65535 + 2): + if state == 'findstart': + port_range_obj = {} + if port in port_set: + port_range_obj['start'] = port + state = 'findend' + continue + if state == 'findend': + if port not in port_set: + port_range_obj['end'] = port - 1 + new_port_ranges.append(port_range_obj) + state = 'findstart' - if new_port_ranges: - proc_data['normalized_' + protocol + '_ranges'] = new_port_ranges + if new_port_ranges: + profile['normalized_' + protocol + '_ranges'] = new_port_ranges return proc_data @@ -254,12 +263,13 @@ def parse(data, raw=False, quiet=False): Returns: - Dictionary. Raw or processed structured data. + List of Dictionaries. Raw or processed structured data. """ if not quiet: jc.utils.compatibility(__name__, info.compatible) - raw_output = {} + raw_output = [] + item_obj = {} if jc.utils.has_data(data): @@ -267,16 +277,22 @@ def parse(data, raw=False, quiet=False): for line in filter(None, data.splitlines()): + if line.startswith('--'): + if item_obj: + raw_output.append(item_obj) + item_obj = {} + continue + if line.startswith('Profile:'): - raw_output['profile'] = line.split(': ')[1] + item_obj['profile'] = line.split(': ')[1] continue if line.startswith('Title:'): - raw_output['title'] = line.split(': ')[1] + item_obj['title'] = line.split(': ')[1] continue if line.startswith('Description:'): - raw_output['description'] = line.split(': ')[1] + item_obj['description'] = line.split(': ')[1] continue if line.startswith('Port'): @@ -289,20 +305,20 @@ def parse(data, raw=False, quiet=False): if line_list[1] == 'tcp': tcp_prot_list = _parse_port_list(line_list[0]) if tcp_prot_list: - raw_output['tcp_list'] = tcp_prot_list + item_obj['tcp_list'] = tcp_prot_list tcp_prot_range = _parse_port_range(line_list[0]) if tcp_prot_range: - raw_output['tcp_ranges'] = tcp_prot_range + item_obj['tcp_ranges'] = tcp_prot_range elif line_list[1] == 'udp': udp_prot_list = _parse_port_list(line_list[0]) if udp_prot_list: - raw_output['udp_list'] = udp_prot_list + item_obj['udp_list'] = udp_prot_list udp_prot_range = _parse_port_range(line_list[0]) if udp_prot_range: - raw_output['udp_ranges'] = udp_prot_range + item_obj['udp_ranges'] = udp_prot_range # 'any' case else: @@ -311,35 +327,36 @@ def parse(data, raw=False, quiet=False): u_list = [] u_range = [] - if 'tcp_list' in raw_output: - t_list = raw_output['tcp_list'] + if 'tcp_list' in item_obj: + t_list = item_obj['tcp_list'] - if 'tcp_ranges' in raw_output: - t_range = raw_output['tcp_ranges'] + if 'tcp_ranges' in item_obj: + t_range = item_obj['tcp_ranges'] - if 'udp_list' in raw_output: - u_list = raw_output['udp_list'] + if 'udp_list' in item_obj: + u_list = item_obj['udp_list'] - if 'udp_ranges' in raw_output: - u_range = raw_output['udp_ranges'] + if 'udp_ranges' in item_obj: + u_range = item_obj['udp_ranges'] t_p_list = _parse_port_list(line, t_list) if t_p_list: - raw_output['tcp_list'] = t_p_list + item_obj['tcp_list'] = t_p_list t_r_list = _parse_port_range(line, t_range) if t_r_list: - raw_output['tcp_ranges'] = t_r_list + item_obj['tcp_ranges'] = t_r_list u_p_list = _parse_port_list(line, u_list) if u_p_list: - raw_output['udp_list'] = u_p_list + item_obj['udp_list'] = u_p_list u_r_list = _parse_port_range(line, u_range) if u_r_list: - raw_output['udp_ranges'] = u_r_list + item_obj['udp_ranges'] = u_r_list - raw_output.update(raw_output) + if item_obj: + raw_output.append(item_obj) if raw: return raw_output diff --git a/tests/fixtures/generic/ufw-appinfo-msn.json b/tests/fixtures/generic/ufw-appinfo-msn.json index 3532d142..36a1bb72 100644 --- a/tests/fixtures/generic/ufw-appinfo-msn.json +++ b/tests/fixtures/generic/ufw-appinfo-msn.json @@ -1 +1 @@ -{"profile":"MSN","title":"MSN Chat","description":"MSN chat protocol (with file transfer and voice)","tcp_list":[1863,6901],"udp_list":[1863,6901],"tcp_ranges":[{"start":6891,"end":6900}],"normalized_tcp_list":[1863,6901],"normalized_tcp_ranges":[{"start":6891,"end":6900}],"normalized_udp_list":[1863,6901]} +[{"profile":"MSN","title":"MSN Chat","description":"MSN chat protocol (with file transfer and voice)","tcp_list":[1863,6901],"udp_list":[1863,6901],"tcp_ranges":[{"start":6891,"end":6900}],"normalized_tcp_list":[1863,6901],"normalized_tcp_ranges":[{"start":6891,"end":6900}],"normalized_udp_list":[1863,6901]}] diff --git a/tests/fixtures/generic/ufw-appinfo-test.json b/tests/fixtures/generic/ufw-appinfo-test.json index 64eb243e..c3e6e6b5 100644 --- a/tests/fixtures/generic/ufw-appinfo-test.json +++ b/tests/fixtures/generic/ufw-appinfo-test.json @@ -1 +1 @@ -{"profile":"TEST","title":"My test app","description":"a longer description of the test app here.","tcp_list":[1,2,3,4,5,6,7,8,9,10,9,8,7,30,53],"tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"udp_ranges":[{"start":50,"end":51},{"start":40,"end":60}],"udp_list":[53],"normalized_tcp_list":[1,2,3,4,5,6,7,8,9,10,30,53],"normalized_tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"normalized_udp_ranges":[{"start":40,"end":60}]} +[{"profile":"TEST","title":"My test app","description":"a longer description of the test app here.","tcp_list":[1,2,3,4,5,6,7,8,9,10,9,8,7,30,53],"tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"udp_ranges":[{"start":50,"end":51},{"start":40,"end":60}],"udp_list":[53],"normalized_tcp_list":[1,2,3,4,5,6,7,8,9,10,30,53],"normalized_tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"normalized_udp_ranges":[{"start":40,"end":60}]}] diff --git a/tests/fixtures/generic/ufw-appinfo-test2.json b/tests/fixtures/generic/ufw-appinfo-test2.json index b48fb46a..25d964d7 100644 --- a/tests/fixtures/generic/ufw-appinfo-test2.json +++ b/tests/fixtures/generic/ufw-appinfo-test2.json @@ -1 +1 @@ -{"profile":"TEST2","title":"My test app2","description":"a longer description of the test app here.","tcp_ranges":[{"start":0,"end":65535}],"udp_ranges":[{"start":50,"end":51}],"tcp_list":[53],"udp_list":[53],"normalized_tcp_ranges":[{"start":0,"end":65535}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]} +[{"profile":"TEST2","title":"My test app2","description":"a longer description of the test app here.","tcp_ranges":[{"start":0,"end":65535}],"udp_ranges":[{"start":50,"end":51}],"tcp_list":[53],"udp_list":[53],"normalized_tcp_ranges":[{"start":0,"end":65535}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]}] diff --git a/tests/fixtures/generic/ufw-appinfo-test3.json b/tests/fixtures/generic/ufw-appinfo-test3.json index 20eb606c..54a91e4c 100644 --- a/tests/fixtures/generic/ufw-appinfo-test3.json +++ b/tests/fixtures/generic/ufw-appinfo-test3.json @@ -1 +1 @@ -{"profile":"TEST3","title":"My test app3","description":"test overlapping ports","tcp_list":[80,83,80,53],"tcp_ranges":[{"start":70,"end":90}],"udp_ranges":[{"start":50,"end":51}],"udp_list":[53],"normalized_tcp_list":[53],"normalized_tcp_ranges":[{"start":70,"end":90}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]} +[{"profile":"TEST3","title":"My test app3","description":"test overlapping ports","tcp_list":[80,83,80,53],"tcp_ranges":[{"start":70,"end":90}],"udp_ranges":[{"start":50,"end":51}],"udp_list":[53],"normalized_tcp_list":[53],"normalized_tcp_ranges":[{"start":70,"end":90}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]}] diff --git a/tests/fixtures/ubuntu-18.04/ufw-appinfo-all.json b/tests/fixtures/ubuntu-18.04/ufw-appinfo-all.json new file mode 100644 index 00000000..cbdcec44 --- /dev/null +++ b/tests/fixtures/ubuntu-18.04/ufw-appinfo-all.json @@ -0,0 +1 @@ +[{"profile":"MSN","title":"MSN Chat","description":"MSN chat protocol (with file transfer and voice)","tcp_list":[1863,6901],"udp_list":[1863,6901],"tcp_ranges":[{"start":6891,"end":6900}],"normalized_tcp_list":[1863,6901],"normalized_tcp_ranges":[{"start":6891,"end":6900}],"normalized_udp_list":[1863,6901]},{"profile":"OpenSSH","title":"Secure shell server, an rshd replacement","description":"OpenSSH is a free implementation of the Secure Shell protocol.","tcp_list":[22],"normalized_tcp_list":[22]},{"profile":"TEST","title":"My test app","description":"a longer description of the test app here.","tcp_list":[1,2,3,4,5,6,7,8,9,10,30,53],"tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"udp_ranges":[{"start":50,"end":51}],"udp_list":[53],"normalized_tcp_list":[1,2,3,4,5,6,7,8,9,10,30,53],"normalized_tcp_ranges":[{"start":80,"end":90},{"start":8080,"end":8090}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]},{"profile":"TEST2","title":"My test app2","description":"a longer description of the test app here.","tcp_ranges":[{"start":0,"end":65535}],"udp_ranges":[{"start":50,"end":51}],"tcp_list":[53],"udp_list":[53],"normalized_tcp_ranges":[{"start":0,"end":65535}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]},{"profile":"TEST3","title":"My test app3","description":"test overlapping ports","tcp_list":[80,83,80,53],"tcp_ranges":[{"start":70,"end":90}],"udp_ranges":[{"start":50,"end":51}],"udp_list":[53],"normalized_tcp_list":[53],"normalized_tcp_ranges":[{"start":70,"end":90}],"normalized_udp_list":[53],"normalized_udp_ranges":[{"start":50,"end":51}]}] diff --git a/tests/fixtures/ubuntu-18.04/ufw-appinfo-all.out b/tests/fixtures/ubuntu-18.04/ufw-appinfo-all.out new file mode 100644 index 00000000..55e71c72 --- /dev/null +++ b/tests/fixtures/ubuntu-18.04/ufw-appinfo-all.out @@ -0,0 +1,51 @@ +Profile: MSN +Title: MSN Chat +Description: MSN chat protocol (with file transfer and voice) + +Ports: + 1863 + 6891:6900/tcp + 6901 + +-- + +Profile: OpenSSH +Title: Secure shell server, an rshd replacement +Description: OpenSSH is a free implementation of the Secure Shell protocol. + +Port: + 22/tcp + +-- + +Profile: TEST +Title: My test app +Description: a longer description of the test app here. + +Ports: + 1,2,3,4,5,6,7,8,9,10,30,80:90,8080:8090/tcp + 50:51/udp + 53 + +-- + +Profile: TEST2 +Title: My test app2 +Description: a longer description of the test app here. + +Ports: + any/tcp + 50:51/udp + 53 + +-- + +Profile: TEST3 +Title: My test app3 +Description: test overlapping ports + +Ports: + 80,83,80,70:90/tcp + 50:51/udp + 53 + diff --git a/tests/test_ufw_appinfo.py b/tests/test_ufw_appinfo.py index ae5acb88..0e408042 100644 --- a/tests/test_ufw_appinfo.py +++ b/tests/test_ufw_appinfo.py @@ -10,6 +10,9 @@ class MyTests(unittest.TestCase): def setUp(self): # input + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ufw-appinfo-all.out'), 'r', encoding='utf-8') as f: + self.ubuntu_18_04_ufw_appinfo_all = f.read() + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ufw-appinfo-test.out'), 'r', encoding='utf-8') as f: self.generic_ufw_appinfo_test = f.read() @@ -23,6 +26,9 @@ class MyTests(unittest.TestCase): self.generic_ufw_appinfo_msn = f.read() # output + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ufw-appinfo-all.json'), 'r', encoding='utf-8') as f: + self.ubuntu_18_04_ufw_appinfo_all_json = json.loads(f.read()) + with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/generic/ufw-appinfo-test.json'), 'r', encoding='utf-8') as f: self.generic_ufw_appinfo_test_json = json.loads(f.read()) @@ -39,7 +45,13 @@ class MyTests(unittest.TestCase): """ Test 'ufw_appinfo' with no data """ - self.assertEqual(jc.parsers.ufw_appinfo.parse('', quiet=True), {}) + self.assertEqual(jc.parsers.ufw_appinfo.parse('', quiet=True), []) + + def test_ufw_appinfo_ubuntu_18_04_all(self): + """ + Test 'ufw app info all' on Ubuntu 18.04 + """ + self.assertEqual(jc.parsers.ufw_appinfo.parse(self.ubuntu_18_04_ufw_appinfo_all, quiet=True), self.ubuntu_18_04_ufw_appinfo_all_json) def test_ufw_appinfo_generic_test(self): """