Merge pull request #275 from kellyjonbrazil/dev

Dev v1.21.1

CHANGELOG

@@ -1,5 +1,11 @@
 jc changelog
 
+20220828 v1.21.1
+- Fix IP Address string parser for older python versions that don't cleanly
+  accept decimal input format (e.g. python 3.6)
+- Fix `arp -a` parser for cases where incomplete hardware addresses are found
+  in the arp table on linux
+
 20220821 v1.21.0
 - Add IP Address string parser
 - Add Syslog standard and streaming string parsers (RFC 3164 and RFC 5424)

README.md

@@ -292,7 +292,9 @@ option.
 
 ### Exit Codes
 Any fatal errors within `jc` will generate an exit code of `100`, otherwise the
-exit code will be `0`. When using the "magic" syntax (e.g. `jc ifconfig eth0`),
+exit code will be `0`.
+
+When using the "magic" syntax (e.g. `jc ifconfig eth0`),
 `jc` will store the exit code of the program being parsed and add it to the `jc`
 exit code. This way it is easier to determine if an error was from the parsed
 program or `jc`.
@@ -306,6 +308,44 @@ Consider the following examples using `ifconfig`:
 | `0`                  | `100`          | `100`              | Error in  `jc`                     |
 | `1`                  | `100`          | `101`              | Error in  both `ifconfig` and `jc` |
 
+When using the "magic" syntax you can also retrieve the exit code of the called
+program by using the `--meta-out` or `-M` option. This will append a `_jc_meta`
+object to the output that will include the magic command information, including
+the exit code.
+
+Here is an example with `ping`:
+```bash
+$ jc --meta-out -p ping -c2 192.168.1.252
+{
+  "destination_ip": "192.168.1.252",
+  "data_bytes": 56,
+  "pattern": null,
+  "destination": "192.168.1.252",
+  "packets_transmitted": 2,
+  "packets_received": 0,
+  "packet_loss_percent": 100.0,
+  "duplicates": 0,
+  "responses": [
+    {
+      "type": "timeout",
+      "icmp_seq": 0,
+      "duplicate": false
+    }
+  ],
+  "_jc_meta": {
+    "parser": "ping",
+    "timestamp": 1661357115.27949,
+    "magic_command": [
+      "ping",
+      "-c2",
+      "192.168.1.252"
+    ],
+    "magic_command_exit": 2
+  }
+}
+$ echo $?
+2
+```
 
 ### Setting Custom Colors via Environment Variable
 You can specify custom colors via the `JC_COLORS` environment variable. The

docs/lib.md

@@ -34,11 +34,22 @@ Parse the string data using the supplied parser module.
 This function provides a high-level API to simplify parser use. This
 function will call built-in parsers and custom plugin parsers.
 
-Example:
+Example (standard parsers):
 
     >>> import jc
-    >>> jc.parse('date', 'Tue Jan 18 10:23:07 PST 2022')
-    {'year': 2022, 'month': 'Jan', 'month_num': 1, 'day'...}
+    >>> date_obj = jc.parse('date', 'Tue Jan 18 10:23:07 PST 2022')
+    >>> print(f'The year is: {date_obj["year"]}')
+    The year is: 2022
+
+Example (streaming parsers):
+
+    >>> import jc
+    >>> ping_gen = jc.parse('ping_s', ping_output.splitlines())
+    >>> for item in ping_gen:
+    >>>     print(f'Response time: {item["time_ms"]} ms')
+    Response time: 102 ms
+    Response time: 109 ms
+    ...
 
 To get a list of available parser module names, use `parser_mod_list()`.
 

docs/parsers/arp.md

@@ -140,4 +140,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, aix, freebsd, darwin
 
-Version 1.10 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.11 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/parsers/ip_address.md

@@ -487,4 +487,4 @@ Returns:
 ### Parser Information
 Compatibility:  linux, darwin, cygwin, win32, aix, freebsd
 
-Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)
+Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/readme.md

@@ -11,12 +11,17 @@ and file-types to dictionaries and lists of dictionaries.
 
 ## Interactive Documentation
 
+Using `jc` in your python programs:
+
     >>> help('jc')
     >>> help('jc.lib')
+    >>> jc.get_help('parser_module_name')
+
+Developing `jc` parsers:
+
     >>> help('jc.utils')
     >>> help('jc.streaming')
     >>> help('jc.parsers.universal')
-    >>> jc.get_help('parser_module_name')
 
 ## Online Documentation
 

jc/__init__.py

@@ -7,12 +7,17 @@ and file-types to dictionaries and lists of dictionaries.
 
 ## Interactive Documentation
 
+Using `jc` in your python programs:
+
     >>> help('jc')
     >>> help('jc.lib')
+    >>> jc.get_help('parser_module_name')
+
+Developing `jc` parsers:
+
     >>> help('jc.utils')
     >>> help('jc.streaming')
     >>> help('jc.parsers.universal')
-    >>> jc.get_help('parser_module_name')
 
 ## Online Documentation
 

jc/lib.py

@@ -6,7 +6,7 @@ import importlib
 from typing import Dict, List, Iterable, Union, Iterator
 from jc import appdirs
 
-__version__ = '1.21.0'
+__version__ = '1.21.1'
 
 parsers = [
     'acpi',
@@ -197,11 +197,22 @@ def parse(
     This function provides a high-level API to simplify parser use. This
     function will call built-in parsers and custom plugin parsers.
 
-    Example:
+    Example (standard parsers):
 
         >>> import jc
-        >>> jc.parse('date', 'Tue Jan 18 10:23:07 PST 2022')
-        {'year': 2022, 'month': 'Jan', 'month_num': 1, 'day'...}
+        >>> date_obj = jc.parse('date', 'Tue Jan 18 10:23:07 PST 2022')
+        >>> print(f'The year is: {date_obj["year"]}')
+        The year is: 2022
+
+    Example (streaming parsers):
+
+        >>> import jc
+        >>> ping_gen = jc.parse('ping_s', ping_output.splitlines())
+        >>> for item in ping_gen:
+        >>>     print(f'Response time: {item["time_ms"]} ms')
+        Response time: 102 ms
+        Response time: 109 ms
+        ...
 
     To get a list of available parser module names, use `parser_mod_list()`.
 

jc/parsers/arp.py

@@ -119,7 +119,7 @@ import jc.parsers.universal
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.10'
+    version = '1.11'
     description = '`arp` command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -221,13 +221,24 @@ def parse(
         else:
             for line in cleandata:
                 splitline = line.split()
-                output_line = {
-                    'name': splitline[0],
-                    'address': splitline[1].lstrip('(').rstrip(')'),
-                    'hwtype': splitline[4].lstrip('[').rstrip(']'),
-                    'hwaddress': splitline[3],
-                    'iface': splitline[6],
-                }
+                if '<incomplete>' not in splitline:
+                    output_line = {
+                        'name': splitline[0],
+                        'address': splitline[1].lstrip('(').rstrip(')'),
+                        'hwtype': splitline[4].lstrip('[').rstrip(']'),
+                        'hwaddress': splitline[3],
+                        'iface': splitline[6],
+                    }
+
+                else:
+                    output_line = {
+                        'name': splitline[0],
+                        'address': splitline[1].lstrip('(').rstrip(')'),
+                        'hwtype': None,
+                        'hwaddress': None,
+                        'iface': splitline[5],
+                    }
+
                 raw_output.append(output_line)
 
     return raw_output if raw else _process(raw_output)

jc/parsers/ip_address.py

@@ -468,7 +468,7 @@ import jc.utils
 
 class info():
     """Provides parser metadata (version, author, etc.)"""
-    version = '1.0'
+    version = '1.1'
     description = 'IPv4 and IPv6 Address string parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -565,10 +565,20 @@ def parse(
         broadcast_string = str(network.broadcast_address)
         broadcast_ipobj = ipaddress.ip_address(broadcast_string)
 
-        hostmask_string = str(interface.with_hostmask).split('/')[1]
+        # older versions of python (e.g. 3.6) don't provide hostmask when a decimal IP is entered
+        try:
+            hostmask_string = str(interface.hostmask)
+        except AttributeError:
+            hostmask_string = '0.0.0.0'
+
         hostmask_ipobj = ipaddress.ip_address(hostmask_string)
 
-        netmask_string = str(interface.with_netmask).split('/')[1]
+        # older versions of python (e.g. 3.6) don't provide netmask when a decimal IP is entered
+        try:
+            netmask_string = str(interface.netmask)
+        except AttributeError:
+            netmask_string = '255.255.255.255'
+
         netmask_ipobj = ipaddress.ip_address(netmask_string)
 
         bare_ip_string = str(interface.ip)

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2022-08-21 1.21.0 "JSON Convert"
+.TH jc 1 2022-08-28 1.21.1 "JSON Convert"
 .SH NAME
 \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools and file-types
 .SH SYNOPSIS
@@ -682,7 +682,9 @@ Generate Bash shell completion script
 Generate Zsh shell completion script
 
 .SH EXIT CODES
-Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP. When using the "Magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store the exit code of the program being parsed and add it to the \fBjc\fP exit code. This way it is easier to determine if an error was from the parsed program or \fBjc\fP.
+Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP.
+
+When using the "magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store the exit code of the program being parsed and add it to the \fBjc\fP exit code. This way it is easier to determine if an error was from the parsed program or \fBjc\fP.
 
 Consider the following examples using \fBifconfig\fP:
 
@@ -696,6 +698,47 @@ ifconfig exit code = \fB0\fP, jc exit code = \fB100\fP, combined exit code = \fB
 ifconfig exit code = \fB1\fP, jc exit code = \fB100\fP, combined exit code = \fB101\fP (error in both ifconfig and jc)
 .RE
 
+When using the "magic" syntax you can also retrieve the exit code of the called
+program by using the \fB--meta-out\fP or \fB-M\fP option. This will append a \fB_jc_meta\fP
+object to the output that will include the magic command information, including
+the exit code.
+
+Here is an example with \fBping\fP:
+.RS
+.nf
+$ jc --meta-out -p ping -c2 192.168.1.252
+{
+  "destination_ip": "192.168.1.252",
+  "data_bytes": 56,
+  "pattern": null,
+  "destination": "192.168.1.252",
+  "packets_transmitted": 2,
+  "packets_received": 0,
+  "packet_loss_percent": 100.0,
+  "duplicates": 0,
+  "responses": [
+    {
+      "type": "timeout",
+      "icmp_seq": 0,
+      "duplicate": false
+    }
+  ],
+  "_jc_meta": {
+    "parser": "ping",
+    "timestamp": 1661357115.27949,
+    "magic_command": [
+      "ping",
+      "-c2",
+      "192.168.1.252"
+    ],
+    "magic_command_exit": 2
+  }
+}
+$ echo $?
+2
+.fi
+.RE
+
 .SH ENVIRONMENT
 
 \fBCustom Colors\fP

setup.py

@@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
 
 setuptools.setup(
     name='jc',
-    version='1.21.0',
+    version='1.21.1',
     author='Kelly Brazil',
     author_email='kellyjonbrazil@gmail.com',
     description='Converts the output of popular command-line tools and file-types to JSON.',

templates/manpage_template

@@ -87,7 +87,9 @@ Generate Bash shell completion script
 Generate Zsh shell completion script
 
 .SH EXIT CODES
-Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP. When using the "Magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store the exit code of the program being parsed and add it to the \fBjc\fP exit code. This way it is easier to determine if an error was from the parsed program or \fBjc\fP.
+Any fatal errors within \fBjc\fP will generate an exit code of \fB100\fP, otherwise the exit code will be \fB0\fP.
+
+When using the "magic" syntax (e.g. \fBjc ifconfig eth0\fP), \fBjc\fP will store the exit code of the program being parsed and add it to the \fBjc\fP exit code. This way it is easier to determine if an error was from the parsed program or \fBjc\fP.
 
 Consider the following examples using \fBifconfig\fP:
 
@@ -101,6 +103,47 @@ ifconfig exit code = \fB0\fP, jc exit code = \fB100\fP, combined exit code = \fB
 ifconfig exit code = \fB1\fP, jc exit code = \fB100\fP, combined exit code = \fB101\fP (error in both ifconfig and jc)
 .RE
 
+When using the "magic" syntax you can also retrieve the exit code of the called
+program by using the \fB--meta-out\fP or \fB-M\fP option. This will append a \fB_jc_meta\fP
+object to the output that will include the magic command information, including
+the exit code.
+
+Here is an example with \fBping\fP:
+.RS
+.nf
+$ jc --meta-out -p ping -c2 192.168.1.252
+{
+  "destination_ip": "192.168.1.252",
+  "data_bytes": 56,
+  "pattern": null,
+  "destination": "192.168.1.252",
+  "packets_transmitted": 2,
+  "packets_received": 0,
+  "packet_loss_percent": 100.0,
+  "duplicates": 0,
+  "responses": [
+    {
+      "type": "timeout",
+      "icmp_seq": 0,
+      "duplicate": false
+    }
+  ],
+  "_jc_meta": {
+    "parser": "ping",
+    "timestamp": 1661357115.27949,
+    "magic_command": [
+      "ping",
+      "-c2",
+      "192.168.1.252"
+    ],
+    "magic_command_exit": 2
+  }
+}
+$ echo $?
+2
+.fi
+.RE
+
 .SH ENVIRONMENT
 
 \fBCustom Colors\fP

templates/readme_template

@@ -173,7 +173,9 @@ option.
 
 ### Exit Codes
 Any fatal errors within `jc` will generate an exit code of `100`, otherwise the
-exit code will be `0`. When using the "magic" syntax (e.g. `jc ifconfig eth0`),
+exit code will be `0`.
+
+When using the "magic" syntax (e.g. `jc ifconfig eth0`),
 `jc` will store the exit code of the program being parsed and add it to the `jc`
 exit code. This way it is easier to determine if an error was from the parsed
 program or `jc`.
@@ -187,6 +189,44 @@ Consider the following examples using `ifconfig`:
 | `0`                  | `100`          | `100`              | Error in  `jc`                     |
 | `1`                  | `100`          | `101`              | Error in  both `ifconfig` and `jc` |
 
+When using the "magic" syntax you can also retrieve the exit code of the called
+program by using the `--meta-out` or `-M` option. This will append a `_jc_meta`
+object to the output that will include the magic command information, including
+the exit code.
+
+Here is an example with `ping`:
+```bash
+$ jc --meta-out -p ping -c2 192.168.1.252
+{
+  "destination_ip": "192.168.1.252",
+  "data_bytes": 56,
+  "pattern": null,
+  "destination": "192.168.1.252",
+  "packets_transmitted": 2,
+  "packets_received": 0,
+  "packet_loss_percent": 100.0,
+  "duplicates": 0,
+  "responses": [
+    {
+      "type": "timeout",
+      "icmp_seq": 0,
+      "duplicate": false
+    }
+  ],
+  "_jc_meta": {
+    "parser": "ping",
+    "timestamp": 1661357115.27949,
+    "magic_command": [
+      "ping",
+      "-c2",
+      "192.168.1.252"
+    ],
+    "magic_command_exit": 2
+  }
+}
+$ echo $?
+2
+```
 
 ### Setting Custom Colors via Environment Variable
 You can specify custom colors via the `JC_COLORS` environment variable. The

tests/fixtures/centos-8/arp-a.json

@@ -0,0 +1 @@
+[{"name":null,"address":"192.168.71.21","hwtype":null,"hwaddress":null,"iface":"ens33"},{"name":null,"address":"192.168.71.128","hwtype":null,"hwaddress":null,"iface":"ens33"},{"name":null,"address":"192.168.71.254","hwtype":"ether","hwaddress":"00:50:56:e2:91:0e","iface":"ens33"},{"name":null,"address":"192.168.71.226","hwtype":null,"hwaddress":null,"iface":"ens33"}]

tests/fixtures/centos-8/arp-a.out

@@ -0,0 +1,4 @@
+? (192.168.71.21) at <incomplete> on ens33
+? (192.168.71.128) at <incomplete> on ens33
+? (192.168.71.254) at 00:50:56:e2:91:0e [ether] on ens33
+? (192.168.71.226) at <incomplete> on ens33

tests/test_arp.py

@@ -40,6 +40,9 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/freebsd12/arp-a.out'), 'r', encoding='utf-8') as f:
             self.freebsd_arp_a = f.read()
 
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-8/arp-a.out'), 'r', encoding='utf-8') as f:
+            self.centos8_arp_a = f.read()
+
         # output
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/arp.json'), 'r', encoding='utf-8') as f:
             self.centos_7_7_arp_json = json.loads(f.read())
@@ -71,6 +74,9 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/freebsd12/arp-a.json'), 'r', encoding='utf-8') as f:
             self.freebsd12_arp_a_json = json.loads(f.read())
 
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-8/arp-a.json'), 'r', encoding='utf-8') as f:
+            self.centos8_arp_a_json = json.loads(f.read())
+
     def test_arp_nodata(self):
         """
         Test 'arp' with no data
@@ -137,6 +143,12 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(jc.parsers.arp.parse(self.freebsd_arp_a, quiet=True), self.freebsd12_arp_a_json)
 
+    def test_arp_a_centos8(self):
+        """
+        Test 'arp -a' on CentOS 8 with incomplete hw addresses
+        """
+        self.assertEqual(jc.parsers.arp.parse(self.centos8_arp_a, quiet=True), self.centos8_arp_a_json)
+
 
 if __name__ == '__main__':
     unittest.main()