Merge pull request #25 from kellyjonbrazil/dev

Dev v1.7.4

README.md

@@ -4,7 +4,6 @@ JSON CLI output utility
 `jc` is used to JSONify the output of many standard linux cli tools and file types for easier parsing in scripts. See the **Parsers** section for supported commands.
 
 This allows further command line processing of output with tools like `jq` simply by piping commands:
-
 ```
 $ ls -l /usr/bin | jc --ls | jq '.[] | select(.size > 50000000)'
 {
@@ -30,9 +29,6 @@ $ jc ls -l /usr/bin | jq '.[] | select(.size > 50000000)'
   "date": "Aug 14 19:41"
 }
 ```
-
-For more information on the motivations for this project, please see my blog post at https://blog.kellybrazil.com/2019/11/26/bringing-the-unix-philosophy-to-the-21st-century/.
-
 The `jc` parsers can also be used as python modules. In this case the output will be a python dictionary instead of JSON:
 ```
 >>> import jc.parsers.ls
@@ -65,6 +61,8 @@ To access the raw, pre-processed JSON, use the `-r` cli option or the `raw=True`
 
 Schemas for each parser can be found in the [`docs/parsers`](https://github.com/kellyjonbrazil/jc/tree/dev/docs/parsers) folder. 
 
+For more information on the motivations for this project, please see my [blog post](https://blog.kellybrazil.com/2019/11/26/bringing-the-unix-philosophy-to-the-21st-century/).
+
 ## Installation
 ```
 $ pip3 install --upgrade jc
@@ -79,7 +77,7 @@ or
 ```
 COMMAND | jc [OPTIONS] PARSER
 ```
-Alternatively, the "magic" syntax can be used by prepending `jc` to the command to be converted. Options can be passed to `jc` immediately before the command is given.
+Alternatively, the "magic" syntax can be used by prepending `jc` to the command to be converted. Options can be passed to `jc` immediately before the command is given. (Note: command aliases are not supported)
 ```
 jc [OPTIONS] COMMAND
 ```
@@ -624,7 +622,7 @@ $ cat /etc/fstab | jc --fstab -p
 ```
 ### history
 ```
-$ history | jc --history -p          # or:  jc -p history
+$ history | jc --history -p
 [
   {
     "line": 118,
@@ -1879,7 +1877,7 @@ Future parsers:
 Feel free to add/improve code or parsers! You can use the `jc/parsers/foo.py` parser as a template and submit your parser with a pull request.
 
 ## Compatibility
-Some parsers like `ls`, `ps`, `dig`, etc. will work on any platform. Other parsers that are platform-specific will generate a warning message if they are used on an unsupported platform. To see all parser information, including compatibility, run `jc -a -p`.
+Some parsers like `ls`, `ps`, `dig`, etc. will work on any platform. Other parsers that are platform-specific will generate a warning message if they are used on an unsupported platform. To see all parser information, including compatibility, run `jc -ap`.
 
 
 You may still use a parser on an unsupported platform - for example, you may want to parse a file with linux `lsof` output on an OSX laptop. In that case you can suppress the warning message with the `-q` cli option or the `quiet=True` function parameter in `parse()`:

changelog.txt

@@ -1,5 +1,8 @@
 jc changelog
 
+20200219 v1.7.4
+- Updated ls parser to support multiple directories, globbing, and -R (recursive)
+
 20200211 v1.7.3
 - Add alternative 'magic' syntax: e.g. `jc ls -al`
 - Options can now be condensed (e.g. -prq is equivalant to -p -r -q)

docs/parsers/ls.md

@@ -7,7 +7,8 @@ Usage:
 
     ls options supported:
     - None
-    - la
+    - laR
+    --time-style=full-iso
     - h       file sizes will be available in text form with -r but larger file sizes
               with human readable suffixes will be converted to Null in default view
               since the parser attempts to convert this field to an integer.
@@ -165,6 +166,7 @@ Returns:
         "filename": string,
         "flags":    string,
         "links":    integer,
+        "parent":   string,
         "owner":    string,
         "group":    string,
         "size":     integer,

jc/cli.py

@@ -13,7 +13,7 @@ import jc.utils
 
 
 class info():
-    version = '1.7.3'
+    version = '1.7.4'
     description = 'jc cli output JSON conversion tool'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'

jc/parsers/history.py

@@ -51,7 +51,6 @@ class info():
 
     # compatible options: linux, darwin, cygwin, win32, aix, freebsd
     compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd']
-    magic_commands = ['history']
 
 
 __version__ = info.version

jc/parsers/ls.py

@@ -6,7 +6,8 @@ Usage:
 
     ls options supported:
     - None
-    - la
+    - laR
+    --time-style=full-iso
     - h       file sizes will be available in text form with -r but larger file sizes
               with human readable suffixes will be converted to Null in default view
               since the parser attempts to convert this field to an integer.
@@ -144,7 +145,7 @@ import jc.utils
 
 
 class info():
-    version = '1.0'
+    version = '1.1'
     description = 'ls command parser'
     author = 'Kelly Brazil'
     author_email = 'kellyjonbrazil@gmail.com'
@@ -174,6 +175,7 @@ def process(proc_data):
             "filename": string,
             "flags":    string,
             "links":    integer,
+            "parent":   string,
             "owner":    string,
             "group":    string,
             "size":     integer,
@@ -216,22 +218,40 @@ def parse(data, raw=False, quiet=False):
 
     linedata = data.splitlines()
 
-    # Delete first line if it starts with 'total'
+    # Delete first line if it starts with 'total 1234'
     if linedata:
-        if linedata[0].find('total') == 0:
+        if re.match('^total [0-9]+', linedata[0]):
             linedata.pop(0)
 
-    # Clear any blank lines
-    cleandata = list(filter(None, linedata))
+    parent = ''
+    next_is_parent = False
 
-    if cleandata:
+    # Look for parent line if glob or -R is used
+    if not re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', linedata[0]) \
+       and linedata[0].endswith(':'):
+        parent = linedata.pop(0)[:-1]
+        # Pop following total line
+        linedata.pop(0)
+
+    if linedata:
         # Check if -l was used to parse extra data
-        if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', cleandata[0]):
-            for entry in cleandata:
+        if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', linedata[0]):
+            for entry in linedata:
                 output_line = {}
 
                 parsed_line = entry.split(maxsplit=8)
 
+                if not re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', entry) \
+                   and entry.endswith(':'):
+                    parent = entry[:-1]
+                    continue
+
+                if re.match('^total [0-9]+', entry):
+                    continue
+
+                if entry == '':
+                    continue
+
                 # split filenames and links
                 filename_field = parsed_line[8].split(' -> ')
 
@@ -241,6 +261,9 @@ def parse(data, raw=False, quiet=False):
                 if len(filename_field) > 1:
                     output_line['link_to'] = filename_field[1]
 
+                if parent:
+                    output_line['parent'] = parent
+
                 output_line['flags'] = parsed_line[0]
                 output_line['links'] = parsed_line[1]
                 output_line['owner'] = parsed_line[2]
@@ -249,9 +272,23 @@ def parse(data, raw=False, quiet=False):
                 output_line['date'] = ' '.join(parsed_line[5:8])
                 raw_output.append(output_line)
         else:
-            for entry in cleandata:
+            for entry in linedata:
                 output_line = {}
+
+                if entry == '':
+                    next_is_parent = True
+                    continue
+
+                if next_is_parent:
+                    parent = entry[:-1]
+                    next_is_parent = False
+                    continue
+
                 output_line['filename'] = entry
+
+                if parent:
+                    output_line['parent'] = parent
+
                 raw_output.append(output_line)
 
     if raw:

setup.py

@@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
 
 setuptools.setup(
     name='jc',
-    version='1.7.3',
+    version='1.7.4',
     author='Kelly Brazil',
     author_email='kellyjonbrazil@gmail.com',
     description='This tool serializes the output of popular command line tools and filetypes to structured JSON output.',

tests/fixtures/create_fixtures.sh

@@ -35,6 +35,11 @@ jobs       > jobs.out
 ls /       > ls.out
 ls -al /   > ls-al.out
 ls -alh /  > ls-alh.out
+
+ls -R /usr     > ls-R.out
+ls -alR /usr   > ls-alR.out
+ls /usr/*      > ls-glob.out
+
 lsblk      > lsblk.out
 lsblk -o +KNAME,FSTYPE,LABEL,UUID,PARTLABEL,PARTUUID,RA,MODEL,SERIAL,STATE,OWNER,GROUP,MODE,ALIGNMENT,MIN-IO,OPT-IO,PHY-SEC,LOG-SEC,ROTA,SCHED,RQ-SIZE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,WSAME,WWN,RAND,PKNAME,HCTL,TRAN,REV,VENDOR > lsblk-allcols.out
 lsmod      > lsmod.out

tests/test_ls.py

@@ -46,6 +46,33 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/ls-alh.out'), 'r') as f:
             self.osx_10_14_6_ls_alh = f.read()
 
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/ls-R.out'), 'r') as f:
+            self.centos_7_7_ls_R = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ls-R.out'), 'r') as f:
+            self.ubuntu_18_4_ls_R = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/ls-R.out'), 'r') as f:
+            self.osx_10_14_6_ls_R = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/ls-alR.out'), 'r') as f:
+            self.centos_7_7_ls_alR = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ls-alR.out'), 'r') as f:
+            self.ubuntu_18_4_ls_alR = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/ls-alR.out'), 'r') as f:
+            self.osx_10_14_6_ls_alR = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/ls-glob.out'), 'r') as f:
+            self.centos_7_7_ls_glob = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ls-glob.out'), 'r') as f:
+            self.ubuntu_18_4_ls_glob = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/ls-glob.out'), 'r') as f:
+            self.osx_10_14_6_ls_glob = f.read()
+
         # output
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/ls.json'), 'r') as f:
             self.centos_7_7_ls_json = json.loads(f.read())
@@ -83,6 +110,33 @@ class MyTests(unittest.TestCase):
         with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/ls-alh.json'), 'r') as f:
             self.osx_10_14_6_ls_alh_json = json.loads(f.read())
 
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/ls-R.json'), 'r') as f:
+            self.centos_7_7_ls_R_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ls-R.json'), 'r') as f:
+            self.ubuntu_18_4_ls_R_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/ls-R.json'), 'r') as f:
+            self.osx_10_14_6_ls_R_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/ls-alR.json'), 'r') as f:
+            self.centos_7_7_ls_alR_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ls-alR.json'), 'r') as f:
+            self.ubuntu_18_4_ls_alR_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/ls-alR.json'), 'r') as f:
+            self.osx_10_14_6_ls_alR_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/ls-glob.json'), 'r') as f:
+            self.centos_7_7_ls_glob_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/ls-glob.json'), 'r') as f:
+            self.ubuntu_18_4_ls_glob_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/ls-glob.json'), 'r') as f:
+            self.osx_10_14_6_ls_glob_json = json.loads(f.read())
+
     def test_ls_centos_7_7(self):
         """
         Test plain 'ls /' on Centos 7.7
@@ -155,6 +209,60 @@ class MyTests(unittest.TestCase):
         """
         self.assertEqual(jc.parsers.ls.parse(self.osx_10_14_6_ls_alh, quiet=True), self.osx_10_14_6_ls_alh_json)
 
+    def test_ls_R_centos_7_7(self):
+        """
+        Test 'ls -R /usr' on Centos 7.7
+        """
+        self.assertEqual(jc.parsers.ls.parse(self.centos_7_7_ls_R, quiet=True), self.centos_7_7_ls_R_json)
+
+    def test_ls_R_ubuntu_18_4(self):
+        """
+        Test 'ls -R /usr' on Ubuntu 18.4
+        """
+        self.assertEqual(jc.parsers.ls.parse(self.ubuntu_18_4_ls_R, quiet=True), self.ubuntu_18_4_ls_R_json)
+
+    def test_ls_R_osx_10_14_6(self):
+        """
+        Test 'ls -R /usr' on OSX 10.14.6
+        """
+        self.assertEqual(jc.parsers.ls.parse(self.osx_10_14_6_ls_R, quiet=True), self.osx_10_14_6_ls_R_json)
+
+    def test_ls_alR_centos_7_7(self):
+        """
+        Test 'ls -alR /usr' on Centos 7.7
+        """
+        self.assertEqual(jc.parsers.ls.parse(self.centos_7_7_ls_alR, quiet=True), self.centos_7_7_ls_alR_json)
+
+    def test_ls_alR_ubuntu_18_4(self):
+        """
+        Test 'ls -alR /usr' on Ubuntu 18.4
+        """
+        self.assertEqual(jc.parsers.ls.parse(self.ubuntu_18_4_ls_alR, quiet=True), self.ubuntu_18_4_ls_alR_json)
+
+    def test_ls_alR_osx_10_14_6(self):
+        """
+        Test 'ls -alR /usr' on OSX 10.14.6
+        """
+        self.assertEqual(jc.parsers.ls.parse(self.osx_10_14_6_ls_alR, quiet=True), self.osx_10_14_6_ls_alR_json)
+
+    def test_ls_glob_centos_7_7(self):
+        """
+        Test 'ls /usr/*' on Centos 7.7
+        """
+        self.assertEqual(jc.parsers.ls.parse(self.centos_7_7_ls_glob, quiet=True), self.centos_7_7_ls_glob_json)
+
+    def test_ls_glob_ubuntu_18_4(self):
+        """
+        Test 'ls /usr/*' on Ubuntu 18.4
+        """
+        self.assertEqual(jc.parsers.ls.parse(self.ubuntu_18_4_ls_glob, quiet=True), self.ubuntu_18_4_ls_glob_json)
+
+    def test_ls_glob_osx_10_14_6(self):
+        """
+        Test 'ls /usr/*' on OSX 10.14.6
+        """
+        self.assertEqual(jc.parsers.ls.parse(self.osx_10_14_6_ls_glob, quiet=True), self.osx_10_14_6_ls_glob_json)
+
 
 if __name__ == '__main__':
     unittest.main()