Merge pull request #186 from kellyjonbrazil/dev

Dev v1.17.5

CHANGELOG

@@ -1,5 +1,8 @@
 jc changelog
 
+20211221 v1.17.5
+- Add zipinfo parser tested on linux and macOS
+
 20211207 v1.17.4
 - Add support for the NO_COLOR environment variable to set mono (http://no-color.org/)
 - Add -C option to force color output even when using pipes (overrides -m and NO_COLOR)

EXAMPLES.md

@@ -3793,5 +3793,36 @@ cat istio.yaml | jc --yaml -p
   }
 ]
 ```
+### zipinfo
+```bash
+zipinfo file.zip | jc --zipinfo -p          # or:  jc -p zipinfo file.zip
+```
+```json
+[
+  {
+    "archive": "file.zip",
+    "size": 4116,
+    "size_unit": "bytes",
+    "number_entries": 1,
+    "number_files": 1,
+    "bytes_uncompressed": 11837,
+    "bytes_compressed": 3966,
+    "percent_compressed": 66.5,
+    "files": [
+      {
+        "flags": "-rw-r--r--",
+        "zipversion": "2.1",
+        "zipunder": "unx",
+        "filesize": 11837,
+        "type": "bX",
+        "method": "defN",
+        "date": "21-Dec-08",
+        "time": "20:50",
+        "filename": "compressed_file"
+      }
+    ]
+  }
+]
+```
 
 © 2019-2021 Kelly Brazil

README.md

@@ -188,6 +188,7 @@ The JSON output can be compact (default) or pretty formatted with the `-p` optio
 - `--who` enables the `who` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/who))
 - `--xml` enables the XML file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/xml))
 - `--yaml` enables the YAML file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/yaml))
+- `--zipinfo` enables the `zipinfo` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/zipinfo))
 
 ### Options
 - `-a` about `jc`. Prints information about `jc` and the parsers (in JSON, of course!)

docs/parsers/zipinfo.md

@@ -0,0 +1,106 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+
+# jc.parsers.zipinfo
+jc - JSON CLI output utility `zipinfo` command output parser
+
+Options supported:
+- none
+
+Note: The default listing format.
+
+Usage (cli):
+
+    $ zipinfo <archive> | jc --zipinfo
+
+    or
+
+    $ jc zipinfo
+
+Usage (module):
+
+    import jc.parsers.zipinfo
+    result = jc.parsers.zipinfo.parse(zipinfo_command_output)
+
+Schema:
+    [
+      {
+        "archive":              string,
+        "size":                 integer,
+        "size_unit":            string,
+        "number_entries":       integer,
+        "number_files":         integer,
+        "bytes_uncompressed":   integer,
+        "bytes_compressed":     integer,
+        "percent_compressed":   float,
+        "files": [
+          {
+            "flags":            string,
+            "zipversion":       string,
+            "zipunder":         string
+            "filesize":         integer,
+            "type":             string,
+            "method":           string,
+            "date":             string,
+            "time":             string,
+            "filename":         string
+          }
+        ]
+      }
+    ]
+
+Examples:
+
+    $ zipinfo log4j-core-2.16.0.jar | jc --zipinfo -p
+
+    [
+      {
+        "archive": "log4j-core-2.16.0.jar",
+        "size": 1789565,
+        "size_unit": "bytes",
+        "number_entries": 1218,
+        "number_files": 1218,
+        "bytes_uncompressed": 3974141,
+        "bytes_compressed": 1515455,
+        "percent_compressed": 61.9,
+        "files": [
+          {
+            "flags": "-rw-r--r--",
+            "zipversion": "2.0",
+            "zipunder": "unx",
+            "filesize": 19810,
+            "type": "bl",
+            "method": "defN",
+            "date": "21-Dec-12",
+            "time": "23:35",
+            "filename": "META-INF/MANIFEST.MF"
+          },
+    ...
+
+
+## info
+```python
+info()
+```
+Provides parser metadata (version, author, etc.)
+
+## parse
+```python
+parse(data, raw=False, quiet=False)
+```
+
+Main text parsing function
+
+Parameters:
+
+    data:        (string)  text data to parse
+    raw:         (boolean) output preprocessed JSON if True
+    quiet:       (boolean) suppress warning messages if True
+
+Returns:
+
+    List of Dictionaries. Raw or processed structured data.
+
+## Parser Information
+Compatibility:  linux, darwin
+
+Version 0.01 by Matt J (https://github.com/listuser)

jc/__init__.py

@@ -73,4 +73,4 @@ Module Example:
 """
 
 name = 'jc'
-__version__ = '1.17.4'
+__version__ = '1.17.5'

jc/cli.py

@@ -125,7 +125,8 @@ parsers = [
     'wc',
     'who',
     'xml',
-    'yaml'
+    'yaml',
+    'zipinfo'
 ]
 
 JC_ERROR_EXIT = 100

jc/parsers/zipinfo.py

@@ -0,0 +1,205 @@
+"""jc - JSON CLI output utility `zipinfo` command output parser
+
+Options supported:
+- none
+
+Note: The default listing format.
+
+Usage (cli):
+
+    $ zipinfo <archive> | jc --zipinfo
+
+    or
+
+    $ jc zipinfo
+
+Usage (module):
+
+    import jc.parsers.zipinfo
+    result = jc.parsers.zipinfo.parse(zipinfo_command_output)
+
+Schema:
+    [
+      {
+        "archive":              string,
+        "size":                 integer,
+        "size_unit":            string,
+        "number_entries":       integer,
+        "number_files":         integer,
+        "bytes_uncompressed":   integer,
+        "bytes_compressed":     integer,
+        "percent_compressed":   float,
+        "files": [
+          {
+            "flags":            string,
+            "zipversion":       string,
+            "zipunder":         string
+            "filesize":         integer,
+            "type":             string,
+            "method":           string,
+            "date":             string,
+            "time":             string,
+            "filename":         string
+          }
+        ]
+      }
+    ]
+
+Examples:
+
+    $ zipinfo log4j-core-2.16.0.jar | jc --zipinfo -p
+
+    [
+      {
+        "archive": "log4j-core-2.16.0.jar",
+        "size": 1789565,
+        "size_unit": "bytes",
+        "number_entries": 1218,
+        "number_files": 1218,
+        "bytes_uncompressed": 3974141,
+        "bytes_compressed": 1515455,
+        "percent_compressed": 61.9,
+        "files": [
+          {
+            "flags": "-rw-r--r--",
+            "zipversion": "2.0",
+            "zipunder": "unx",
+            "filesize": 19810,
+            "type": "bl",
+            "method": "defN",
+            "date": "21-Dec-12",
+            "time": "23:35",
+            "filename": "META-INF/MANIFEST.MF"
+          },
+    ...
+"""
+import jc.utils
+import jc.parsers.universal
+
+
+class info():
+    """Provides parser metadata (version, author, etc.)"""
+    version = '0.01'
+    description = '`zipinfo` command parser'
+    author = 'Matt J'
+    author_email = 'https://github.com/listuser'
+    compatible = ['linux', 'darwin']
+    magic_commands = ['zipinfo']
+
+
+__version__ = info.version
+
+
+def _process(proc_data):
+    """
+    Final processing to conform to the schema.
+
+    Parameters:
+
+        proc_data:   (List of Dictionaries) raw structured data to process
+
+    Returns:
+
+        List of Dictionaries. Structured data to conform to the schema.
+    """
+
+    for entry in proc_data:
+        int_list = ['bytes_compressed', 'bytes_uncompressed', 'number_entries',
+                    'number_files', 'size', 'filesize']
+        float_list = ['percent_compressed']
+        for key in entry:
+            if key in int_list:
+                entry[key] = jc.utils.convert_to_int(entry[key])
+            if key in float_list:
+                entry[key] = jc.utils.convert_to_float(entry[key])
+
+            if 'files' in key:
+                for item in entry['files']:
+                    for key in item:
+                        if key in int_list:
+                            item[key] = jc.utils.convert_to_int(item[key])
+    return proc_data
+
+
+def parse(data, raw=False, quiet=False):
+    """
+    Main text parsing function
+
+    Parameters:
+
+        data:        (string)  text data to parse
+        raw:         (boolean) output preprocessed JSON if True
+        quiet:       (boolean) suppress warning messages if True
+
+    Returns:
+
+        List of Dictionaries. Raw or processed structured data.
+    """
+    jc.utils.compatibility(__name__, info.compatible, quiet)
+    jc.utils.input_type_check(data)
+
+    raw_output = []
+    archives = []
+
+    if jc.utils.has_data(data):
+        datalines = data.splitlines()
+
+        # remove last line of multi-archive output since it is not needed
+        if datalines[-1].endswith('archives were successfully processed.'):
+            datalines.pop(-1)
+
+        # extract each archive into its own list of lines.
+        # archives are separated by a blank line
+        this_archive = []
+        for row in datalines:
+            if row == '':
+                archives.append(this_archive)
+                this_archive = []
+                continue
+
+            this_archive.append(row)
+
+        if this_archive:
+            archives.append(this_archive)
+
+        # iterate through list of archives and parse
+        for archive_item in archives:
+            archive_info = {}
+
+            # 1st line
+            # Archive:  log4j-core-2.16.0.jar
+            line = archive_item.pop(0)
+            _, archive = line.split()
+
+            # 2nd line
+            # Zip file size: 1789565 bytes, number of entries: 1218
+            line = archive_item.pop(0)
+            _, _, _, size, size_unit, *_, number_entries = line.split()
+            size_unit = size_unit.rstrip(',')
+
+            # last line
+            # 1218 files, 3974141 bytes uncompressed, 1515455 bytes compressed:  61.9%
+            line = archive_item.pop(-1)
+            number_files, _, bytes_uncompressed, _, _, bytes_compressed, *_, percent_compressed = line.split()
+            percent_compressed = percent_compressed.rstrip("%")
+
+            # Add header row for parsing
+            archive_item[:0] = ['flags zipversion zipunder filesize type method date time filename']
+
+            file_list = jc.parsers.universal.simple_table_parse(archive_item)
+
+            archive_info = {
+                'archive': archive,
+                'size': size,
+                'size_unit': size_unit,
+                'number_entries': number_entries,
+                'number_files': number_files,
+                'bytes_uncompressed': bytes_uncompressed,
+                'bytes_compressed': bytes_compressed,
+                'percent_compressed': percent_compressed,
+                'files': file_list
+            }
+
+            raw_output.append(archive_info)
+
+    return raw_output if raw else _process(raw_output)

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2021-12-08 1.17.4 "JSON CLI output utility"
+.TH jc 1 2021-12-21 1.17.5 "JSON CLI output utility"
 .SH NAME
 jc \- JSONifies the output of many CLI tools and file-types
 .SH SYNOPSIS
@@ -427,6 +427,11 @@ XML file parser
 \fB--yaml\fP
 YAML file parser
 
+.TP
+.B
+\fB--zipinfo\fP
+`zipinfo` command parser
+
 
 .RE
 .PP

setup.py

@@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
 
 setuptools.setup(
     name='jc',
-    version='1.17.4',
+    version='1.17.5',
     author='Kelly Brazil',
     author_email='kellyjonbrazil@gmail.com',
     description='Converts the output of popular command-line tools and file-types to JSON.',

tests/fixtures/osx-10.14.6/zipinfo-multi.json

@@ -0,0 +1 @@
+[{"archive":"jc1.zip","size":4116,"size_unit":"bytes","number_entries":1,"number_files":1,"bytes_uncompressed":11837,"bytes_compressed":3966,"percent_compressed":66.5,"files":[{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":11837,"type":"bX","method":"defN","date":"21-Dec-08","time":"20:50","filename":"jc.1"}]},{"archive":"testzip.zip","size":8106,"size_unit":"bytes","number_entries":2,"number_files":2,"bytes_uncompressed":8539,"bytes_compressed":7651,"percent_compressed":10.4,"files":[{"flags":"-rw-r--r--","zipversion":"3.0","zipunder":"unx","filesize":197,"type":"tx","method":"defN","date":"21-Aug-03","time":"15:12","filename":"round-table.gv"},{"flags":"-rw-r--r--","zipversion":"3.0","zipunder":"unx","filesize":8342,"type":"bx","method":"defN","date":"21-Aug-03","time":"15:12","filename":"round-table.gv.pdf"}]},{"archive":"micro.zip","size":6144,"size_unit":"bytes","number_entries":8,"number_files":8,"bytes_uncompressed":22839,"bytes_compressed":4908,"percent_compressed":78.5,"files":[{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":10688,"type":"bX","method":"defN","date":"19-Sep-30","time":"16:47","filename":"microsimservermac.py"},{"flags":"drwxrwxr-x","zipversion":"2.1","zipunder":"unx","filesize":0,"type":"bx","method":"stor","date":"21-Dec-20","time":"14:33","filename":"__MACOSX/"},{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":176,"type":"bX","method":"defN","date":"19-Sep-30","time":"16:47","filename":"__MACOSX/._microsimservermac.py"},{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":528,"type":"bX","method":"defN","date":"19-Aug-27","time":"07:46","filename":"Dockerfile"},{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":10538,"type":"bX","method":"defN","date":"19-Oct-01","time":"13:22","filename":"microsimserver.py"},{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":380,"type":"bX","method":"defN","date":"19-Oct-01","time":"13:22","filename":"changelog.txt"},{"flags":"-rwxr-xr-x","zipversion":"2.1","zipunder":"unx","filesize":263,"type":"bX","method":"defN","date":"19-Oct-01","time":"12:09","filename":"dockerhub.sh"},{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":266,"type":"bX","method":"defN","date":"19-Oct-01","time":"12:09","filename":"__MACOSX/._dockerhub.sh"}]}]

tests/fixtures/osx-10.14.6/zipinfo-multi.out

@@ -0,0 +1,24 @@
+Archive:  jc1.zip
+Zip file size: 4116 bytes, number of entries: 1
+-rw-r--r--  2.1 unx    11837 bX defN 21-Dec-08 20:50 jc.1
+1 file, 11837 bytes uncompressed, 3966 bytes compressed:  66.5%
+
+Archive:  testzip.zip
+Zip file size: 8106 bytes, number of entries: 2
+-rw-r--r--  3.0 unx      197 tx defN 21-Aug-03 15:12 round-table.gv
+-rw-r--r--  3.0 unx     8342 bx defN 21-Aug-03 15:12 round-table.gv.pdf
+2 files, 8539 bytes uncompressed, 7651 bytes compressed:  10.4%
+
+Archive:  micro.zip
+Zip file size: 6144 bytes, number of entries: 8
+-rw-r--r--  2.1 unx    10688 bX defN 19-Sep-30 16:47 microsimservermac.py
+drwxrwxr-x  2.1 unx        0 bx stor 21-Dec-20 14:33 __MACOSX/
+-rw-r--r--  2.1 unx      176 bX defN 19-Sep-30 16:47 __MACOSX/._microsimservermac.py
+-rw-r--r--  2.1 unx      528 bX defN 19-Aug-27 07:46 Dockerfile
+-rw-r--r--  2.1 unx    10538 bX defN 19-Oct-01 13:22 microsimserver.py
+-rw-r--r--  2.1 unx      380 bX defN 19-Oct-01 13:22 changelog.txt
+-rwxr-xr-x  2.1 unx      263 bX defN 19-Oct-01 12:09 dockerhub.sh
+-rw-r--r--  2.1 unx      266 bX defN 19-Oct-01 12:09 __MACOSX/._dockerhub.sh
+8 files, 22839 bytes uncompressed, 4908 bytes compressed:  78.5%
+
+3 archives were successfully processed.

tests/test_zipinfo.py

@@ -0,0 +1,46 @@
+import os
+import unittest
+import json
+import jc.parsers.zipinfo
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class MyTests(unittest.TestCase):
+
+    def setUp(self):
+        # input
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/rhel-8/zipinfo.out'), 'r', encoding='utf-8') as f:
+            self.rhel_8_zipinfo = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/zipinfo-multi.out'), 'r', encoding='utf-8') as f:
+            self.osx_10_14_6_zipinfo_multi = f.read()
+
+        # output
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/rhel-8/zipinfo.json'), 'r', encoding='utf-8') as f:
+            self.rhel_8_zipinfo_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/zipinfo-multi.json'), 'r', encoding='utf-8') as f:
+            self.osx_10_14_6_zipinfo_multi_json = json.loads(f.read())
+
+    def test_zipinfo_nodata(self):
+        """
+        Test 'zipinfo' parser with no data
+        """
+        self.assertEqual(jc.parsers.zipinfo.parse('', quiet=True), [])
+
+    def test_zipinfo_rhel_8(self):
+        """
+        Test 'zipinfo' on Red Hat 8
+        """
+        self.assertEqual(jc.parsers.zipinfo.parse(self.rhel_8_zipinfo, quiet=True), self.rhel_8_zipinfo_json)
+
+    def test_zipinfo_multi_osx_10_14_6(self):
+        """
+        Test 'zipinfo' with multiple archives on OSX 10.14.6
+        """
+        self.assertEqual(jc.parsers.zipinfo.parse(self.osx_10_14_6_zipinfo_multi, quiet=True), self.osx_10_14_6_zipinfo_multi_json)
+
+
+if __name__ == '__main__':
+    unittest.main()