1
0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2026-04-05 17:50:11 +02:00

Compare commits

...

17 Commits

Author SHA1 Message Date
Kelly Brazil
d753e71a74 Merge pull request #186 from kellyjonbrazil/dev
Dev v1.17.5
2021-12-21 15:21:18 -06:00
Kelly Brazil
2e4f5a508b version bump 2021-12-21 12:19:17 -08:00
Kelly Brazil
88b960eff6 doc update 2021-12-21 12:14:20 -08:00
Kelly Brazil
88c77bd89e add zipinfo tests 2021-12-21 12:08:16 -08:00
Kelly Brazil
51a7a4251f add multi-archive test output 2021-12-21 11:11:56 -08:00
Kelly Brazil
51d2f316f3 add multi-archive support 2021-12-21 11:11:44 -08:00
Kelly Brazil
ff78a46c48 add zipinfo parser 2021-12-21 08:13:17 -08:00
Kelly Brazil
ed4a9dc1d4 formatting 2021-12-21 08:13:00 -08:00
Kelly Brazil
63182dba26 Merge pull request #185 from listuser/new_branch
contributed zipinfo parser
2021-12-21 10:08:46 -06:00
Matt J
9c1eaa9389 revised zipinfo.py nested version 2021-12-20 21:53:34 -08:00
Matt J
bc520fcbcd added zipinfo.py nested version 2021-12-20 14:29:50 -08:00
Matt J
46faac1a12 add test data zipinfo.json and zipinfo.out 2021-12-19 18:08:11 -08:00
Matt J
3c424c0cb3 initial commit zipinfo.py to new_branch 2021-12-19 14:05:48 -08:00
Kelly Brazil
3ac8d0362b use quotes around python versions 2021-12-16 07:04:35 -08:00
Kelly Brazil
d88b998e6c formatting 2021-12-09 10:58:06 -08:00
Kelly Brazil
a9ed55c006 fix spelling 2021-12-09 10:54:04 -08:00
Kelly Brazil
ea61434123 fix schema docs 2021-12-09 10:21:37 -08:00
30 changed files with 1683 additions and 30 deletions

View File

@@ -14,7 +14,7 @@ jobs:
strategy:
matrix:
os: [macos-latest, ubuntu-latest, windows-latest]
python-version: [3.7, 3.8, 3.9, 3.10.0]
python-version: ["3.7", "3.8", "3.9", "3.10"]
steps:
- uses: actions/checkout@v2

View File

@@ -1,5 +1,8 @@
jc changelog
20211221 v1.17.5
- Add zipinfo parser tested on linux and macOS
20211207 v1.17.4
- Add support for the NO_COLOR environment variable to set mono (http://no-color.org/)
- Add -C option to force color output even when using pipes (overrides -m and NO_COLOR)

View File

@@ -3793,5 +3793,36 @@ cat istio.yaml | jc --yaml -p
}
]
```
### zipinfo
```bash
zipinfo file.zip | jc --zipinfo -p # or: jc -p zipinfo file.zip
```
```json
[
{
"archive": "file.zip",
"size": 4116,
"size_unit": "bytes",
"number_entries": 1,
"number_files": 1,
"bytes_uncompressed": 11837,
"bytes_compressed": 3966,
"percent_compressed": 66.5,
"files": [
{
"flags": "-rw-r--r--",
"zipversion": "2.1",
"zipunder": "unx",
"filesize": 11837,
"type": "bX",
"method": "defN",
"date": "21-Dec-08",
"time": "20:50",
"filename": "compressed_file"
}
]
}
]
```
© 2019-2021 Kelly Brazil

View File

@@ -188,6 +188,7 @@ The JSON output can be compact (default) or pretty formatted with the `-p` optio
- `--who` enables the `who` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/who))
- `--xml` enables the XML file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/xml))
- `--yaml` enables the YAML file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/yaml))
- `--zipinfo` enables the `zipinfo` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/zipinfo))
### Options
- `-a` about `jc`. Prints information about `jc` and the parsers (in JSON, of course!)

View File

@@ -22,8 +22,10 @@ Schema:
{
"variables": [
"name": string,
"value": string
{
"name": string,
"value": string
}
],
"schedule": [
{

View File

@@ -18,8 +18,10 @@ Schema:
{
"variables": [
"name": string,
"value": string
{
"name": string,
"value": string
}
],
"schedule": [
{

View File

@@ -27,11 +27,11 @@ Schema:
{
"column_name1": string,
"column_name2": string,
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}
}

View File

@@ -73,7 +73,7 @@ Schema:
"percent_wrqm": float,
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}

View File

@@ -39,7 +39,7 @@ Schema:
"epoch_utc": integer, # timezone aware timestamp if date field is in UTC and can be converted
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}

View File

@@ -46,7 +46,7 @@ Schema:
"round_trip_ms_stddev": float,
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}

View File

@@ -63,7 +63,7 @@ Schema:
"epoch_utc": integer # aware timestamp if -t flag is used and UTC TZ
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}

106
docs/parsers/zipinfo.md Normal file
View File

@@ -0,0 +1,106 @@
[Home](https://kellyjonbrazil.github.io/jc/)
# jc.parsers.zipinfo
jc - JSON CLI output utility `zipinfo` command output parser
Options supported:
- none
Note: The default listing format.
Usage (cli):
$ zipinfo <archive> | jc --zipinfo
or
$ jc zipinfo
Usage (module):
import jc.parsers.zipinfo
result = jc.parsers.zipinfo.parse(zipinfo_command_output)
Schema:
[
{
"archive": string,
"size": integer,
"size_unit": string,
"number_entries": integer,
"number_files": integer,
"bytes_uncompressed": integer,
"bytes_compressed": integer,
"percent_compressed": float,
"files": [
{
"flags": string,
"zipversion": string,
"zipunder": string
"filesize": integer,
"type": string,
"method": string,
"date": string,
"time": string,
"filename": string
}
]
}
]
Examples:
$ zipinfo log4j-core-2.16.0.jar | jc --zipinfo -p
[
{
"archive": "log4j-core-2.16.0.jar",
"size": 1789565,
"size_unit": "bytes",
"number_entries": 1218,
"number_files": 1218,
"bytes_uncompressed": 3974141,
"bytes_compressed": 1515455,
"percent_compressed": 61.9,
"files": [
{
"flags": "-rw-r--r--",
"zipversion": "2.0",
"zipunder": "unx",
"filesize": 19810,
"type": "bl",
"method": "defN",
"date": "21-Dec-12",
"time": "23:35",
"filename": "META-INF/MANIFEST.MF"
},
...
## info
```python
info()
```
Provides parser metadata (version, author, etc.)
## parse
```python
parse(data, raw=False, quiet=False)
```
Main text parsing function
Parameters:
data: (string) text data to parse
raw: (boolean) output preprocessed JSON if True
quiet: (boolean) suppress warning messages if True
Returns:
List of Dictionaries. Raw or processed structured data.
## Parser Information
Compatibility: linux, darwin
Version 0.01 by Matt J (https://github.com/listuser)

View File

@@ -73,4 +73,4 @@ Module Example:
"""
name = 'jc'
__version__ = '1.17.4'
__version__ = '1.17.5'

View File

@@ -125,7 +125,8 @@ parsers = [
'wc',
'who',
'xml',
'yaml'
'yaml',
'zipinfo'
]
JC_ERROR_EXIT = 100

View File

@@ -19,8 +19,10 @@ Schema:
{
"variables": [
"name": string,
"value": string
{
"name": string,
"value": string
}
],
"schedule": [
{

View File

@@ -15,8 +15,10 @@ Schema:
{
"variables": [
"name": string,
"value": string
{
"name": string,
"value": string
}
],
"schedule": [
{

View File

@@ -24,11 +24,11 @@ Schema:
{
"column_name1": string,
"column_name2": string,
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}
}

View File

@@ -21,7 +21,7 @@ Schema:
"foo": string,
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}

View File

@@ -70,7 +70,7 @@ Schema:
"percent_wrqm": float,
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}

View File

@@ -36,7 +36,7 @@ Schema:
"epoch_utc": integer, # timezone aware timestamp if date field is in UTC and can be converted
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}

View File

@@ -43,7 +43,7 @@ Schema:
"round_trip_ms_stddev": float,
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}

View File

@@ -60,7 +60,7 @@ Schema:
"epoch_utc": integer # aware timestamp if -t flag is used and UTC TZ
"_jc_meta": # This object only exists if using -qq or ignore_exceptions=True
{
"success": booean, # true if successfully parsed, false if error
"success": boolean, # true if successfully parsed, false if error
"error": string, # exists if "success" is false
"line": string # exists if "success" is false
}

205
jc/parsers/zipinfo.py Normal file
View File

@@ -0,0 +1,205 @@
"""jc - JSON CLI output utility `zipinfo` command output parser
Options supported:
- none
Note: The default listing format.
Usage (cli):
$ zipinfo <archive> | jc --zipinfo
or
$ jc zipinfo
Usage (module):
import jc.parsers.zipinfo
result = jc.parsers.zipinfo.parse(zipinfo_command_output)
Schema:
[
{
"archive": string,
"size": integer,
"size_unit": string,
"number_entries": integer,
"number_files": integer,
"bytes_uncompressed": integer,
"bytes_compressed": integer,
"percent_compressed": float,
"files": [
{
"flags": string,
"zipversion": string,
"zipunder": string
"filesize": integer,
"type": string,
"method": string,
"date": string,
"time": string,
"filename": string
}
]
}
]
Examples:
$ zipinfo log4j-core-2.16.0.jar | jc --zipinfo -p
[
{
"archive": "log4j-core-2.16.0.jar",
"size": 1789565,
"size_unit": "bytes",
"number_entries": 1218,
"number_files": 1218,
"bytes_uncompressed": 3974141,
"bytes_compressed": 1515455,
"percent_compressed": 61.9,
"files": [
{
"flags": "-rw-r--r--",
"zipversion": "2.0",
"zipunder": "unx",
"filesize": 19810,
"type": "bl",
"method": "defN",
"date": "21-Dec-12",
"time": "23:35",
"filename": "META-INF/MANIFEST.MF"
},
...
"""
import jc.utils
import jc.parsers.universal
class info():
"""Provides parser metadata (version, author, etc.)"""
version = '0.01'
description = '`zipinfo` command parser'
author = 'Matt J'
author_email = 'https://github.com/listuser'
compatible = ['linux', 'darwin']
magic_commands = ['zipinfo']
__version__ = info.version
def _process(proc_data):
"""
Final processing to conform to the schema.
Parameters:
proc_data: (List of Dictionaries) raw structured data to process
Returns:
List of Dictionaries. Structured data to conform to the schema.
"""
for entry in proc_data:
int_list = ['bytes_compressed', 'bytes_uncompressed', 'number_entries',
'number_files', 'size', 'filesize']
float_list = ['percent_compressed']
for key in entry:
if key in int_list:
entry[key] = jc.utils.convert_to_int(entry[key])
if key in float_list:
entry[key] = jc.utils.convert_to_float(entry[key])
if 'files' in key:
for item in entry['files']:
for key in item:
if key in int_list:
item[key] = jc.utils.convert_to_int(item[key])
return proc_data
def parse(data, raw=False, quiet=False):
"""
Main text parsing function
Parameters:
data: (string) text data to parse
raw: (boolean) output preprocessed JSON if True
quiet: (boolean) suppress warning messages if True
Returns:
List of Dictionaries. Raw or processed structured data.
"""
jc.utils.compatibility(__name__, info.compatible, quiet)
jc.utils.input_type_check(data)
raw_output = []
archives = []
if jc.utils.has_data(data):
datalines = data.splitlines()
# remove last line of multi-archive output since it is not needed
if datalines[-1].endswith('archives were successfully processed.'):
datalines.pop(-1)
# extract each archive into its own list of lines.
# archives are separated by a blank line
this_archive = []
for row in datalines:
if row == '':
archives.append(this_archive)
this_archive = []
continue
this_archive.append(row)
if this_archive:
archives.append(this_archive)
# iterate through list of archives and parse
for archive_item in archives:
archive_info = {}
# 1st line
# Archive: log4j-core-2.16.0.jar
line = archive_item.pop(0)
_, archive = line.split()
# 2nd line
# Zip file size: 1789565 bytes, number of entries: 1218
line = archive_item.pop(0)
_, _, _, size, size_unit, *_, number_entries = line.split()
size_unit = size_unit.rstrip(',')
# last line
# 1218 files, 3974141 bytes uncompressed, 1515455 bytes compressed: 61.9%
line = archive_item.pop(-1)
number_files, _, bytes_uncompressed, _, _, bytes_compressed, *_, percent_compressed = line.split()
percent_compressed = percent_compressed.rstrip("%")
# Add header row for parsing
archive_item[:0] = ['flags zipversion zipunder filesize type method date time filename']
file_list = jc.parsers.universal.simple_table_parse(archive_item)
archive_info = {
'archive': archive,
'size': size,
'size_unit': size_unit,
'number_entries': number_entries,
'number_files': number_files,
'bytes_uncompressed': bytes_uncompressed,
'bytes_compressed': bytes_compressed,
'percent_compressed': percent_compressed,
'files': file_list
}
raw_output.append(archive_info)
return raw_output if raw else _process(raw_output)

View File

@@ -1,4 +1,4 @@
.TH jc 1 2021-12-08 1.17.4 "JSON CLI output utility"
.TH jc 1 2021-12-21 1.17.5 "JSON CLI output utility"
.SH NAME
jc \- JSONifies the output of many CLI tools and file-types
.SH SYNOPSIS
@@ -427,6 +427,11 @@ XML file parser
\fB--yaml\fP
YAML file parser
.TP
.B
\fB--zipinfo\fP
`zipinfo` command parser
.RE
.PP

View File

@@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
setuptools.setup(
name='jc',
version='1.17.4',
version='1.17.5',
author='Kelly Brazil',
author_email='kellyjonbrazil@gmail.com',
description='Converts the output of popular command-line tools and file-types to JSON.',

View File

@@ -0,0 +1 @@
[{"archive":"jc1.zip","size":4116,"size_unit":"bytes","number_entries":1,"number_files":1,"bytes_uncompressed":11837,"bytes_compressed":3966,"percent_compressed":66.5,"files":[{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":11837,"type":"bX","method":"defN","date":"21-Dec-08","time":"20:50","filename":"jc.1"}]},{"archive":"testzip.zip","size":8106,"size_unit":"bytes","number_entries":2,"number_files":2,"bytes_uncompressed":8539,"bytes_compressed":7651,"percent_compressed":10.4,"files":[{"flags":"-rw-r--r--","zipversion":"3.0","zipunder":"unx","filesize":197,"type":"tx","method":"defN","date":"21-Aug-03","time":"15:12","filename":"round-table.gv"},{"flags":"-rw-r--r--","zipversion":"3.0","zipunder":"unx","filesize":8342,"type":"bx","method":"defN","date":"21-Aug-03","time":"15:12","filename":"round-table.gv.pdf"}]},{"archive":"micro.zip","size":6144,"size_unit":"bytes","number_entries":8,"number_files":8,"bytes_uncompressed":22839,"bytes_compressed":4908,"percent_compressed":78.5,"files":[{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":10688,"type":"bX","method":"defN","date":"19-Sep-30","time":"16:47","filename":"microsimservermac.py"},{"flags":"drwxrwxr-x","zipversion":"2.1","zipunder":"unx","filesize":0,"type":"bx","method":"stor","date":"21-Dec-20","time":"14:33","filename":"__MACOSX/"},{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":176,"type":"bX","method":"defN","date":"19-Sep-30","time":"16:47","filename":"__MACOSX/._microsimservermac.py"},{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":528,"type":"bX","method":"defN","date":"19-Aug-27","time":"07:46","filename":"Dockerfile"},{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":10538,"type":"bX","method":"defN","date":"19-Oct-01","time":"13:22","filename":"microsimserver.py"},{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":380,"type":"bX","method":"defN","date":"19-Oct-01","time":"13:22","filename":"changelog.txt"},{"flags":"-rwxr-xr-x","zipversion":"2.1","zipunder":"unx","filesize":263,"type":"bX","method":"defN","date":"19-Oct-01","time":"12:09","filename":"dockerhub.sh"},{"flags":"-rw-r--r--","zipversion":"2.1","zipunder":"unx","filesize":266,"type":"bX","method":"defN","date":"19-Oct-01","time":"12:09","filename":"__MACOSX/._dockerhub.sh"}]}]

View File

@@ -0,0 +1,24 @@
Archive: jc1.zip
Zip file size: 4116 bytes, number of entries: 1
-rw-r--r-- 2.1 unx 11837 bX defN 21-Dec-08 20:50 jc.1
1 file, 11837 bytes uncompressed, 3966 bytes compressed: 66.5%
Archive: testzip.zip
Zip file size: 8106 bytes, number of entries: 2
-rw-r--r-- 3.0 unx 197 tx defN 21-Aug-03 15:12 round-table.gv
-rw-r--r-- 3.0 unx 8342 bx defN 21-Aug-03 15:12 round-table.gv.pdf
2 files, 8539 bytes uncompressed, 7651 bytes compressed: 10.4%
Archive: micro.zip
Zip file size: 6144 bytes, number of entries: 8
-rw-r--r-- 2.1 unx 10688 bX defN 19-Sep-30 16:47 microsimservermac.py
drwxrwxr-x 2.1 unx 0 bx stor 21-Dec-20 14:33 __MACOSX/
-rw-r--r-- 2.1 unx 176 bX defN 19-Sep-30 16:47 __MACOSX/._microsimservermac.py
-rw-r--r-- 2.1 unx 528 bX defN 19-Aug-27 07:46 Dockerfile
-rw-r--r-- 2.1 unx 10538 bX defN 19-Oct-01 13:22 microsimserver.py
-rw-r--r-- 2.1 unx 380 bX defN 19-Oct-01 13:22 changelog.txt
-rwxr-xr-x 2.1 unx 263 bX defN 19-Oct-01 12:09 dockerhub.sh
-rw-r--r-- 2.1 unx 266 bX defN 19-Oct-01 12:09 __MACOSX/._dockerhub.sh
8 files, 22839 bytes uncompressed, 4908 bytes compressed: 78.5%
3 archives were successfully processed.

1
tests/fixtures/rhel-8/zipinfo.json vendored Normal file

File diff suppressed because one or more lines are too long

1221
tests/fixtures/rhel-8/zipinfo.out vendored Normal file

File diff suppressed because it is too large Load Diff

46
tests/test_zipinfo.py Normal file
View File

@@ -0,0 +1,46 @@
import os
import unittest
import json
import jc.parsers.zipinfo
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
class MyTests(unittest.TestCase):
def setUp(self):
# input
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/rhel-8/zipinfo.out'), 'r', encoding='utf-8') as f:
self.rhel_8_zipinfo = f.read()
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/zipinfo-multi.out'), 'r', encoding='utf-8') as f:
self.osx_10_14_6_zipinfo_multi = f.read()
# output
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/rhel-8/zipinfo.json'), 'r', encoding='utf-8') as f:
self.rhel_8_zipinfo_json = json.loads(f.read())
with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/zipinfo-multi.json'), 'r', encoding='utf-8') as f:
self.osx_10_14_6_zipinfo_multi_json = json.loads(f.read())
def test_zipinfo_nodata(self):
"""
Test 'zipinfo' parser with no data
"""
self.assertEqual(jc.parsers.zipinfo.parse('', quiet=True), [])
def test_zipinfo_rhel_8(self):
"""
Test 'zipinfo' on Red Hat 8
"""
self.assertEqual(jc.parsers.zipinfo.parse(self.rhel_8_zipinfo, quiet=True), self.rhel_8_zipinfo_json)
def test_zipinfo_multi_osx_10_14_6(self):
"""
Test 'zipinfo' with multiple archives on OSX 10.14.6
"""
self.assertEqual(jc.parsers.zipinfo.parse(self.osx_10_14_6_zipinfo_multi, quiet=True), self.osx_10_14_6_zipinfo_multi_json)
if __name__ == '__main__':
unittest.main()