diff --git a/CHANGELOG b/CHANGELOG index 65217c36..e23c6912 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,11 @@ jc changelog +20210910 v1.17.0 +- Note to Package Maintainers: please see note at 20210720 v1.16.0 +- Add generic ASCII tables parser +- Add support for streaming parsers +- Add ls command streaming parser tested on linux, macOS, and freeBSD + 20210830 v1.16.2 - Note to Package Maintainers: please see note at 20210720 v1.16.0 - Update sfdisk parser to support the -F option and newer versions of sfdisk diff --git a/README.md b/README.md index 0344e460..1e2f976c 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,7 @@ The JSON output can be compact (default) or pretty formatted with the `-p` optio - `--kv` enables the Key/Value file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/kv)) - `--last` enables the `last` and `lastb` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/last)) - `--ls` enables the `ls` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ls)) +- `--ls-s` enables the `ls` command streaming parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ls_s)) - `--lsblk` enables the `lsblk` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/lsblk)) - `--lsmod` enables the `lsmod` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/lsmod)) - `--lsof` enables the `lsof` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/lsof)) diff --git a/docs/parsers/ls_s.md b/docs/parsers/ls_s.md new file mode 100644 index 00000000..3e406cbb --- /dev/null +++ b/docs/parsers/ls_s.md @@ -0,0 +1,89 @@ +[Home](https://kellyjonbrazil.github.io/jc/) + +# jc.parsers.ls_s +jc - JSON CLI output utility `ls` and `vdir` command output streaming parser + +Options supported: +- `lbaR1` +- `--time-style=full-iso` + +Note: The `-1`, `-l`, or `-b` option of `ls` should be used to correctly parse filenames that include newline characters. Since `ls` does not encode newlines in filenames when outputting to a pipe it will cause `jc` to see multiple files instead of a single file if `-1`, `-l`, or `-b` is not used. Alternatively, `vdir` can be used, which is the same as running `ls -lb`. + +The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) + +The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. + +Usage (cli): + + $ ls | jc --ls-s + +Usage (module): + + import jc.parsers.ls_s + result = jc.parsers.ls_s.parse(ls_command_output) # result is an iterable object + for item in result: + # do something + +Schema: + + { + "filename": string, + "flags": string, + "links": integer, + "parent": string, # not yet implemented + "owner": string, + "group": string, + "size": integer, + "date": string, + "epoch": integer, # naive timestamp if date field exists and can be converted + "epoch_utc": integer, # timezone aware timestamp if date field is in UTC and can be converted + "_meta": + { + "success": booean, # true if successfully parsed, false if error + "error_msg": string, # exists if "success" is false + "line": string # exists if "success" is false + } + } + +Examples: + + $ ls -l /usr/bin | jc --ls-s + {"filename":"2to3-","flags":"-rwxr-xr-x","links":4,"owner":"root","group":"wheel","size":925,"date":"Feb 22 2019","_meta":{"success":true}} + {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7","flags":"lrwxr-xr-x","links":1,"owner":"root","group":"wheel","size":74,"date":"May 4 2019","_meta":{"success":true}} + {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":1,"owner":"root","group":"wheel","size":55152,"date":"May 3 2019","_meta":{"success":true}} + ... + + $ ls -l /usr/bin | jc --ls-s -r + {"filename":"2to3-","flags":"-rwxr-xr-x","links":"4","owner":"root","group":"wheel","size":"925","date":"Feb 22 2019","_meta":{"success":true}} + {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7","flags":"lrwxr-xr-x","links":"1","owner":"root","group":"wheel","size":"74","date":"May 4 2019","_meta":{"success":true}} + {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":"1","owner":"root","group":"wheel","size":"55152","date":"May 3 2019","_meta":{"success":true}} + ... + + +## info +```python +info() +``` +Provides parser metadata (version, author, etc.) + +## parse +```python +parse(data, raw=False, quiet=False) +``` + +Main text parsing function + +Parameters: + + data: (string) line-based text data to parse + raw: (boolean) output preprocessed JSON if True + quiet: (boolean) suppress warning messages if True + +Returns: + + List of Dictionaries. Raw or processed structured data. + +## Parser Information +Compatibility: linux, darwin, cygwin, aix, freebsd + +Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) diff --git a/docs/readme.md b/docs/readme.md index 334f85d7..0f5aa6b5 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -59,26 +59,13 @@ Module Example: >>> import jc.parsers.dig >>> - >>> data = '''; <<>> DiG 9.10.6 <<>> example.com - ... ;; global options: +cmd - ... ;; Got answer: - ... ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 64612 - ... ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 - ... - ... ;; OPT PSEUDOSECTION: - ... ; EDNS: version: 0, flags:; udp: 4096 - ... ;; QUESTION SECTION: - ... ;example.com. IN A - ... - ... ;; ANSWER SECTION: - ... example.com. 29658 IN A 93.184.216.34 - ... - ... ;; Query time: 52 msec - ... ;; SERVER: 2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1) - ... ;; WHEN: Fri Apr 16 16:13:00 PDT 2021 - ... ;; MSG SIZE rcvd: 56''' + >>> import subprocess + >>> import jc.parsers.dig >>> - >>> jc.parsers.dig.parse(data) + >>> cmd_output = subprocess.check_output(['dig', 'example.com'], text=True) + >>> data = jc.parsers.dig.parse(cmd_output) + >>> + >>> data [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', 'flags': ['qr', 'rd', 'ra'], 'query_num': 1, 'answer_num': 1, 'authority_num': 0, 'additional_num': 1, 'opt_pseudosection': {'edns': {'version': 0, 'flags': [], 'udp': 4096}}, 'question': {'name': 'example.com.', 'class': 'IN', 'type': 'A'}, 'answer': [{'name': 'example.com.', diff --git a/jc/__init__.py b/jc/__init__.py index ea00f83d..96b1d5c0 100644 --- a/jc/__init__.py +++ b/jc/__init__.py @@ -57,26 +57,13 @@ Module Example: >>> import jc.parsers.dig >>> - >>> data = '''; <<>> DiG 9.10.6 <<>> example.com - ... ;; global options: +cmd - ... ;; Got answer: - ... ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 64612 - ... ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 - ... - ... ;; OPT PSEUDOSECTION: - ... ; EDNS: version: 0, flags:; udp: 4096 - ... ;; QUESTION SECTION: - ... ;example.com. IN A - ... - ... ;; ANSWER SECTION: - ... example.com. 29658 IN A 93.184.216.34 - ... - ... ;; Query time: 52 msec - ... ;; SERVER: 2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1) - ... ;; WHEN: Fri Apr 16 16:13:00 PDT 2021 - ... ;; MSG SIZE rcvd: 56''' + >>> import subprocess + >>> import jc.parsers.dig + >>> + >>> cmd_output = subprocess.check_output(['dig', 'example.com'], text=True) + >>> data = jc.parsers.dig.parse(cmd_output) >>> - >>> jc.parsers.dig.parse(data) + >>> data [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', 'flags': ['qr', 'rd', 'ra'], 'query_num': 1, 'answer_num': 1, 'authority_num': 0, 'additional_num': 1, 'opt_pseudosection': {'edns': {'version': 0, 'flags': [], 'udp': 4096}}, 'question': {'name': 'example.com.', 'class': 'IN', 'type': 'A'}, 'answer': [{'name': 'example.com.', @@ -86,4 +73,4 @@ Module Example: """ name = 'jc' -__version__ = '1.16.2' +__version__ = '1.17.0' diff --git a/jc/man/jc.1.gz b/jc/man/jc.1.gz deleted file mode 100644 index 25fde223..00000000 Binary files a/jc/man/jc.1.gz and /dev/null differ diff --git a/man/jc.1 b/man/jc.1 index f8ca23df..7a7b407e 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2021-08-31 1.16.2 "JSON CLI output utility" +.TH jc 1 2021-09-10 1.17.0 "JSON CLI output utility" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS @@ -202,6 +202,11 @@ Key/Value file parser \fB--ls\fP `ls` command parser +.TP +.B +\fB--ls-s\fP +`ls` command streaming parser + .TP .B \fB--lsblk\fP diff --git a/man/jc.1.gz b/man/jc.1.gz deleted file mode 100644 index 25fde223..00000000 Binary files a/man/jc.1.gz and /dev/null differ diff --git a/mangen.py b/mangen.py index aaa9759c..ed2629ab 100755 --- a/mangen.py +++ b/mangen.py @@ -15,11 +15,3 @@ output = template.render(today=date.today(), with open('man/jc.1', 'w') as f: f.write(output) - -with open('man/jc.1', 'rb') as f_in: - with gzip.open('man/jc.1.gz', 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - -shutil.copyfile('man/jc.1.gz', 'jc/man/jc.1.gz') - -# os.remove('man/jc.1') diff --git a/setup.py b/setup.py index 5df70139..dfe9d25a 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='1.16.2', + version='1.17.0', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='Converts the output of popular command-line tools and file-types to JSON.',