doc update

CHANGELOG

@@ -1,5 +1,11 @@
 jc changelog
 
+20210910 v1.17.0
+- Note to Package Maintainers: please see note at 20210720 v1.16.0
+- Add generic ASCII tables parser
+- Add support for streaming parsers
+- Add ls command streaming parser tested on linux, macOS, and freeBSD
+
 20210830 v1.16.2
 - Note to Package Maintainers: please see note at 20210720 v1.16.0
 - Update sfdisk parser to support the -F option and newer versions of sfdisk

README.md

@@ -141,6 +141,7 @@ The JSON output can be compact (default) or pretty formatted with the `-p` optio
 - `--kv` enables the Key/Value file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/kv))
 - `--last` enables the `last` and `lastb` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/last))
 - `--ls` enables the `ls` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ls))
+- `--ls-s` enables the `ls` command streaming parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ls_s))
 - `--lsblk` enables the `lsblk` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/lsblk))
 - `--lsmod` enables the `lsmod` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/lsmod))
 - `--lsof` enables the `lsof` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/lsof))

docs/parsers/ls_s.md

@@ -0,0 +1,89 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+
+# jc.parsers.ls_s
+jc - JSON CLI output utility `ls` and `vdir` command output streaming parser
+
+Options supported:
+- `lbaR1`
+- `--time-style=full-iso`
+
+Note: The `-1`, `-l`, or `-b` option of `ls` should be used to correctly parse filenames that include newline characters. Since `ls` does not encode newlines in filenames when outputting to a pipe it will cause `jc` to see multiple files instead of a single file if `-1`, `-l`, or `-b` is not used. Alternatively, `vdir` can be used, which is the same as running `ls -lb`.
+
+The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on)
+
+The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC.
+
+Usage (cli):
+
+    $ ls | jc --ls-s
+
+Usage (module):
+
+    import jc.parsers.ls_s
+    result = jc.parsers.ls_s.parse(ls_command_output)    # result is an iterable object
+    for item in result:
+        # do something
+
+Schema:
+
+    {
+      "filename":     string,
+      "flags":        string,
+      "links":        integer,
+      "parent":       string,      # not yet implemented
+      "owner":        string,
+      "group":        string,
+      "size":         integer,
+      "date":         string,
+      "epoch":        integer,     # naive timestamp if date field exists and can be converted
+      "epoch_utc":    integer,     # timezone aware timestamp if date field is in UTC and can be converted
+      "_meta":
+        {
+          "success":    booean,    # true if successfully parsed, false if error
+          "error_msg":  string,    # exists if "success" is false
+          "line":       string     # exists if "success" is false
+        }
+    }
+
+Examples:
+
+    $ ls -l /usr/bin | jc --ls-s
+    {"filename":"2to3-","flags":"-rwxr-xr-x","links":4,"owner":"root","group":"wheel","size":925,"date":"Feb 22 2019","_meta":{"success":true}}
+    {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7","flags":"lrwxr-xr-x","links":1,"owner":"root","group":"wheel","size":74,"date":"May 4 2019","_meta":{"success":true}}
+    {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":1,"owner":"root","group":"wheel","size":55152,"date":"May 3 2019","_meta":{"success":true}}
+    ...
+
+    $ ls -l /usr/bin | jc --ls-s -r
+    {"filename":"2to3-","flags":"-rwxr-xr-x","links":"4","owner":"root","group":"wheel","size":"925","date":"Feb 22 2019","_meta":{"success":true}}
+    {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7","flags":"lrwxr-xr-x","links":"1","owner":"root","group":"wheel","size":"74","date":"May 4 2019","_meta":{"success":true}}
+    {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":"1","owner":"root","group":"wheel","size":"55152","date":"May 3 2019","_meta":{"success":true}}
+    ...
+
+
+## info
+```python
+info()
+```
+Provides parser metadata (version, author, etc.)
+
+## parse
+```python
+parse(data, raw=False, quiet=False)
+```
+
+Main text parsing function
+
+Parameters:
+
+    data:        (string)  line-based text data to parse
+    raw:         (boolean) output preprocessed JSON if True
+    quiet:       (boolean) suppress warning messages if True
+
+Returns:
+
+    List of Dictionaries. Raw or processed structured data.
+
+## Parser Information
+Compatibility:  linux, darwin, cygwin, aix, freebsd
+
+Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

docs/readme.md

@@ -59,26 +59,13 @@ Module Example:
 
     >>> import jc.parsers.dig
     >>>
-    >>> data = '''; <<>> DiG 9.10.6 <<>> example.com
+    >>> import subprocess
-    ... ;; global options: +cmd
+    >>> import jc.parsers.dig
-    ... ;; Got answer:
-    ... ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 64612
-    ... ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1
-    ...
-    ... ;; OPT PSEUDOSECTION:
-    ... ; EDNS: version: 0, flags:; udp: 4096
-    ... ;; QUESTION SECTION:
-    ... ;example.com.           IN  A
-    ...
-    ... ;; ANSWER SECTION:
-    ... example.com.        29658   IN  A   93.184.216.34
-    ...
-    ... ;; Query time: 52 msec
-    ... ;; SERVER: 2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)
-    ... ;; WHEN: Fri Apr 16 16:13:00 PDT 2021
-    ... ;; MSG SIZE  rcvd: 56'''
     >>>
-    >>> jc.parsers.dig.parse(data)
+    >>> cmd_output = subprocess.check_output(['dig', 'example.com'], text=True)
+    >>> data = jc.parsers.dig.parse(cmd_output)
+    >>>
+    >>> data
     [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', 'flags': ['qr', 'rd', 'ra'], 'query_num': 1, 'answer_num':
     1, 'authority_num': 0, 'additional_num': 1, 'opt_pseudosection': {'edns': {'version': 0, 'flags': [], 'udp':
     4096}}, 'question': {'name': 'example.com.', 'class': 'IN', 'type': 'A'}, 'answer': [{'name': 'example.com.',

jc/__init__.py

@@ -57,26 +57,13 @@ Module Example:
 
     >>> import jc.parsers.dig
     >>>
-    >>> data = '''; <<>> DiG 9.10.6 <<>> example.com
+    >>> import subprocess
-    ... ;; global options: +cmd
+    >>> import jc.parsers.dig
-    ... ;; Got answer:
+    >>> 
-    ... ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 64612
+    >>> cmd_output = subprocess.check_output(['dig', 'example.com'], text=True)
-    ... ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1
+    >>> data = jc.parsers.dig.parse(cmd_output)
-    ...
-    ... ;; OPT PSEUDOSECTION:
-    ... ; EDNS: version: 0, flags:; udp: 4096
-    ... ;; QUESTION SECTION:
-    ... ;example.com.           IN  A
-    ...
-    ... ;; ANSWER SECTION:
-    ... example.com.        29658   IN  A   93.184.216.34
-    ...
-    ... ;; Query time: 52 msec
-    ... ;; SERVER: 2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)
-    ... ;; WHEN: Fri Apr 16 16:13:00 PDT 2021
-    ... ;; MSG SIZE  rcvd: 56'''
     >>>
-    >>> jc.parsers.dig.parse(data)
+    >>> data
     [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', 'flags': ['qr', 'rd', 'ra'], 'query_num': 1, 'answer_num':
     1, 'authority_num': 0, 'additional_num': 1, 'opt_pseudosection': {'edns': {'version': 0, 'flags': [], 'udp':
     4096}}, 'question': {'name': 'example.com.', 'class': 'IN', 'type': 'A'}, 'answer': [{'name': 'example.com.',
@@ -86,4 +73,4 @@ Module Example:
 """
 
 name = 'jc'
-__version__ = '1.16.2'
+__version__ = '1.17.0'

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2021-08-31 1.16.2 "JSON CLI output utility"
+.TH jc 1 2021-09-10 1.17.0 "JSON CLI output utility"
 .SH NAME
 jc \- JSONifies the output of many CLI tools and file-types
 .SH SYNOPSIS
@@ -202,6 +202,11 @@ Key/Value file parser
 \fB--ls\fP
 `ls` command parser
 
+.TP
+.B
+\fB--ls-s\fP
+`ls` command streaming parser
+
 .TP
 .B
 \fB--lsblk\fP

mangen.py

@@ -15,11 +15,3 @@ output = template.render(today=date.today(),
 
 with open('man/jc.1', 'w') as f:
     f.write(output)
-
-with open('man/jc.1', 'rb') as f_in:
-    with gzip.open('man/jc.1.gz', 'wb') as f_out:
-        shutil.copyfileobj(f_in, f_out)
-
-shutil.copyfile('man/jc.1.gz', 'jc/man/jc.1.gz')
-
-# os.remove('man/jc.1')

setup.py

@@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
 
 setuptools.setup(
     name='jc',
-    version='1.16.2',
+    version='1.17.0',
     author='Kelly Brazil',
     author_email='kellyjonbrazil@gmail.com',
     description='Converts the output of popular command-line tools and file-types to JSON.',