Merge pull request #195 from kellyjonbrazil/dev

Dev v1.17.7

CHANGELOG

@@ -1,5 +1,8 @@
 jc changelog
 
+20220106 v1.17.7
+- Add stat command streaming parser tested on linux and macOS
+
 20220103 v1.17.6
 - Add jar-manifest file parser (for MANIFEST.MF files)
 - Fix CSV parsers for some files that include doublequotes

README.md

@@ -170,6 +170,7 @@ The JSON output can be compact (default) or pretty formatted with the `-p` optio
 - `--shadow` enables the `/etc/shadow` file parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/shadow))
 - `--ss` enables the `ss` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/ss))
 - `--stat` enables the `stat` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/stat))
+- `--stat-s` enables the `stat` command streaming parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/stat_s))
 - `--sysctl` enables the `sysctl` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/sysctl))
 - `--systemctl` enables the `systemctl` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl))
 - `--systemctl-lj` enables the `systemctl list-jobs` command parser ([documentation](https://kellyjonbrazil.github.io/jc/docs/parsers/systemctl_lj))

docs/parsers/stat_s.md

@@ -0,0 +1,105 @@
+[Home](https://kellyjonbrazil.github.io/jc/)
+
+# jc.parsers.stat_s
+jc - JSON CLI output utility `stat` command output streaming parser
+
+> This streaming parser outputs JSON Lines
+
+The `xxx_epoch` calculated timestamp fields are naive (i.e. based on the local time of the system the parser is run on).
+
+The `xxx_epoch_utc` calculated timestamp fields are timezone-aware and are only available if the timezone field is UTC.
+
+Usage (cli):
+
+    $ stat * | jc --stat-s
+
+Usage (module):
+
+    import jc.parsers.stat_s
+    result = jc.parsers.stat_s.parse(stat_command_output.splitlines())    # result is an iterable object
+    for item in result:
+        # do something
+
+Schema:
+
+    {
+      "file":                     string,
+      "link_to"                   string,
+      "size":                     integer,
+      "blocks":                   integer,
+      "io_blocks":                integer,
+      "type":                     string,
+      "device":                   string,
+      "inode":                    integer,
+      "links":                    integer,
+      "access":                   string,
+      "flags":                    string,
+      "uid":                      integer,
+      "user":                     string,
+      "gid":                      integer,
+      "group":                    string,
+      "access_time":              string,    # - = null
+      "access_time_epoch":        integer,   # naive timestamp
+      "access_time_epoch_utc":    integer,   # timezone-aware timestamp
+      "modify_time":              string,    # - = null
+      "modify_time_epoch":        integer,   # naive timestamp
+      "modify_time_epoch_utc":    integer,   # timezone-aware timestamp
+      "change_time":              string,    # - = null
+      "change_time_epoch":        integer,   # naive timestamp
+      "change_time_epoch_utc":    integer,   # timezone-aware timestamp
+      "birth_time":               string,    # - = null
+      "birth_time_epoch":         integer,   # naive timestamp
+      "birth_time_epoch_utc":     integer,   # timezone-aware timestamp
+      "unix_device":              integer,
+      "rdev":                     integer,
+      "block_size":               integer,
+      "unix_flags":               string,
+      "_jc_meta":                            # This object only exists if using -qq or ignore_exceptions=True
+        {
+          "success":              boolean,   # true if successfully parsed, false if error
+          "error":                string,    # exists if "success" is false
+          "line":                 string     # exists if "success" is false
+        }
+    }
+
+Examples:
+
+    $ stat | jc --stat-s
+    {"file":"(stdin)","unix_device":1027739696,"inode":1155,"flags":"crw--w----","links":1,"user":"kbrazil","group":"tty","rdev":268435456,"size":0,"access_time":"Jan  4 15:27:44 2022","modify_time":"Jan  4 15:27:44 2022","change_time":"Jan  4 15:27:44 2022","birth_time":"Dec 31 16:00:00 1969","block_size":131072,"blocks":0,"unix_flags":"0","access_time_epoch":1641338864,"access_time_epoch_utc":null,"modify_time_epoch":1641338864,"modify_time_epoch_utc":null,"change_time_epoch":1641338864,"change_time_epoch_utc":null,"birth_time_epoch":null,"birth_time_epoch_utc":null}
+
+    $ stat | jc --stat-s -r
+    {"file":"(stdin)","unix_device":"1027739696","inode":"1155","flags":"crw--w----","links":"1","user":"kbrazil","group":"tty","rdev":"268435456","size":"0","access_time":"Jan  4 15:28:08 2022","modify_time":"Jan  4 15:28:08 2022","change_time":"Jan  4 15:28:08 2022","birth_time":"Dec 31 16:00:00 1969","block_size":"131072","blocks":"0","unix_flags":"0"}
+
+
+## info
+```python
+info()
+```
+Provides parser metadata (version, author, etc.)
+
+## parse
+```python
+parse(data, raw=False, quiet=False, ignore_exceptions=False)
+```
+
+Main text parsing generator function. Returns an iterator object.
+
+Parameters:
+
+    data:              (iterable)  line-based text data to parse (e.g. sys.stdin or str.splitlines())
+    raw:               (boolean)   output preprocessed JSON if True
+    quiet:             (boolean)   suppress warning messages if True
+    ignore_exceptions: (boolean)   ignore parsing exceptions if True
+
+Yields:
+
+    Dictionary. Raw or processed structured data.
+
+Returns:
+
+    Iterator object
+
+## Parser Information
+Compatibility:  linux, darwin, freebsd
+
+Version 0.5 by Kelly Brazil (kellyjonbrazil@gmail.com)

jc/__init__.py

@@ -73,4 +73,4 @@ Module Example:
 """
 
 name = 'jc'
-__version__ = '1.17.6'
+__version__ = '1.17.7'

jc/cli.py

@@ -105,6 +105,7 @@ parsers = [
     'shadow',
     'ss',
     'stat',
+    'stat-s',
     'sysctl',
     'systemctl',
     'systemctl-lj',

jc/parsers/stat_s.py

@@ -0,0 +1,280 @@
+"""jc - JSON CLI output utility `stat` command output streaming parser
+
+> This streaming parser outputs JSON Lines
+
+The `xxx_epoch` calculated timestamp fields are naive (i.e. based on the local time of the system the parser is run on).
+
+The `xxx_epoch_utc` calculated timestamp fields are timezone-aware and are only available if the timezone field is UTC.
+
+Usage (cli):
+
+    $ stat * | jc --stat-s
+
+Usage (module):
+
+    import jc.parsers.stat_s
+    result = jc.parsers.stat_s.parse(stat_command_output.splitlines())    # result is an iterable object
+    for item in result:
+        # do something
+
+Schema:
+
+    {
+      "file":                     string,
+      "link_to"                   string,
+      "size":                     integer,
+      "blocks":                   integer,
+      "io_blocks":                integer,
+      "type":                     string,
+      "device":                   string,
+      "inode":                    integer,
+      "links":                    integer,
+      "access":                   string,
+      "flags":                    string,
+      "uid":                      integer,
+      "user":                     string,
+      "gid":                      integer,
+      "group":                    string,
+      "access_time":              string,    # - = null
+      "access_time_epoch":        integer,   # naive timestamp
+      "access_time_epoch_utc":    integer,   # timezone-aware timestamp
+      "modify_time":              string,    # - = null
+      "modify_time_epoch":        integer,   # naive timestamp
+      "modify_time_epoch_utc":    integer,   # timezone-aware timestamp
+      "change_time":              string,    # - = null
+      "change_time_epoch":        integer,   # naive timestamp
+      "change_time_epoch_utc":    integer,   # timezone-aware timestamp
+      "birth_time":               string,    # - = null
+      "birth_time_epoch":         integer,   # naive timestamp
+      "birth_time_epoch_utc":     integer,   # timezone-aware timestamp
+      "unix_device":              integer,
+      "rdev":                     integer,
+      "block_size":               integer,
+      "unix_flags":               string,
+      "_jc_meta":                            # This object only exists if using -qq or ignore_exceptions=True
+        {
+          "success":              boolean,   # true if successfully parsed, false if error
+          "error":                string,    # exists if "success" is false
+          "line":                 string     # exists if "success" is false
+        }
+    }
+
+Examples:
+
+    $ stat | jc --stat-s
+    {"file":"(stdin)","unix_device":1027739696,"inode":1155,"flags":"crw--w----","links":1,"user":"kbrazil","group":"tty","rdev":268435456,"size":0,"access_time":"Jan  4 15:27:44 2022","modify_time":"Jan  4 15:27:44 2022","change_time":"Jan  4 15:27:44 2022","birth_time":"Dec 31 16:00:00 1969","block_size":131072,"blocks":0,"unix_flags":"0","access_time_epoch":1641338864,"access_time_epoch_utc":null,"modify_time_epoch":1641338864,"modify_time_epoch_utc":null,"change_time_epoch":1641338864,"change_time_epoch_utc":null,"birth_time_epoch":null,"birth_time_epoch_utc":null}
+
+    $ stat | jc --stat-s -r
+    {"file":"(stdin)","unix_device":"1027739696","inode":"1155","flags":"crw--w----","links":"1","user":"kbrazil","group":"tty","rdev":"268435456","size":"0","access_time":"Jan  4 15:28:08 2022","modify_time":"Jan  4 15:28:08 2022","change_time":"Jan  4 15:28:08 2022","birth_time":"Dec 31 16:00:00 1969","block_size":"131072","blocks":"0","unix_flags":"0"}
+"""
+import shlex
+import jc.utils
+from jc.utils import stream_success, stream_error
+from jc.exceptions import ParseError
+
+
+class info():
+    """Provides parser metadata (version, author, etc.)"""
+    version = '0.5'
+    description = '`stat` command streaming parser'
+    author = 'Kelly Brazil'
+    author_email = 'kellyjonbrazil@gmail.com'
+    compatible = ['linux', 'darwin', 'freebsd']
+    streaming = True
+
+
+__version__ = info.version
+
+
+def _process(proc_data):
+    """
+    Final processing to conform to the schema.
+
+    Parameters:
+
+        proc_data:   (Dictionary) raw structured data to process
+
+    Returns:
+
+        Dictionary. Structured data to conform to the schema.
+    """
+    int_list = ['size', 'blocks', 'io_blocks', 'inode', 'links', 'uid', 'gid', 'unix_device',
+                'rdev', 'block_size']
+    for key in proc_data:
+        if key in int_list:
+            proc_data[key] = jc.utils.convert_to_int(proc_data[key])
+
+    # turn - into null for time fields and add calculated timestamp fields
+    null_list = ['access_time', 'modify_time', 'change_time', 'birth_time']
+    for key in null_list:
+        if key in proc_data:
+            if proc_data[key] == '-':
+                proc_data[key] = None
+            ts = jc.utils.timestamp(proc_data[key])
+            proc_data[key + '_epoch'] = ts.naive
+            proc_data[key + '_epoch_utc'] = ts.utc
+
+    return proc_data
+
+def parse(data, raw=False, quiet=False, ignore_exceptions=False):
+    """
+    Main text parsing generator function. Returns an iterator object.
+
+    Parameters:
+
+        data:              (iterable)  line-based text data to parse (e.g. sys.stdin or str.splitlines())
+        raw:               (boolean)   output preprocessed JSON if True
+        quiet:             (boolean)   suppress warning messages if True
+        ignore_exceptions: (boolean)   ignore parsing exceptions if True
+
+    Yields:
+
+        Dictionary. Raw or processed structured data.
+
+    Returns:
+
+        Iterator object
+    """
+    jc.utils.compatibility(__name__, info.compatible, quiet)
+    jc.utils.streaming_input_type_check(data)
+
+    output_line = {}
+    os_type = ''
+
+    for line in data:
+        try:
+            jc.utils.streaming_line_input_type_check(line)
+            line = line.rstrip()
+
+            # ignore blank lines
+            if line == '':
+                continue
+
+            # linux output
+            if line.startswith('  File: '):
+                os_type = 'linux'
+
+            if os_type == 'linux':
+                # stats output contains 9 lines
+                # line #1
+                if line.startswith('  File: '):
+                    if output_line:
+                        yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
+
+                    output_line = {}
+                    line_list = line.split(maxsplit=1)
+                    output_line['file'] = line_list[1]
+
+                    # populate link_to field if -> found
+                    if ' -> ' in output_line['file']:
+                        filename = output_line['file'].split(' -> ')[0].strip('\u2018').rstrip('\u2019')
+                        link = output_line['file'].split(' -> ')[1].strip('\u2018').rstrip('\u2019')
+                        output_line['file'] = filename
+                        output_line['link_to'] = link
+                    else:
+                        filename = output_line['file'].split(' -> ')[0].strip('\u2018').rstrip('\u2019')
+                        output_line['file'] = filename
+
+                    continue
+
+                # line #2
+                if line.startswith('  Size: '):
+                    line_list = line.split(maxsplit=7)
+                    output_line['size'] = line_list[1]
+                    output_line['blocks'] = line_list[3]
+                    output_line['io_blocks'] = line_list[6]
+                    output_line['type'] = line_list[7]
+                    continue
+
+                # line #3
+                if line.startswith('Device: '):
+                    line_list = line.split()
+                    output_line['device'] = line_list[1]
+                    output_line['inode'] = line_list[3]
+                    output_line['links'] = line_list[5]
+                    continue
+
+                # line #4
+                if line.startswith('Access: ('):
+                    line = line.replace('(', ' ').replace(')', ' ').replace('/', ' ')
+                    line_list = line.split()
+                    output_line['access'] = line_list[1]
+                    output_line['flags'] = line_list[2]
+                    output_line['uid'] = line_list[4]
+                    output_line['user'] = line_list[5]
+                    output_line['gid'] = line_list[7]
+                    output_line['group'] = line_list[8]
+                    continue
+
+                # line #5
+                # not implemented
+                if line.startswith('Context: '):
+                    continue
+
+                # line #6
+                if line.startswith('Access: 2'):
+                    line_list = line.split(maxsplit=1)
+                    output_line['access_time'] = line_list[1]
+                    continue
+
+                # line #7
+                if line.startswith('Modify: '):
+                    line_list = line.split(maxsplit=1)
+                    output_line['modify_time'] = line_list[1]
+                    continue
+
+                # line #8
+                if line.startswith('Change: '):
+                    line_list = line.split(maxsplit=1)
+                    output_line['change_time'] = line_list[1]
+                    continue
+
+                # line #9
+                if line.startswith(' Birth: '):
+                    line_list = line.split(maxsplit=1)
+                    output_line['birth_time'] = line_list[1]
+                    continue
+
+                # catch non-stat data
+                raise ParseError('Not stat data')
+
+            # FreeBSD/OSX output
+            if os_type != 'linux':
+                value = shlex.split(line)
+
+                if not value[0].isdigit() or not value[1].isdigit():
+                    raise ParseError('Not stat data')
+
+                output_line = {
+                    'file': ' '.join(value[15:]),
+                    'unix_device': value[0],
+                    'inode': value[1],
+                    'flags': value[2],
+                    'links': value[3],
+                    'user': value[4],
+                    'group': value[5],
+                    'rdev': value[6],
+                    'size': value[7],
+                    'access_time': value[8],
+                    'modify_time': value[9],
+                    'change_time': value[10],
+                    'birth_time': value[11],
+                    'block_size': value[12],
+                    'blocks': value[13],
+                    'unix_flags': value[14]
+                }
+
+                if output_line:
+                    yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
+                    output_line = {}
+
+        except Exception as e:
+            yield stream_error(e, ignore_exceptions, line)
+            output_line = {}
+
+    # gather final item
+    if output_line:
+        try:
+            yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions)
+        except Exception as e:
+            yield stream_error(e, ignore_exceptions, line)

man/jc.1

@@ -1,4 +1,4 @@
-.TH jc 1 2022-01-03 1.17.6 "JSON CLI output utility"
+.TH jc 1 2022-01-14 1.17.7 "JSON CLI output utility"
 .SH NAME
 jc \- JSONifies the output of many CLI tools and file-types
 .SH SYNOPSIS
@@ -322,6 +322,11 @@ Key/Value file parser
 \fB--stat\fP
 `stat` command parser
 
+.TP
+.B
+\fB--stat-s\fP
+`stat` command streaming parser
+
 .TP
 .B
 \fB--sysctl\fP

setup.py

@@ -5,7 +5,7 @@ with open('README.md', 'r') as f:
 
 setuptools.setup(
     name='jc',
-    version='1.17.6',
+    version='1.17.7',
     author='Kelly Brazil',
     author_email='kellyjonbrazil@gmail.com',
     description='Converts the output of popular command-line tools and file-types to JSON.',

tests/fixtures/osx-10.14.6/stat-filename-with-spaces-streaming.json

@@ -0,0 +1 @@
+[{"file":"file name with spaces.txt","unix_device":16777220,"inode":161929661,"flags":"-rw-r--r--","links":1,"user":"kbrazil","group":"staff","rdev":0,"size":0,"access_time":"Aug 13 15:03:52 2021","modify_time":"Aug 13 14:37:03 2021","change_time":"Aug 13 14:37:03 2021","birth_time":"Aug 13 14:37:03 2021","block_size":4096,"blocks":0,"unix_flags":"0","access_time_epoch":1628892232,"access_time_epoch_utc":null,"modify_time_epoch":1628890623,"modify_time_epoch_utc":null,"change_time_epoch":1628890623,"change_time_epoch_utc":null,"birth_time_epoch":1628890623,"birth_time_epoch_utc":null}]

tests/test_stat_s.py

@@ -0,0 +1,100 @@
+import os
+import sys
+import time
+import json
+import unittest
+from jc.exceptions import ParseError
+import jc.parsers.stat_s
+
+THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Set the timezone on POSIX systems. Need to manually set for Windows tests
+if not sys.platform.startswith('win32'):
+    os.environ['TZ'] = 'America/Los_Angeles'
+    time.tzset()
+
+
+# To create streaming output use:
+# $ cat stat.out | jc --stat-s | jello -c > stat-streaming.json
+
+
+class MyTests(unittest.TestCase):
+
+    def setUp(self):
+        # input
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/stat.out'), 'r', encoding='utf-8') as f:
+            self.centos_7_7_stat = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/stat.out'), 'r', encoding='utf-8') as f:
+            self.ubuntu_18_4_stat = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/stat.out'), 'r', encoding='utf-8') as f:
+            self.osx_10_14_6_stat = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/stat-filename-with-spaces.out'), 'r', encoding='utf-8') as f:
+            self.osx_10_14_6_stat_filename_with_spaces = f.read()
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/freebsd12/stat.out'), 'r', encoding='utf-8') as f:
+            self.freebsd12_stat = f.read()
+
+        # output
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/centos-7.7/stat-streaming.json'), 'r', encoding='utf-8') as f:
+            self.centos_7_7_stat_streaming_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/ubuntu-18.04/stat-streaming.json'), 'r', encoding='utf-8') as f:
+            self.ubuntu_18_4_stat_streaming_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/stat-streaming.json'), 'r', encoding='utf-8') as f:
+            self.osx_10_14_6_stat_streaming_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/osx-10.14.6/stat-filename-with-spaces-streaming.json'), 'r', encoding='utf-8') as f:
+            self.osx_10_14_6_stat_filename_with_spaces_streaming_json = json.loads(f.read())
+
+        with open(os.path.join(THIS_DIR, os.pardir, 'tests/fixtures/freebsd12/stat-streaming.json'), 'r', encoding='utf-8') as f:
+            self.freebsd12_stat_streaming_json = json.loads(f.read())
+
+    def test_stat_s_nodata(self):
+        """
+        Test 'stat' with no data
+        """
+        self.assertEqual(list(jc.parsers.stat_s.parse([], quiet=True)), [])
+
+    def test_stat_s_unparsable(self):
+        data = 'unparsable data'
+        g = jc.parsers.stat_s.parse(data.splitlines(), quiet=True)
+        with self.assertRaises(ParseError):
+            list(g)
+
+    def test_stat_s_centos_7_7(self):
+        """
+        Test 'stat /bin/*' on Centos 7.7
+        """
+        self.assertEqual(list(jc.parsers.stat_s.parse(self.centos_7_7_stat.splitlines(), quiet=True)), self.centos_7_7_stat_streaming_json)
+
+    def test_stat_s_ubuntu_18_4(self):
+        """
+        Test 'stat /bin/*' on Ubuntu 18.4
+        """
+        self.assertEqual(list(jc.parsers.stat_s.parse(self.ubuntu_18_4_stat.splitlines(), quiet=True)), self.ubuntu_18_4_stat_streaming_json)
+
+    def test_stat_s_osx_10_14_6(self):
+        """
+        Test 'stat /bin/*' on OSX 10.14.6
+        """
+        self.assertEqual(list(jc.parsers.stat_s.parse(self.osx_10_14_6_stat.splitlines(), quiet=True)), self.osx_10_14_6_stat_streaming_json)
+
+    def test_stat_s_filename_with_spaces_osx_10_14_6(self):
+        """
+        Test 'stat' filename with spaces on OSX 10.14.6
+        """
+        self.assertEqual(list(jc.parsers.stat_s.parse(self.osx_10_14_6_stat_filename_with_spaces.splitlines(), quiet=True)), self.osx_10_14_6_stat_filename_with_spaces_streaming_json)
+
+    def test_stat_s_freebsd12(self):
+        """
+        Test 'stat /foo/*' on FreeBSD12
+        """
+        self.assertEqual(list(jc.parsers.stat_s.parse(self.freebsd12_stat.splitlines(), quiet=True)), self.freebsd12_stat_streaming_json)
+
+
+if __name__ == '__main__':
+    unittest.main()