commit 3e576250b1bfe1c62c556f2fa0425103b0ede1f9 Author: Kelly Brazil Date: Tue Oct 15 15:06:09 2019 -0700 First commit diff --git a/.gitignore b/.gitignore new file mode 100755 index 00000000..84688f41 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__ +*.pyc +dist/ +build/ +*.egg-info/ +jc/parsers.old/ diff --git a/LICENSE.md b/LICENSE.md new file mode 100755 index 00000000..c040d4bb --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Kelly Brazil + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100755 index 00000000..ba1f8a48 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# JC +JSON CLI output utility +v0.1 + +`jc` is used to JSONify the output of many standard linux cli tools for easier parsing in scripts. Parsers for `ls`, `ifconfig`, and `netstat` are currently included and more can be added via modules. + +## Usage +`jc` accepts piped input from `STDIN` and outputs a JSON representation of the previous command to `STDOUT`. The JSON output can be compact or pretty formatted. + +The first argument is required and identifies the command that is piping output to `jc` input. For example: +- `--ls` enables the `ls` parser +- `--ifconfig` enables the `ifconfig` parser +- `--netstat` enables the `netstat` parser + +The second `-p` argument is optional and specifies whether to pretty format the JSON output. + +## Examples +``` +$ ls -l /bin | jc --ls -p +[ + { + "filename": "bash", + "flags": "-r-xr-xr-x", + "links": 1, + "owner": "root", + "group": "wheel", + "bytes": 618416, + "date": "May 3 22:26" + }, + { + "filename": "cat", + "flags": "-rwxr-xr-x", + "links": 1, + "owner": "root", + "group": "wheel", + "bytes": 23648, + "date": "May 3 22:26" + }, + { + "filename": "chmod", + "flags": "-rwxr-xr-x", + "links": 1, + "owner": "root", + "group": "wheel", + "bytes": 30016, + "date": "May 3 22:26" + }, + ... +] +``` + + + diff --git a/build-package.sh b/build-package.sh new file mode 100755 index 00000000..6c6ca2f4 --- /dev/null +++ b/build-package.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# build jc PIP package +# to install locally, run: pip3 install jc-x.x.tar.gz + +python3 setup.py sdist bdist_wheel diff --git a/jc/__init__.py b/jc/__init__.py new file mode 100644 index 00000000..fb8c1668 --- /dev/null +++ b/jc/__init__.py @@ -0,0 +1,61 @@ +"""JC - JSON CLI output utility + +v0.1 + +* kellyjonbrazil@gmail.com + +This module serializes standard unix command line output to structured JSON +output. + +Example: + +$ ls -al | jc | jq . +[ + { + "filename": ".", + "suffix": Null, + "bytes": 224, + "date_updated": "Oct 1 12:09", + "owner_user": "joeuser", + "owner_group": "staff", + "flags": "drwxr-xr-x+", + "link_to": Null, + "links": 47 + }, + { + "filename": "..", + "suffix": Null, + "bytes": 224, + "date_updated": "Oct 1 12:09", + "owner_user": "admin", + "owner_group": "root", + "flags": "drwxr-xr-x", + "link_to": Null, + "links": 7 + }, + { + "filename": "testfile.txt", + "suffix": "txt", + "bytes": 14686, + "date_updated": "Oct 1 12:09", + "owner_user": "joeuser", + "owner_group": "staff", + "flags": "-rwxr-xr-x@", + "link_to": Null, + "links": 1 + }, + { + "filename": "ncat", + "suffix": Null, + "bytes": 14686, + "date_updated": "Oct 1 12:09", + "owner_user": "joeuser", + "owner_group": "staff", + "flags": "lrwxr-xr-x", + "link_to": "../Cellar/nmap/7.70/bin/ncat", + "links": 1 + } +] +""" + +name = 'jc' diff --git a/jc/jc.py b/jc/jc.py new file mode 100755 index 00000000..ebfaf61a --- /dev/null +++ b/jc/jc.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""jc - JSON CLI output utility + +Main input module +""" + +import sys +import json +import parsers +from parsers import * + +pretty = False +data = sys.stdin.read() + +if len(sys.argv) < 2: + print(f'\nError: {sys.argv[0]}\n Must specify parser. (e.g. --ls, --netstat, --ifconfig, etc.)') + print(' Use -p to pretty print') + print(f'\nExample: ls -al | {sys.argv[0]} --ls -p\n') + exit() + +arg = sys.argv[1] + +if len(sys.argv) > 2: + if sys.argv[2] == '-p': + pretty = True + +if arg == '--ifconfig': + result = parsers.ifconfig.parse(data) +elif arg == '--ls': + result = parsers.ls.parse(data) +elif arg == '--netstat': + result = parsers.netstat.parse(data) + +# output resulting dictionary as json +if pretty: + print(json.dumps(result, indent=2)) +else: + print(json.dumps(result)) diff --git a/jc/parsers/__init__.py b/jc/parsers/__init__.py new file mode 100644 index 00000000..628d1c12 --- /dev/null +++ b/jc/parsers/__init__.py @@ -0,0 +1,3 @@ +__all__ = ['ifconfig', + 'ls', + 'netstat'] \ No newline at end of file diff --git a/jc/parsers/ifconfig.py b/jc/parsers/ifconfig.py new file mode 100644 index 00000000..80bce2e5 --- /dev/null +++ b/jc/parsers/ifconfig.py @@ -0,0 +1,30 @@ +"""jc - JSON CLI output utility ifconfig Parser + +Usage: + specify --ifconfig as the first argument if the piped input is coming from ifconfig + + no ifconfig options are supported. + +Example: + +$ ifconfig | ./jc.py --ifconfig -p + +""" +from collections import namedtuple +from ifconfigparser import IfconfigParser + +def parse(data): + output = [] + + parsed = IfconfigParser(console_output=data) + interfaces = parsed.get_interfaces() + + # convert ifconfigparser output to a dictionary + for iface in interfaces: + d = interfaces[iface]._asdict() + dct = dict(d) + output.append(dct) + + return output + + diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py new file mode 100644 index 00000000..78b275bf --- /dev/null +++ b/jc/parsers/ls.py @@ -0,0 +1,130 @@ +"""jc - JSON CLI output utility ls Parser + +Usage: + specify --ls as the first argument if the piped input is coming from ls + + ls options supported: + - None + - l + - a + +Examples: + +$ ls -a /usr/bin | jc --ls -p +[ + { + "filename": "." + }, + { + "filename": ".." + }, + { + "filename": "2to3-" + }, + { + "filename": "2to3-2.7" + }, + { + "filename": "AssetCacheLocatorUtil" + }, + ... +] + +$ ls -al /usr/bin | jc --ls -p +[ + { + "filename": ".", + "flags": "drwxr-xr-x", + "links": 970, + "owner": "root", + "group": "wheel", + "bytes": 31040, + "date": "Aug 27 21:20" + }, + { + "filename": "..", + "flags": "drwxr-xr-x@", + "links": 9, + "owner": "root", + "group": "wheel", + "bytes": 288, + "date": "May 3 22:14" + }, + { + "filename": "2to3-", + "flags": "-rwxr-xr-x", + "links": 4, + "owner": "root", + "group": "wheel", + "bytes": 925, + "date": "Feb 22 2019" + }, + { + "filename": "2to3-2.7", + "link_to": "../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7", + "flags": "lrwxr-xr-x", + "links": 1, + "owner": "root", + "group": "wheel", + "bytes": 74, + "date": "May 4 02:12" + }, + ... +] + +$ $ ls -l /usr/bin | jc --ls | jq .[] | jq 'select(.bytes > 50000000)' +{ + "filename": "emacs", + "flags": "-r-xr-xr-x", + "links": 1, + "owner": "root", + "group": "wheel", + "bytes": 117164432, + "date": "May 3 22:26" +} +""" +import re + +def parse(data): + output = [] + + cleandata = data.splitlines() + + # Delete first line if it starts with 'total' + if cleandata[0].find('total') == 0: + cleandata.pop(0) + + # Delete last line if it is blank + if cleandata[-1] == '': + cleandata.pop(-1) + + # Check if -l was used to parse extra data + if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', cleandata[0]): + for entry in cleandata: + output_line = {} + + parsed_line = entry.split() + + # split filenames and links + filename_field = ' '.join(parsed_line[8:]).split(' -> ') + + # create list of dictionaries + output_line['filename'] = filename_field[0] + + if len(filename_field) > 1: + output_line['link_to'] = filename_field[1] + + output_line['flags'] = parsed_line[0] + output_line['links'] = int(parsed_line[1]) + output_line['owner'] = parsed_line[2] + output_line['group'] = parsed_line[3] + output_line['bytes'] = int(parsed_line[4]) + output_line['date'] = ' '.join(parsed_line[5:8]) + output.append(output_line) + else: + for entry in cleandata: + output_line = {} + output_line['filename'] = entry + output.append(output_line) + + return output diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py new file mode 100644 index 00000000..348aa204 --- /dev/null +++ b/jc/parsers/netstat.py @@ -0,0 +1,55 @@ +"""jc - JSON CLI output utility netstat Parser + +Usage: + specify --netstat as the first argument if the piped input is coming from netstat + +Example: + +$ netstat | jc --netstat -p +""" + +import re + +def parse(data): + output = [] + + cleandata = data.splitlines() + + # Delete last line if it is blank + if cleandata[-1] == '': + cleandata.pop(-1) + + # Delete first line if it starts with 'total' + if cleandata[0].find('total') == 0: + cleandata.pop(0) + + # Check if -l was used to parse extra data + if re.match('^[-dclpsbDCMnP?]([-r][-w][-xsS]){2}([-r][-w][-xtT])[+]?', cleandata[0]): + for entry in cleandata: + output_line = {} + + parsed_line = entry.split() + + # split filenames and links + filename_field = ' '.join(parsed_line[8:]).split(' -> ') + + # create list of dictionaries + output_line['filename'] = filename_field[0] + + if len(filename_field) > 1: + output_line['link_to'] = filename_field[1] + + output_line['flags'] = parsed_line[0] + output_line['links'] = int(parsed_line[1]) + output_line['owner'] = parsed_line[2] + output_line['group'] = parsed_line[3] + output_line['bytes'] = int(parsed_line[4]) + output_line['date'] = ' '.join(parsed_line[5:8]) + output.append(output_line) + else: + for entry in cleandata: + output_line = {} + output_line['filename'] = entry + output.append(output_line) + + return output \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100755 index 00000000..aae5809c --- /dev/null +++ b/setup.py @@ -0,0 +1,27 @@ +import setuptools + +with open("README.md", "r") as f: + long_description = f.read() + +setuptools.setup( + name="jc", + version="0.1", + author="Kelly Brazil", + description="This tool serializes the output of popular command line tools to structured JSON output.", + install_requires=[ + collections, + ifconfigparser, + json + ], + license='MIT', + long_description=long_description, + long_description_content_type="text/markdown", + python_requires='~=3.4', + url="https://github.com/kellyjonbrazil/jc", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], +)