diff --git a/CHANGELOG b/CHANGELOG index 43d8c40a..335c160e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,11 @@ jc changelog +20220121 v1.18.0 +- Add high-level parse API for built-in and plugin parsers +- Add python module developer documentation +- Enhance magic syntax exception messages +- Enhance documentation formatting + 20220106 v1.17.7 - Add stat command streaming parser tested on linux and macOS diff --git a/README.md b/README.md index a224d4a5..3be52081 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,10 @@ $ jc dig example.com | jq -r '.[].answer[].data' The `jc` parsers can also be used as python modules. In this case the output will be a python dictionary, or list of dictionaries, instead of JSON: ```python >>> import subprocess ->>> import jc.parsers.dig ->>> +>>> import jc +>>> >>> cmd_output = subprocess.check_output(['dig', 'example.com'], text=True) ->>> data = jc.parsers.dig.parse(cmd_output) +>>> data = jc.parse('dig', cmd_output) >>> >>> data [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', 'flags': ['qr', 'rd', 'ra'], 'query_num': 1, 'answer_num': @@ -45,6 +45,9 @@ The `jc` parsers can also be used as python modules. In this case the output wil '2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)', 'when': 'Fri Apr 16 16:13:00 PDT 2021', 'rcvd': 56, 'when_epoch': 1618614780, 'when_epoch_utc': None}] ``` + +> For `jc` Python package documentation, use `help('jc')`, `help('jc.lib')`, or see the [online documentation](https://github.com/kellyjonbrazil/jc/tree/master/docs). + Two representations of the data are available. The default representation uses a strict schema per parser and converts known numbers to int/float JSON values. Certain known values of `None` are converted to JSON `null`, known boolean values are converted, and, in some cases, additional semantic context fields are added. To access the raw, pre-processed JSON, use the `-r` cli option or the `raw=True` function parameter in `parse()`. @@ -290,9 +293,9 @@ Streaming parsers accept any iterable object and return a generator iterator obj To use the generator object in your code, simply loop through it or use the [next()](https://docs.python.org/3/library/functions.html#next) builtin function: ```python -import jc.parsers.ls_s +import jc -result = jc.parsers.ls_s.parse(ls_command_output.splitlines()) +result = jc.parse('ls_s', ls_command_output.splitlines()) for item in result: print(item["filename"]) ``` diff --git a/docgen.sh b/docgen.sh index 3b100ae3..994891b8 100755 --- a/docgen.sh +++ b/docgen.sh @@ -5,9 +5,16 @@ cd jc echo Building docs for: package pydocmd simple jc+ > ../docs/readme.md + +echo Building docs for: lib +pydocmd simple lib+ > ../docs/lib.md + echo Building docs for: utils pydocmd simple utils+ > ../docs/utils.md +echo Building docs for: universal parser +pydocmd simple jc.parsers.universal+ > ../docs/parsers/universal.md + # a bit of inception here... jc is being used to help # automate the generation of its own documentation. :) diff --git a/docs/_config.yml b/docs/_config.yml deleted file mode 100644 index c4192631..00000000 --- a/docs/_config.yml +++ /dev/null @@ -1 +0,0 @@ -theme: jekyll-theme-cayman \ No newline at end of file diff --git a/docs/lib.md b/docs/lib.md new file mode 100644 index 00000000..4601a0bb --- /dev/null +++ b/docs/lib.md @@ -0,0 +1,90 @@ + +# lib +jc - JSON CLI output utility +JC lib module + + +## parse +```python +parse(parser_mod_name, + data, + quiet=False, + raw=False, + ignore_exceptions=None, + **kwargs) +``` + +Parse the string data using the supplied parser module. + +This function provides a high-level API to simplify parser use. This +function will call built-in parsers and custom plugin parsers. + +Example: + + >>> import jc + >>> jc.parse('date', 'Tue Jan 18 10:23:07 PST 2022') + {'year': 2022, 'month': 'Jan', 'month_num': 1, 'day'...} + +To get a list of available parser module names, use `parser_mod_list()` +or `plugin_parser_mod_list()`. `plugin_parser_mod_list()` is a subset +of `parser_mod_list()`. + +You can also use the lower-level parser modules directly: + + >>> import jc.parsers.date + >>> jc.parsers.date.parse('Tue Jan 18 10:23:07 PST 2022') + +Though, accessing plugin parsers directly is a bit more cumbersome, so +this higher-level API is recommended. Here is how you can access plugin +parsers without this API: + + >>> import os + >>> import sys + >>> import jc.appdirs + >>> data_dir = jc.appdirs.user_data_dir('jc', 'jc') + >>> local_parsers_dir = os.path.join(data_dir, 'jcparsers') + >>> sys.path.append(local_parsers_dir) + >>> import my_custom_parser + >>> my_custom_parser.parse('command_data') + +Parameters: + + parser_mod_name: (string) name of the parser module + + data: (string or data to parse (string for normal + iterator) parsers, iterator of strings for + streaming parsers) + + raw: (boolean) output preprocessed JSON if True + + quiet: (boolean) suppress warning messages if True + + ignore_exceptions: (boolean) ignore parsing exceptions if True + (streaming parsers only) + +Returns: + + Standard Parsers: Dictionary or List of Dictionaries + Streaming Parsers: Generator Object containing Dictionaries + + +## parser_mod_list +```python +parser_mod_list() +``` +Returns a list of all available parser module names. + +## plugin_parser_mod_list +```python +plugin_parser_mod_list() +``` + +Returns a list of plugin parser module names. This function is a +subset of `parser_mod_list()`. + + +## get_help +```python +get_help(parser_mod_name) +``` +Show help screen for the selected parser. diff --git a/docs/parsers/acpi.md b/docs/parsers/acpi.md index ecf0122b..50e5f378 100644 --- a/docs/parsers/acpi.md +++ b/docs/parsers/acpi.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('acpi', acpi_command_output) + + or + import jc.parsers.acpi result = jc.parsers.acpi.parse(acpi_command_output) @@ -242,7 +247,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/airport.md b/docs/parsers/airport.md index b45bde86..dc178d52 100644 --- a/docs/parsers/airport.md +++ b/docs/parsers/airport.md @@ -15,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('airport', airport_command_output) + + or + import jc.parsers.airport result = jc.parsers.airport.parse(airport_command_output) @@ -95,7 +100,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/airport_s.md b/docs/parsers/airport_s.md index 49fb032d..035a5d7a 100644 --- a/docs/parsers/airport_s.md +++ b/docs/parsers/airport_s.md @@ -15,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('airport_s', airport_s_command_output) + + or + import jc.parsers.airport_s result = jc.parsers.airport_s.parse(airport_s_command_output) @@ -123,7 +128,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/arp.md b/docs/parsers/arp.md index 223b5219..70f5757b 100644 --- a/docs/parsers/arp.md +++ b/docs/parsers/arp.md @@ -15,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('arp', arp_command_output) + + or + import jc.parsers.arp result = jc.parsers.arp.parse(arp_command_output) @@ -132,7 +137,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/blkid.md b/docs/parsers/blkid.md index 94ae4308..c982ba9e 100644 --- a/docs/parsers/blkid.md +++ b/docs/parsers/blkid.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('blkid', blkid_command_output) + + or + import jc.parsers.blkid result = jc.parsers.blkid.parse(blkid_command_output) @@ -135,7 +140,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/cksum.md b/docs/parsers/cksum.md index e278bac6..afcb754a 100644 --- a/docs/parsers/cksum.md +++ b/docs/parsers/cksum.md @@ -17,6 +17,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('cksum', cksum_command_output) + + or + import jc.parsers.cksum result = jc.parsers.cksum.parse(cksum_command_output) @@ -69,7 +74,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/crontab.md b/docs/parsers/crontab.md index 553ddf0a..1efd7d31 100644 --- a/docs/parsers/crontab.md +++ b/docs/parsers/crontab.md @@ -1,7 +1,8 @@ [Home](https://kellyjonbrazil.github.io/jc/) # jc.parsers.crontab -jc - JSON CLI output utility `crontab -l` command output and crontab file parser +jc - JSON CLI output utility `crontab -l` command output and crontab +file parser Supports `crontab -l` command output and crontab files. @@ -15,6 +16,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('crontab', crontab_output) + + or + import jc.parsers.crontab result = jc.parsers.crontab.parse(crontab_output) @@ -187,7 +193,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/crontab_u.md b/docs/parsers/crontab_u.md index 570b673f..dc66508b 100644 --- a/docs/parsers/crontab_u.md +++ b/docs/parsers/crontab_u.md @@ -1,9 +1,11 @@ [Home](https://kellyjonbrazil.github.io/jc/) # jc.parsers.crontab_u -jc - JSON CLI output utility `crontab -l` command output and crontab file parser +jc - JSON CLI output utility `crontab -l` command output and crontab +file parser -This version of the `crontab -l` parser supports output that contains user information for processes. +This version of the `crontab -l` parser supports output that contains user +information for processes. Usage (cli): @@ -11,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('crontab_u', crontab_u_output) + + or + import jc.parsers.crontab_u result = jc.parsers.crontab_u.parse(crontab_u_output) @@ -55,7 +62,7 @@ Examples: "variables": [ { "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" + "value": "/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sb..." }, { "name": "SHELL", @@ -80,7 +87,7 @@ Examples: "*" ], "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.daily )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." }, { "minute": [ @@ -99,7 +106,7 @@ Examples: "7" ], "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.weekly )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." }, { "minute": [ @@ -118,7 +125,7 @@ Examples: "*" ], "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.monthly )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." } ] } @@ -128,7 +135,7 @@ Examples: "variables": [ { "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" + "value": "/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/..." }, { "name": "SHELL", @@ -143,7 +150,7 @@ Examples: "month": "*", "day_of_week": "*", "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.daily )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." }, { "minute": "47", @@ -152,7 +159,7 @@ Examples: "month": "*", "day_of_week": "7", "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.weekly )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." }, { "minute": "52", @@ -161,7 +168,7 @@ Examples: "month": "*", "day_of_week": "*", "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.monthly )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." } ] } @@ -183,7 +190,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/csv.md b/docs/parsers/csv.md index 91936dc8..adc29ded 100644 --- a/docs/parsers/csv.md +++ b/docs/parsers/csv.md @@ -3,7 +3,9 @@ # jc.parsers.csv jc - JSON CLI output utility `csv` file parser -The `csv` parser will attempt to automatically detect the delimiter character. If the delimiter cannot be detected it will default to comma. The first row of the file must be a header row. +The `csv` parser will attempt to automatically detect the delimiter +character. If the delimiter cannot be detected it will default to comma. +The first row of the file must be a header row. Usage (cli): @@ -11,12 +13,18 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('csv', csv_output) + + or + import jc.parsers.csv result = jc.parsers.csv.parse(csv_output) Schema: - csv file converted to a Dictionary: https://docs.python.org/3/library/csv.html + csv file converted to a Dictionary: + https://docs.python.org/3/library/csv.html [ { @@ -28,7 +36,7 @@ Schema: Examples: $ cat homes.csv - "Sell", "List", "Living", "Rooms", "Beds", "Baths", "Age", "Acres", "Taxes" + "Sell", "List", "Living", "Rooms", "Beds", "Baths", "Age", "Acres"... 142, 160, 28, 10, 5, 3, 60, 0.28, 3167 175, 180, 18, 8, 4, 1, 12, 0.43, 4033 129, 132, 13, 6, 3, 1, 41, 0.33, 1471 @@ -89,7 +97,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/csv_s.md b/docs/parsers/csv_s.md index 1d462e2f..6392d6aa 100644 --- a/docs/parsers/csv_s.md +++ b/docs/parsers/csv_s.md @@ -5,9 +5,12 @@ jc - JSON CLI output utility `csv` file streaming parser > This streaming parser outputs JSON Lines -The `csv` streaming parser will attempt to automatically detect the delimiter character. If the delimiter cannot be detected it will default to comma. The first row of the file must be a header row. +The `csv` streaming parser will attempt to automatically detect the +delimiter character. If the delimiter cannot be detected it will default +to comma. The first row of the file must be a header row. -Note: The first 100 rows are read into memory to enable delimiter detection, then the rest of the rows are loaded lazily. +Note: The first 100 rows are read into memory to enable delimiter detection, +then the rest of the rows are loaded lazily. Usage (cli): @@ -15,21 +18,34 @@ Usage (cli): Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('csv_s', csv_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.csv_s - result = jc.parsers.csv_s.parse(csv_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.csv_s.parse(csv_output.splitlines()) for item in result: # do something Schema: - csv file converted to a Dictionary: https://docs.python.org/3/library/csv.html + csv file converted to a Dictionary: + https://docs.python.org/3/library/csv.html { "column_name1": string, "column_name2": string, - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + + # below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error + "success": boolean, # false if error parsing "error": string, # exists if "success" is false "line": string # exists if "success" is false } @@ -38,16 +54,16 @@ Schema: Examples: $ cat homes.csv - "Sell", "List", "Living", "Rooms", "Beds", "Baths", "Age", "Acres", "Taxes" + "Sell", "List", "Living", "Rooms", "Beds", "Baths", "Age", "Acres"... 142, 160, 28, 10, 5, 3, 60, 0.28, 3167 175, 180, 18, 8, 4, 1, 12, 0.43, 4033 129, 132, 13, 6, 3, 1, 41, 0.33, 1471 ... $ cat homes.csv | jc --csv-s - {"Sell":"142","List":"160","Living":"28","Rooms":"10","Beds":"5","Baths":"3","Age":"60","Acres":"0.28","Taxes":"3167"} - {"Sell":"175","List":"180","Living":"18","Rooms":"8","Beds":"4","Baths":"1","Age":"12","Acres":"0.43","Taxes":"4033"} - {"Sell":"129","List":"132","Living":"13","Rooms":"6","Beds":"3","Baths":"1","Age":"41","Acres":"0.33","Taxes":"1471"} + {"Sell":"142","List":"160","Living":"28","Rooms":"10","Beds":"5"...} + {"Sell":"175","List":"180","Living":"18","Rooms":"8","Beds":"4"...} + {"Sell":"129","List":"132","Living":"13","Rooms":"6","Beds":"3"...} ... @@ -66,8 +82,10 @@ Main text parsing generator function. Returns an iterator object. Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/docs/parsers/date.md b/docs/parsers/date.md index 4872bf4c..7132ed80 100644 --- a/docs/parsers/date.md +++ b/docs/parsers/date.md @@ -3,9 +3,11 @@ # jc.parsers.date jc - JSON CLI output utility `date` command output parser -The `epoch` calculated timestamp field is naive. (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the timezone field is UTC. Usage (cli): @@ -17,6 +19,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('date', date_command_output) + + or + import jc.parsers.date result = jc.parsers.date.parse(date_command_output) @@ -35,15 +42,19 @@ Schema: "second": integer, "period": string, "timezone": string, - "utc_offset": string, # null if timezone field is not UTC + "utc_offset": string, # null if timezone field is not UTC "day_of_year": integer, "week_of_year": integer, "iso": string, - "epoch": integer, # naive timestamp - "epoch_utc": integer, # timezone-aware timestamp. Only available if timezone field is UTC - "timezone_aware": boolean # if true, all fields are correctly based on UTC + "epoch": integer, # [0] + "epoch_utc": integer, # [1] + "timezone_aware": boolean # [2] } + [0] naive timestamp + [1] timezone-aware timestamp. Only available if timezone field is UTC + [2] if true, all fields are correctly based on UTC + Examples: $ date | jc --date -p @@ -86,7 +97,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/df.md b/docs/parsers/df.md index 5db298d2..6087b001 100644 --- a/docs/parsers/df.md +++ b/docs/parsers/df.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('df', df_command_output) + + or + import jc.parsers.df result = jc.parsers.df.parse(df_command_output) @@ -112,7 +117,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/dig.md b/docs/parsers/dig.md index c4abdcd1..c727c763 100644 --- a/docs/parsers/dig.md +++ b/docs/parsers/dig.md @@ -4,12 +4,15 @@ jc - JSON CLI output utility `dig` command output parser Options supported: -- `+noall +answer` options are supported in cases where only the answer information is desired. +- `+noall +answer` options are supported in cases where only the answer + information is desired. - `+axfr` option is supported on its own -The `when_epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `when_epoch` calculated timestamp field is naive. (i.e. based on the +local time of the system the parser is run on) -The `when_epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `when_epoch_utc` calculated timestamp field is timezone-aware and is +only available if the timezone field is UTC. Usage (cli): @@ -21,6 +24,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('dig', dig_command_output) + + or + import jc.parsers.dig result = jc.parsers.dig.parse(dig_command_output) @@ -93,13 +101,16 @@ Schema: "query_time": integer, # in msec "server": string, "when": string, - "when_epoch": integer, # naive timestamp if when field is parsable, else null - "when_epoch_utc": integer, # timezone aware timestamp availabe for UTC, else null + "when_epoch": integer, # [0] + "when_epoch_utc": integer, # [1] "rcvd": integer "size": string } ] + [0] naive timestamp if "when" field is parsable, else null + [1] timezone aware timestamp availabe for UTC, else null + Examples: $ dig example.com | jc --dig -p @@ -331,7 +342,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/dir.md b/docs/parsers/dir.md index a7a48e69..9d59d602 100644 --- a/docs/parsers/dir.md +++ b/docs/parsers/dir.md @@ -9,9 +9,11 @@ Options supported: - `/C, /-C` - `/S` -The "Magic" syntax is not supported since the `dir` command is a shell builtin. +The "Magic" syntax is not supported since the `dir` command is a shell +builtin. -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) Usage (cli): @@ -19,6 +21,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('dir', dir_command_output) + + or + import jc.parsers.dir result = jc.parsers.dir.parse(dir_command_output) @@ -133,7 +140,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/dmidecode.md b/docs/parsers/dmidecode.md index 5be09999..6a548ffe 100644 --- a/docs/parsers/dmidecode.md +++ b/docs/parsers/dmidecode.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('dmidecode', dmidecode_command_output) + + or + import jc.parsers.dmidecode result = jc.parsers.dmidecode.parse(dmidecode_command_output) @@ -24,7 +29,7 @@ Schema: "type": integer, "bytes": integer, "description": string, - "values": { (null if empty) + "values": { # null if empty "lowercase_no_spaces_keys": string, "multiline_key_values": [ string, @@ -140,7 +145,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/dpkg_l.md b/docs/parsers/dpkg_l.md index 706847f5..9080b111 100644 --- a/docs/parsers/dpkg_l.md +++ b/docs/parsers/dpkg_l.md @@ -3,7 +3,8 @@ # jc.parsers.dpkg_l jc - JSON CLI output utility `dpkg -l` command output parser -Set the `COLUMNS` environment variable to a large value to avoid field truncation. For example: +Set the `COLUMNS` environment variable to a large value to avoid field +truncation. For example: $ COLUMNS=500 dpkg -l | jc --dpkg-l @@ -17,6 +18,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('dpkg_l', dpkg_command_output) + + or + import jc.parsers.dpkg_l result = jc.parsers.dpkg_l.parse(dpkg_command_output) @@ -72,7 +78,7 @@ Examples: "name": "acpid", "version": "1:2.0.28-1ubuntu1", "architecture": "amd64", - "description": "Advanced Configuration and Power Interface event daemon", + "description": "Advanced Configuration and Power Interface...", "desired": "remove", "status": "half installed" }, @@ -116,7 +122,7 @@ Examples: "name": "acpid", "version": "1:2.0.28-1ubuntu1", "architecture": "amd64", - "description": "Advanced Configuration and Power Interface event daemon" + "description": "Advanced Configuration and Power Interface..." }, { "codes": "pn", @@ -145,7 +151,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/du.md b/docs/parsers/du.md index 5e795d39..c17c26b2 100644 --- a/docs/parsers/du.md +++ b/docs/parsers/du.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('du', du_command_output) + + or + import jc.parsers.du result = jc.parsers.du.parse(du_command_output) @@ -35,23 +40,23 @@ Examples: }, { "size": 56, - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/_CodeSignature" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": 0, - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr/local/standalone" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": 0, - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr/local" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": 0, - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": 1008, - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/dfu" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, ... ] @@ -64,23 +69,23 @@ Examples: }, { "size": "56", - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/_CodeSignature" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": "0", - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr/local/standalone" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": "0", - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr/local" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": "0", - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": "1008", - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/dfu" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, ... ] @@ -102,7 +107,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/env.md b/docs/parsers/env.md index e141320e..e2b8525a 100644 --- a/docs/parsers/env.md +++ b/docs/parsers/env.md @@ -3,7 +3,10 @@ # jc.parsers.env jc - JSON CLI output utility `env` and `printenv` command output parser -This parser will output a list of dictionaries each containing `name` and `value` keys. If you would like a simple dictionary output, then use the `-r` command-line option or the `raw=True` argument in the `parse()` function. +This parser will output a list of dictionaries each containing `name` and +`value` keys. If you would like a simple dictionary output, then use the +`-r` command-line option or the `raw=True` argument in the `parse()` +function. Usage (cli): @@ -15,6 +18,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('env', env_command_output) + + or + import jc.parsers.env result = jc.parsers.env.parse(env_command_output) @@ -84,7 +92,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/file.md b/docs/parsers/file.md index a9c7bfeb..bb8a211e 100644 --- a/docs/parsers/file.md +++ b/docs/parsers/file.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('file', file_command_output) + + or + import jc.parsers.file result = jc.parsers.file.parse(file_command_output) @@ -21,7 +26,7 @@ Schema: [ { "filename": string, - "type ": string + "type": string } ] @@ -51,11 +56,11 @@ Examples: }, { "filename": "cd_catalog.xml", - "type": "XML 1.0 document text, ASCII text, with CRLF line terminators" + "type": "XML 1.0 document text, ASCII text, with CRLF line ..." }, { "filename": "centosserial.sh", - "type": "Bourne-Again shell script text executable, UTF-8 Unicode text" + "type": "Bourne-Again shell script text executable, UTF-8 ..." }, ... ] @@ -77,7 +82,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/finger.md b/docs/parsers/finger.md index 111f4043..09e59400 100644 --- a/docs/parsers/finger.md +++ b/docs/parsers/finger.md @@ -15,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('finger', finger_command_output) + + or + import jc.parsers.finger result = jc.parsers.finger.parse(finger_command_output) @@ -105,7 +110,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/free.md b/docs/parsers/free.md index b30a22cc..25e7b826 100644 --- a/docs/parsers/free.md +++ b/docs/parsers/free.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('free', free_command_output) + + or + import jc.parsers.free result = jc.parsers.free.parse(free_command_output) @@ -87,7 +92,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/fstab.md b/docs/parsers/fstab.md index bf94c524..ede53d87 100644 --- a/docs/parsers/fstab.md +++ b/docs/parsers/fstab.md @@ -9,6 +9,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('fstab', fstab_command_output) + + or + import jc.parsers.fstab result = jc.parsers.fstab.parse(fstab_command_output) @@ -100,7 +105,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/group.md b/docs/parsers/group.md index a644c566..04d2be6d 100644 --- a/docs/parsers/group.md +++ b/docs/parsers/group.md @@ -9,6 +9,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('group', group_file_output) + + or + import jc.parsers.group result = jc.parsers.group.parse(group_file_output) @@ -124,7 +129,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/gshadow.md b/docs/parsers/gshadow.md index 46709d9d..8fb1c089 100644 --- a/docs/parsers/gshadow.md +++ b/docs/parsers/gshadow.md @@ -9,6 +9,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('gshadow', gshadow_file_output) + + or + import jc.parsers.gshadow result = jc.parsers.gshadow.parse(gshadow_file_output) @@ -92,7 +97,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/hash.md b/docs/parsers/hash.md index 5042bf45..c2874b41 100644 --- a/docs/parsers/hash.md +++ b/docs/parsers/hash.md @@ -9,6 +9,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('hash', hash_command_output) + + or + import jc.parsers.hash result = jc.parsers.hash.parse(hash_command_output) @@ -52,7 +57,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/hashsum.md b/docs/parsers/hashsum.md index bbe3b318..d4894dc5 100644 --- a/docs/parsers/hashsum.md +++ b/docs/parsers/hashsum.md @@ -23,6 +23,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('hashsum', md5sum_command_output) + + or + import jc.parsers.hashsum result = jc.parsers.hashsum.parse(md5sum_command_output) @@ -83,7 +88,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/hciconfig.md b/docs/parsers/hciconfig.md index d86ce1d3..191f941d 100644 --- a/docs/parsers/hciconfig.md +++ b/docs/parsers/hciconfig.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('hciconfig', hciconfig_command_output) + + or + import jc.parsers.hciconfig result = jc.parsers.hciconfig.parse(hciconfig_command_output) @@ -332,7 +337,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/history.md b/docs/parsers/history.md index aec2f0cd..6b082014 100644 --- a/docs/parsers/history.md +++ b/docs/parsers/history.md @@ -3,9 +3,13 @@ # jc.parsers.history jc - JSON CLI output utility `history` command output parser -This parser will output a list of dictionaries each containing `line` and `command` keys. If you would like a simple dictionary output, then use the `-r` command-line option or the `raw=True` argument in the `parse()` function. +This parser will output a list of dictionaries each containing `line` and +`command` keys. If you would like a simple dictionary output, then use the +`-r` command-line option or the `raw=True` argument in the `parse()` +function. -The "Magic" syntax is not supported since the `history` command is a shell builtin. +The "Magic" syntax is not supported since the `history` command is a shell +builtin. Usage (cli): @@ -13,6 +17,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('history', history_command_output) + + or + import jc.parsers.history result = jc.parsers.history.parse(history_command_output) @@ -74,7 +83,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/hosts.md b/docs/parsers/hosts.md index f4646d4e..0ba8e85f 100644 --- a/docs/parsers/hosts.md +++ b/docs/parsers/hosts.md @@ -9,6 +9,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('hosts', hosts_file_output) + + or + import jc.parsers.hosts result = jc.parsers.hosts.parse(hosts_file_output) @@ -89,7 +94,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/id.md b/docs/parsers/id.md index 2b7a4038..427f93d4 100644 --- a/docs/parsers/id.md +++ b/docs/parsers/id.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('id', id_command_output) + + or + import jc.parsers.id result = jc.parsers.id.parse(id_command_output) @@ -120,7 +125,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ifconfig.md b/docs/parsers/ifconfig.md index 568b6f28..67b2ddf5 100644 --- a/docs/parsers/ifconfig.md +++ b/docs/parsers/ifconfig.md @@ -15,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ifconfig', ifconfig_command_output) + + or + import jc.parsers.ifconfig result = jc.parsers.ifconfig.parse(ifconfig_command_output) @@ -201,7 +206,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ini.md b/docs/parsers/ini.md index 775bdd39..810f5073 100644 --- a/docs/parsers/ini.md +++ b/docs/parsers/ini.md @@ -3,9 +3,13 @@ # jc.parsers.ini jc - JSON CLI output utility `INI` file parser -Parses standard `INI` files and files containing simple key/value pairs. Delimiter can be `=` or `:`. Missing values are supported. Comment prefix can be `#` or `;`. Comments must be on their own line. +Parses standard `INI` files and files containing simple key/value pairs. +Delimiter can be `=` or `:`. Missing values are supported. Comment prefix +can be `#` or `;`. Comments must be on their own line. -Note: Values starting and ending with quotation marks will have the marks removed. If you would like to keep the quotation marks, use the `-r` command-line argument or the `raw=True` argument in `parse()`. +Note: Values starting and ending with quotation marks will have the marks +removed. If you would like to keep the quotation marks, use the `-r` +command-line argument or the `raw=True` argument in `parse()`. Usage (cli): @@ -13,16 +17,18 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ini', ini_file_output) + + or + import jc.parsers.ini result = jc.parsers.ini.parse(ini_file_output) Schema: - ini or key/value document converted to a dictionary - see configparser standard - library documentation for more details. - - Note: Values starting and ending with quotation marks will have the marks removed. - If you would like to keep the quotation marks, use the -r or raw=True argument. + ini or key/value document converted to a dictionary - see the + configparser standard library documentation for more details. { "key1": string, @@ -80,7 +86,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/iostat.md b/docs/parsers/iostat.md index 6d9417ed..e2482252 100644 --- a/docs/parsers/iostat.md +++ b/docs/parsers/iostat.md @@ -15,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('iostat', iostat_command_output) + + or + import jc.parsers.iostat result = jc.parsers.iostat.parse(iostat_command_output) @@ -174,7 +179,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/iostat_s.md b/docs/parsers/iostat_s.md index 0acda54f..3696ff4b 100644 --- a/docs/parsers/iostat_s.md +++ b/docs/parsers/iostat_s.md @@ -13,8 +13,17 @@ Usage (cli): Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('iostat_s', iostat_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.iostat_s - result = jc.parsers.iostat_s.parse(iostat_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.iostat_s.parse(iostat_command_output.splitlines()) for item in result: # do something @@ -71,9 +80,12 @@ Schema: "percent_util": float, "percent_rrqm": float, "percent_wrqm": float, - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error + "success": boolean, # false if error parsing "error": string, # exists if "success" is false "line": string # exists if "success" is false } @@ -82,13 +94,13 @@ Schema: Examples: $ iostat | jc --iostat-s - {"percent_user":0.14,"percent_nice":0.0,"percent_system":0.16,"percent_iowait":0.0,"percent_steal":0.0,"percent_idle":99.7,"type":"cpu"} - {"device":"sda","tps":0.24,"kb_read_s":5.28,"kb_wrtn_s":1.1,"kb_read":203305,"kb_wrtn":42368,"type":"device"} + {"percent_user":0.14,"percent_nice":0.0,"percent_system":0.16,...} + {"device":"sda","tps":0.24,"kb_read_s":5.28,"kb_wrtn_s":1.1...} ... $ iostat | jc --iostat-s -r - {"percent_user":"0.14","percent_nice":"0.00","percent_system":"0.16","percent_iowait":"0.00","percent_steal":"0.00","percent_idle":"99.70","type":"cpu"} - {"device":"sda","tps":"0.24","kb_read_s":"5.28","kb_wrtn_s":"1.10","kb_read":"203305","kb_wrtn":"42368","type":"device"} + {"percent_user":"0.14","percent_nice":"0.00","percent_system":"0.16"...} + {"device":"sda","tps":"0.24","kb_read_s":"5.28","kb_wrtn_s":"1.10"...} ... @@ -107,8 +119,10 @@ Main text parsing generator function. Returns an iterator object. Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/docs/parsers/iptables.md b/docs/parsers/iptables.md index 5660af97..3f3c41fa 100644 --- a/docs/parsers/iptables.md +++ b/docs/parsers/iptables.md @@ -1,7 +1,7 @@ [Home](https://kellyjonbrazil.github.io/jc/) # jc.parsers.iptables -jc - JSON CLI output utility `ipables` command output parser +jc - JSON CLI output utility `iptables` command output parser Supports `-vLn` and `--line-numbers` for all tables. @@ -15,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('iptables', iptables_command_output) + + or + import jc.parsers.iptables result = jc.parsers.iptables.parse(iptables_command_output) @@ -178,7 +183,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/iw_scan.md b/docs/parsers/iw_scan.md index ab0533b1..b9932f17 100644 --- a/docs/parsers/iw_scan.md +++ b/docs/parsers/iw_scan.md @@ -3,7 +3,8 @@ # jc.parsers.iw_scan jc - JSON CLI output utility `iw dev scan` command output parser -This parser is considered beta quality. Not all fields are parsed and there are not enough samples to test. +This parser is considered beta quality. Not all fields are parsed and there +are not enough samples to test. Usage (cli): @@ -15,14 +16,19 @@ Usage (cli): Usage (module): - import jc.parsers.iw-scan - result = jc.parsers.iw-scan.parse(iw-scan_command_output) + import jc + result = jc.parse('iw_scan', iw_scan_command_output) + + or + + import jc.parsers.iw_scan + result = jc.parsers.iw_scan.parse(iw_scan_command_output) Schema: [ { - "foo": string/integer/float, # best guess based on value + "foo": string/integer/float, # best guess based on value "bar": string/integer/float, "baz": string/integer/float } @@ -135,7 +141,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/jar_manifest.md b/docs/parsers/jar_manifest.md index fd30265d..b3014594 100644 --- a/docs/parsers/jar_manifest.md +++ b/docs/parsers/jar_manifest.md @@ -9,6 +9,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('jar_manifest', jar_manifest_file_output) + + or + import jc.parsers.jar_manifest result = jc.parsers.jar_manifest.parse(jar_manifest_file_output) @@ -24,15 +29,16 @@ Schema: Examples: $ cat MANIFEST.MF | jc --jar-manifest -p - $ unzip -c apache-log4j-2.16.0-bin/log4j-core-2.16.0.jar META-INF/MANIFEST.MF | jc --jar-manifest -p - $ unzip -c 'apache-log4j-2.16.0-bin/*.jar' META-INF/MANIFEST.MF | jc --jar-manifest -p + $ unzip -c log4j-core-2.16.0.jar META-INF/MANIFEST.MF | \ + jc --jar-manifest -p + $ unzip -c 'apache-log4j-2.16.0-bin/*.jar' META-INF/MANIFEST.MF | \ + jc --jar-manifest -p $ cat MANIFEST.MF | jc --jar-manifest -p - [ { - "Import_Package": "com.conversantmedia.util.concurrent;resolution:=optional,com.fasterxml.jackson.annotation;version="[2.12,3)";resolution:=optional,com.fasterxml.jackson.core;version="[2.12,3)";resolution:=optional,com.fasterxml.jackson.core.type;version="[2.12,3)";resolution:=optional,com.fasterxml.jackson.cor...", - "Export_Package": "org.apache.logging.log4j.core;uses:="org.apache.logging.log4j,org.apache.logging.log4j.core.config,org.apache.logging.log4j.core.impl,org.apache.logging.log4j.core.layout,org.apache.logging.log4j.core.time,org.apache.logging.log4j.message,org.apache.logging.log4j.spi,org.apache.logging.log4j.status...", + "Import_Package": "com.conversantmedia.util.concurrent;resoluti...", + "Export_Package": "org.apache.logging.log4j.core;uses:="org.ap...", "Manifest_Version": "1.0", "Bundle_License": "https://www.apache.org/licenses/LICENSE-2.0.txt", "Bundle_SymbolicName": "org.apache.logging.log4j.core", @@ -45,28 +51,28 @@ Examples: } ] - $ unzip -c 'apache-log4j-2.16.0-bin/*.jar' META-INF/MANIFEST.MF | jc --jar-manifest -p - + $ unzip -c 'apache-log4j-2.16.0-bin/*.jar' META-INF/MANIFEST.MF | \ + jc --jar-manifest -p [ ... { - "Archive": "apache-log4j-2.16.0-bin/log4j-spring-boot-2.16.0-sources.jar", + "Archive": "apache-log4j-2.16.0-bin/log4j-spring-boot-2.16.0-so...", "Manifest_Version": "1.0", "Built_By": "matt", "Created_By": "Apache Maven 3.8.4", "Build_Jdk": "1.8.0_312" }, { - "Archive": "apache-log4j-2.16.0-bin/log4j-spring-boot-2.16.0-javadoc.jar", + "Archive": "apache-log4j-2.16.0-bin/log4j-spring-boot-2.16.0-ja...", "Manifest_Version": "1.0", "Built_By": "matt", "Created_By": "Apache Maven 3.8.4", "Build_Jdk": "1.8.0_312" }, { - "Bundle_SymbolicName": "org.apache.logging.log4j.spring-cloud-config-client.logging.log4j.core.util;version="[2.16,3)",org.springframework.boot.autoconfigure.condition,org.springframework.cloud.context.environment,org.springframework.context,org.springframework.stereotype", - "Export_Package": "org.apache.logging.log4j.spring.cloud.config.controller;version="2.16.0"ient", - "Archive": "apache-log4j-2.16.0-bin/log4j-spring-cloud-config-client-2.16.0.jar", + "Bundle_SymbolicName": "org.apache.logging.log4j.spring-cloud-c...", + "Export_Package": "org.apache.logging.log4j.spring.cloud.config...", + "Archive": "apache-log4j-2.16.0-bin/log4j-spring-cloud-config-c...", "Manifest_Version": "1.0", "Bundle_License": "https://www.apache.org/licenses/LICENSE-2.0.txt", ... @@ -91,7 +97,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/jobs.md b/docs/parsers/jobs.md index 56aebae2..2810277c 100644 --- a/docs/parsers/jobs.md +++ b/docs/parsers/jobs.md @@ -5,7 +5,8 @@ jc - JSON CLI output utility `jobs` command output parser Also supports the `-l` option. -The "Magic" syntax is not supported since the `jobs` command is a shell builtin. +The "Magic" syntax is not supported since the `jobs` command is a shell +builtin. Usage (cli): @@ -13,6 +14,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('jobs', jobs_command_output) + + or + import jc.parsers.jobs result = jc.parsers.jobs.parse(jobs_command_output) @@ -107,7 +113,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/kv.md b/docs/parsers/kv.md index e8cdfe96..391a82a4 100644 --- a/docs/parsers/kv.md +++ b/docs/parsers/kv.md @@ -3,9 +3,13 @@ # jc.parsers.kv jc - JSON CLI output utility `Key/Value` file parser -Supports files containing simple key/value pairs. Delimiter can be `=` or `:`. Missing values are supported. Comment prefix can be `#` or `;`. Comments must be on their own line. +Supports files containing simple key/value pairs. Delimiter can be `=` or +`:`. Missing values are supported. Comment prefix can be `#` or `;`. +Comments must be on their own line. -Note: Values starting and ending with quotation marks will have the marks removed. If you would like to keep the quotation marks, use the `-r` command-line argument or the `raw=True` argument in `parse()`. +Note: Values starting and ending with quotation marks will have the marks +removed. If you would like to keep the quotation marks, use the `-r` +command-line argument or the `raw=True` argument in `parse()`. Usage (cli): @@ -13,12 +17,18 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('kv', kv_file_output) + + or + import jc.parsers.kv result = jc.parsers.kv.parse(kv_file_output) Schema: - key/value document converted to a dictionary - see configparser standard library documentation for more details. + key/value document converted to a dictionary - see the + configparser standard library documentation for more details. { "key1": string, @@ -65,7 +75,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/last.md b/docs/parsers/last.md index 47c8e1e8..75ac2eea 100644 --- a/docs/parsers/last.md +++ b/docs/parsers/last.md @@ -5,7 +5,9 @@ jc - JSON CLI output utility `last` and `lastb` command output parser Supports `-w` and `-F` options. -Calculated epoch time fields are naive (i.e. based on the local time of the system the parser is run on) since there is no timezone information in the `last` command output. +Calculated epoch time fields are naive (i.e. based on the local time of the +system the parser is run on) since there is no timezone information in the +`last` command output. Usage (cli): @@ -17,6 +19,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('last', last_command_output) + + or + import jc.parsers.last result = jc.parsers.last.parse(last_command_output) @@ -30,9 +37,9 @@ Schema: "login": string, "logout": string, "duration": string, - "login_epoch": integer, # (naive) available with last -F option - "logout_epoch": integer, # (naive) available with last -F option - "duration_seconds": integer # available with last -F option + "login_epoch": integer, # (naive) available w/last -F option + "logout_epoch": integer, # (naive) available w/last -F option + "duration_seconds": integer # available w/last -F option } ] @@ -118,7 +125,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ls.md b/docs/parsers/ls.md index f09aa824..67bc7199 100644 --- a/docs/parsers/ls.md +++ b/docs/parsers/ls.md @@ -7,11 +7,17 @@ Options supported: - `lbaR1` - `--time-style=full-iso` -Note: The `-1`, `-l`, or `-b` option of `ls` should be used to correctly parse filenames that include newline characters. Since `ls` does not encode newlines in filenames when outputting to a pipe it will cause `jc` to see multiple files instead of a single file if `-1`, `-l`, or `-b` is not used. Alternatively, `vdir` can be used, which is the same as running `ls -lb`. +Note: The `-1`, `-l`, or `-b` option of `ls` should be used to correctly +parse filenames that include newline characters. Since `ls` does not encode +newlines in filenames when outputting to a pipe it will cause `jc` to see +multiple files instead of a single file if `-1`, `-l`, or `-b` is not used. +Alternatively, `vdir` can be used, which is the same as running `ls -lb`. -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the timezone field is UTC. Usage (cli): @@ -23,6 +29,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ls', ls_command_output) + + or + import jc.parsers.ls result = jc.parsers.ls.parse(ls_command_output) @@ -38,11 +49,15 @@ Schema: "group": string, "size": integer, "date": string, - "epoch": integer, # naive timestamp if date field exists and can be converted - "epoch_utc": integer # timezone aware timestamp if date field is in UTC and can be converted + "epoch": integer, # [0] + "epoch_utc": integer # [1] } ] + [0] naive timestamp if date field exists and can be converted. + [1] timezone aware timestamp if date field is in UTC and can + be converted. + Examples: $ ls /usr/bin | jc --ls -p @@ -121,7 +136,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ls_s.md b/docs/parsers/ls_s.md index cc1f0409..6a4374cf 100644 --- a/docs/parsers/ls_s.md +++ b/docs/parsers/ls_s.md @@ -1,17 +1,22 @@ [Home](https://kellyjonbrazil.github.io/jc/) # jc.parsers.ls_s -jc - JSON CLI output utility `ls` and `vdir` command output streaming parser +jc - JSON CLI output utility `ls` and `vdir` command output streaming +parser > This streaming parser outputs JSON Lines -Requires the `-l` option to be used on `ls`. If there are newline characters in the filename, then make sure to use the `-b` option on `ls`. +Requires the `-l` option to be used on `ls`. If there are newline characters +in the filename, then make sure to use the `-b` option on `ls`. -The `jc` `-qq` option can be used to ignore parsing errors. (e.g. filenames with newline characters, but `-b` was not used) +The `jc` `-qq` option can be used to ignore parsing errors. (e.g. filenames +with newline characters, but `-b` was not used) -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive (i.e. based on the local +time of the system the parser is run on) -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the timezone field is UTC. Usage (cli): @@ -19,8 +24,17 @@ Usage (cli): Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('ls_s', ls_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.ls_s - result = jc.parsers.ls_s.parse(ls_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.ls_s.parse(ls_command_output.splitlines()) for item in result: # do something @@ -35,28 +49,35 @@ Schema: "group": string, "size": integer, "date": string, - "epoch": integer, # naive timestamp if date field exists and can be converted - "epoch_utc": integer, # timezone aware timestamp if date field is in UTC and can be converted - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + "epoch": integer, # [0] + "epoch_utc": integer, # [1] + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error + "success": boolean, # false if error parsing "error": string, # exists if "success" is false "line": string # exists if "success" is false } } + [0] naive timestamp if date field exists and can be converted. + [1] timezone aware timestamp if date field is in UTC and can + be converted + Examples: $ ls -l /usr/bin | jc --ls-s - {"filename":"2to3-","flags":"-rwxr-xr-x","links":4,"owner":"root","group":"wheel","size":925,"date":"Feb 22 2019"} - {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7","flags":"lrwxr-xr-x","links":1,"owner":"root","group":"wheel","size":74,"date":"May 4 2019"} - {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":1,"owner":"root","group":"wheel","size":55152,"date":"May 3 2019"} + {"filename":"2to3-","flags":"-rwxr-xr-x","links":4,"owner":"root","...} + {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/P...} + {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":1,...} ... $ ls -l /usr/bin | jc --ls-s -r - {"filename":"2to3-","flags":"-rwxr-xr-x","links":"4","owner":"root","group":"wheel","size":"925","date":"Feb 22 2019"} - {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7","flags":"lrwxr-xr-x","links":"1","owner":"root","group":"wheel","size":"74","date":"May 4 2019"} - {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":"1","owner":"root","group":"wheel","size":"55152","date":"May 3 2019"} + {"filename":"2to3-","flags":"-rwxr-xr-x","links":"4","owner":"roo"..."} + {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/P...} + {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":"1...} ... @@ -75,8 +96,10 @@ Main text parsing generator function. Returns an iterator object. Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/docs/parsers/lsblk.md b/docs/parsers/lsblk.md index 685d9146..265593ab 100644 --- a/docs/parsers/lsblk.md +++ b/docs/parsers/lsblk.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('lsblk', lsblk_command_output) + + or + import jc.parsers.lsblk result = jc.parsers.lsblk.parse(lsblk_command_output) @@ -88,7 +93,10 @@ Examples: ... ] - $ lsblk -o +KNAME,FSTYPE,LABEL,UUID,PARTLABEL,PARTUUID,RA,MODEL,SERIAL,STATE,OWNER,GROUP,MODE,ALIGNMENT,MIN-IO,OPT-IO,PHY-SEC,LOG-SEC,ROTA,SCHED,RQ-SIZE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,WSAME,WWN,RAND,PKNAME,HCTL,TRAN,REV,VENDOR | jc --lsblk -p + $ lsblk -o +KNAME,FSTYPE,LABEL,UUID,PARTLABEL,PARTUUID,RA,MODEL,SERIAL,\ + STATE,OWNER,GROUP,MODE,ALIGNMENT,MIN-IO,OPT-IO,PHY-SEC,LOG-SEC,ROTA,\ + SCHED,RQ-SIZE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,WSAME,WWN,RAND,\ + PKNAME,HCTL,TRAN,REV,VENDOR | jc --lsblk -p [ { "name": "sda", @@ -177,7 +185,10 @@ Examples: ... ] - $ lsblk -o +KNAME,FSTYPE,LABEL,UUID,PARTLABEL,PARTUUID,RA,MODEL,SERIAL,STATE,OWNER,GROUP,MODE,ALIGNMENT,MIN-IO,OPT-IO,PHY-SEC,LOG-SEC,ROTA,SCHED,RQ-SIZE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,WSAME,WWN,RAND,PKNAME,HCTL,TRAN,REV,VENDOR | jc --lsblk -p -r + $ lsblk -o +KNAME,FSTYPE,LABEL,UUID,PARTLABEL,PARTUUID,RA,MODEL,SERIAL,\ + STATE,OWNER,GROUP,MODE,ALIGNMENT,MIN-IO,OPT-IO,PHY-SEC,LOG-SEC,ROTA,\ + SCHED,RQ-SIZE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,WSAME,WWN,RAND,\ + PKNAME,HCTL,TRAN,REV,VENDOR | jc --lsblk -p -r [ { "name": "sda", @@ -283,7 +294,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/lsmod.md b/docs/parsers/lsmod.md index 99cfcb2e..2cb65ce4 100644 --- a/docs/parsers/lsmod.md +++ b/docs/parsers/lsmod.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('lsmod', lsmod_command_output) + + or + import jc.parsers.lsmod result = jc.parsers.lsmod.parse(lsmod_command_output) @@ -140,7 +145,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/lsof.md b/docs/parsers/lsof.md index 165c4473..6003595f 100644 --- a/docs/parsers/lsof.md +++ b/docs/parsers/lsof.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('lsof', lsof_command_output) + + or + import jc.parsers.lsof result = jc.parsers.lsof.parse(lsof_command_output) @@ -134,7 +139,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/lsusb.md b/docs/parsers/lsusb.md index 707e66ec..73198739 100644 --- a/docs/parsers/lsusb.md +++ b/docs/parsers/lsusb.md @@ -15,13 +15,19 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('lsusb', lsusb_command_output) + + or + import jc.parsers.lsusb result = jc.parsers.lsusb.parse(lsusb_command_output) Schema: - Note: object keynames are assigned directly from the lsusb output. - If there are duplicate names in a section, only the last one is converted. + Note: object keynames are assigned directly from the lsusb + output. If there are duplicate names in a section, only the + last one is converted. [ { @@ -275,7 +281,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/mount.md b/docs/parsers/mount.md index b0015589..cf85d0f7 100644 --- a/docs/parsers/mount.md +++ b/docs/parsers/mount.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('mount', mount_command_output) + + or + import jc.parsers.mount result = jc.parsers.mount.parse(mount_command_output) @@ -90,7 +95,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/netstat.md b/docs/parsers/netstat.md index 58758941..fcd0173a 100644 --- a/docs/parsers/netstat.md +++ b/docs/parsers/netstat.md @@ -5,7 +5,8 @@ jc - JSON CLI output utility `netstat` command output parser Caveats: - Use of multiple `l` options is not supported on OSX (e.g. `netstat -rlll`) -- Use of the `A` option is not supported on OSX when using the `r` option (e.g. `netstat -rA`) +- Use of the `A` option is not supported on OSX when using the `r` option + (e.g. `netstat -rA`) Usage (cli): @@ -17,6 +18,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('netstat', netstat_command_output) + + or + import jc.parsers.netstat result = jc.parsers.netstat.parse(netstat_command_output) @@ -369,7 +375,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ntpq.md b/docs/parsers/ntpq.md index 5e389e41..71293dc4 100644 --- a/docs/parsers/ntpq.md +++ b/docs/parsers/ntpq.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ntpq', ntpq_command_output) + + or + import jc.parsers.ntpq result = jc.parsers.ntpq.parse(ntpq_command_output) @@ -221,7 +226,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/passwd.md b/docs/parsers/passwd.md index 312a9ab5..6253da68 100644 --- a/docs/parsers/passwd.md +++ b/docs/parsers/passwd.md @@ -9,6 +9,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('passwd', passwd_file_output) + + or + import jc.parsers.passwd result = jc.parsers.passwd.parse(passwd_file_output) @@ -109,7 +114,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ping.md b/docs/parsers/ping.md index 1d25d05c..e20ca0c4 100644 --- a/docs/parsers/ping.md +++ b/docs/parsers/ping.md @@ -7,7 +7,8 @@ Supports `ping` and `ping6` output. Usage (cli): - Note: Use the ping `-c` (count) option, otherwise data will not be piped to `jc`. + Note: Use the ping `-c` (count) option, otherwise data will not be + piped to `jc`. $ ping -c 3 1.2.3.4 | jc --ping @@ -17,6 +18,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ping', ping_command_output) + + or + import jc.parsers.ping result = jc.parsers.ping.parse(ping_command_output) @@ -26,7 +32,7 @@ Schema: "source_ip": string, "destination_ip": string, "data_bytes": integer, - "pattern": string, # (null if not set) + "pattern": string, # null if not set "destination": string, "packets_transmitted": integer, "packets_received": integer, @@ -38,8 +44,8 @@ Schema: "round_trip_ms_stddev": float, "responses": [ { - "type": string, # 'reply', 'timeout', 'unparsable_line', etc. See `_error_type.type_map` for all options - "unparsed_line": string, # only if an 'unparsable_line' type + "type": string, # [0] + "unparsed_line": string, # [1] "timestamp": float, "bytes": integer, "response_ip": string, @@ -47,21 +53,26 @@ Schema: "ttl": integer, "time_ms": float, "duplicate": boolean, - "vr": integer, # hex value converted to decimal - "hl": integer, # hex value converted to decimal - "tos": integer, # hex value converted to decimal - "len": integer, # hex value converted to decimal - "id": integer, # hex value converted to decimal - "flg": integer, # hex value converted to decimal - "off": integer, # hex value converted to decimal - "pro": integer, # hex value converted to decimal - "cks": ingeger, # hex value converted to decimal + "vr": integer, # [2] + "hl": integer, # [2] + "tos": integer, # [2] + "len": integer, # [2] + "id": integer, # [2] + "flg": integer, # [2] + "off": integer, # [2] + "pro": integer, # [2] + "cks": ingeger, # [2] "src": string, "dst": string } ] } + [0] 'reply', 'timeout', 'unparsable_line', etc. See + `_error_type.type_map` for all options + [1] only if an 'unparsable_line' type + [2] hex value converted to decimal + Examples: $ ping -c 3 -p ff cnn.com | jc --ping -p @@ -171,7 +182,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ping_s.md b/docs/parsers/ping_s.md index 33a462b0..6d8913af 100644 --- a/docs/parsers/ping_s.md +++ b/docs/parsers/ping_s.md @@ -11,59 +11,79 @@ Usage (cli): $ ping | jc --ping-s -> Note: When piping `jc` converted `ping` output to other processes it may appear the output is hanging due to the OS pipe buffers. This is because `ping` output is too small to quickly fill up the buffer. Use the `-u` option to unbuffer the `jc` output if you would like immediate output. See the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) for more information. +> Note: When piping `jc` converted `ping` output to other processes it may + appear the output is hanging due to the OS pipe buffers. This is because + `ping` output is too small to quickly fill up the buffer. Use the `-u` + option to unbuffer the `jc` output if you would like immediate output. + See the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) + for more information. Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('ping_s', ping_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.ping_s - result = jc.parsers.ping_s.parse(ping_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.ping_s.parse(ping_command_output.splitlines()) for item in result: # do something Schema: { - "type": string, # 'reply', 'timeout', 'summary', etc. See `_error_type.type_map` for all options. - "source_ip": string, - "destination_ip": string, - "sent_bytes": integer, - "pattern": string, # (null if not set) - "destination": string, - "timestamp": float, - "response_bytes": integer, - "response_ip": string, - "icmp_seq": integer, - "ttl": integer, - "time_ms": float, - "duplicate": boolean, - "packets_transmitted": integer, - "packets_received": integer, - "packet_loss_percent": float, - "duplicates": integer, - "round_trip_ms_min": float, - "round_trip_ms_avg": float, - "round_trip_ms_max": float, - "round_trip_ms_stddev": float, - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + "type": string, # [0] + "source_ip": string, + "destination_ip": string, + "sent_bytes": integer, + "pattern": string, # (null if not set) + "destination": string, + "timestamp": float, + "response_bytes": integer, + "response_ip": string, + "icmp_seq": integer, + "ttl": integer, + "time_ms": float, + "duplicate": boolean, + "packets_transmitted": integer, + "packets_received": integer, + "packet_loss_percent": float, + "duplicates": integer, + "round_trip_ms_min": float, + "round_trip_ms_avg": float, + "round_trip_ms_max": float, + "round_trip_ms_stddev": float, + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error - "error": string, # exists if "success" is false - "line": string # exists if "success" is false + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false } } + [0] 'reply', 'timeout', 'summary', etc. See `_error_type.type_map` + for all options. + Examples: $ ping 1.1.1.1 | jc --ping-s - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"pattern":null,"response_bytes":64,"response_ip":"1.1.1.1","icmp_seq":0,"ttl":56,"time_ms":23.703} - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"pattern":null,"response_bytes":64,"response_ip":"1.1.1.1","icmp_seq":1,"ttl":56,"time_ms":22.862} - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"pattern":null,"response_bytes":64,"response_ip":"1.1.1.1","icmp_seq":2,"ttl":56,"time_ms":22.82} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"patte...} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"patte...} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"patte...} ... $ ping 1.1.1.1 | jc --ping-s -r - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","pattern":null,"response_bytes":"64","response_ip":"1.1.1.1","icmp_seq":"0","ttl":"56","time_ms":"23.054"} - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","pattern":null,"response_bytes":"64","response_ip":"1.1.1.1","icmp_seq":"1","ttl":"56","time_ms":"24.739"} - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","pattern":null,"response_bytes":"64","response_ip":"1.1.1.1","icmp_seq":"2","ttl":"56","time_ms":"23.232"} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","patte...} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","patte...} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","patte...} ... @@ -82,8 +102,10 @@ Main text parsing generator function. Returns an iterator object. Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/docs/parsers/pip_list.md b/docs/parsers/pip_list.md index 4b64e2d5..d63c4afb 100644 --- a/docs/parsers/pip_list.md +++ b/docs/parsers/pip_list.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('pip_list', pip_list_command_output) + + or + import jc.parsers.pip_list result = jc.parsers.pip_list.parse(pip_list_command_output) @@ -62,7 +67,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/pip_show.md b/docs/parsers/pip_show.md index 33a58720..cc6c43c7 100644 --- a/docs/parsers/pip_show.md +++ b/docs/parsers/pip_show.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('pip_show', pip_show_command_output) + + or + import jc.parsers.pip_show result = jc.parsers.pip_show.parse(pip_show_command_output) @@ -80,7 +85,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ps.md b/docs/parsers/ps.md index 47dce780..85f1cc8a 100644 --- a/docs/parsers/ps.md +++ b/docs/parsers/ps.md @@ -17,6 +17,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ps', ps_command_output) + + or + import jc.parsers.ps result = jc.parsers.ps.parse(ps_command_output) @@ -56,7 +61,7 @@ Examples: "stime": "Nov01", "tty": null, "time": "00:00:11", - "cmd": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "cmd": "/usr/lib/systemd/systemd --switched-root --system --dese..." }, { "uid": "root", @@ -91,7 +96,7 @@ Examples: "stime": "Nov01", "tty": "?", "time": "00:00:11", - "cmd": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "cmd": "/usr/lib/systemd/systemd --switched-root --system --dese..." }, { "uid": "root", @@ -129,7 +134,7 @@ Examples: "stat": "Ss", "start": "Nov09", "time": "0:08", - "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "command": "/usr/lib/systemd/systemd --switched-root --system --..." }, { "user": "root", @@ -173,7 +178,7 @@ Examples: "stat": "Ss", "start": "Nov09", "time": "0:08", - "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "command": "/usr/lib/systemd/systemd --switched-root --system --..." }, { "user": "root", @@ -221,7 +226,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/route.md b/docs/parsers/route.md index 119c6711..0615d264 100644 --- a/docs/parsers/route.md +++ b/docs/parsers/route.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('route', route_command_output) + + or + import jc.parsers.route result = jc.parsers.route.parse(route_command_output) @@ -123,7 +128,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/rpm_qi.md b/docs/parsers/rpm_qi.md index 38a08c05..5125ac7a 100644 --- a/docs/parsers/rpm_qi.md +++ b/docs/parsers/rpm_qi.md @@ -5,9 +5,11 @@ jc - JSON CLI output utility `rpm -qi` command output parser Works with `rpm -qi [package]` or `rpm -qia`. -The `..._epoch` calculated timestamp fields are naive (i.e. based on the local time of the system the parser is run on) +The `..._epoch` calculated timestamp fields are naive. (i.e. based on the +local time of the system the parser is run on) -The `..._epoch_utc` calculated timestamp fields are timezone-aware and is only available if the timezone field is UTC. +The `..._epoch_utc` calculated timestamp fields are timezone-aware and is +only available if the timezone field is UTC. Usage (cli): @@ -19,6 +21,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('rpm_qi', rpm_qi_command_output) + + or + import jc.parsers.rpm_qi result = jc.parsers.rpm_qi.parse(rpm_qi_command_output) @@ -32,16 +39,16 @@ Schema: "release": string, "architecture": string, "install_date": string, - "install_date_epoch": integer, # naive timestamp - "install_date_epoch_utc": integer, # Aware timestamp if timezone is UTC + "install_date_epoch": integer, # [0] + "install_date_epoch_utc": integer, # [1] "group": string, "size": integer, "license": string, "signature": string, "source_rpm": string, "build_date": string, - "build_epoch": integer, # naive timestamp - "build_epoch_utc": integer, # Aware timestamp if timezone is UTC + "build_epoch": integer, # [0] + "build_epoch_utc": integer, # [1] "build_host": string, "relocations": string, "packager": string, @@ -52,6 +59,9 @@ Schema: } ] + [0] naive timestamp + [1] Aware timestamp if timezone is UTC + Examples: $ rpm -qia | jc --rpm-qi -p @@ -66,7 +76,7 @@ Examples: "group": "Development/Tools", "size": 1160660, "license": "GPLv2+", - "signature": "RSA/SHA256, Thu 22 Aug 2019 02:34:59 PM PDT, Key ID 24c6a8a7f4a80eb5", + "signature": "RSA/SHA256, Thu 22 Aug 2019 02:34:59 PM PDT, Key ...", "source_rpm": "make-3.82-24.el7.src.rpm", "build_date": "Thu 08 Aug 2019 05:47:25 PM PDT", "build_host": "x86-01.bsys.centos.org", @@ -74,8 +84,8 @@ Examples: "packager": "CentOS BuildSystem ", "vendor": "CentOS", "url": "http://www.gnu.org/software/make/", - "summary": "A GNU tool which simplifies the build process for users", - "description": "A GNU tool for controlling the generation of executables and other...", + "summary": "A GNU tool which simplifies the build process for ...", + "description": "A GNU tool for controlling the generation of ex...", "build_epoch": 1565311645, "build_epoch_utc": null, "install_date_epoch": 1571242902, @@ -90,7 +100,7 @@ Examples: "group": "System Environment/Base", "size": 503608, "license": "GPLv2+", - "signature": "RSA/SHA256, Mon 12 Nov 2018 07:17:49 AM PST, Key ID 24c6a8a7f4a80eb5", + "signature": "RSA/SHA256, Mon 12 Nov 2018 07:17:49 AM PST, Key ...", "source_rpm": "kbd-1.15.5-15.el7.src.rpm", "build_date": "Tue 30 Oct 2018 03:40:00 PM PDT", "build_host": "x86-01.bsys.centos.org", @@ -99,7 +109,7 @@ Examples: "vendor": "CentOS", "url": "http://ftp.altlinux.org/pub/people/legion/kbd", "summary": "Legacy data for kbd package", - "description": "The kbd-legacy package contains original keymaps for kbd package...", + "description": "The kbd-legacy package contains original keymap...", "build_epoch": 1540939200, "build_epoch_utc": null, "install_date_epoch": 1565891588, @@ -120,7 +130,7 @@ Examples: "group": "Development/Tools", "size": "1160660", "license": "GPLv2+", - "signature": "RSA/SHA256, Thu 22 Aug 2019 02:34:59 PM PDT, Key ID 24c6a8a7f4a80eb5", + "signature": "RSA/SHA256, Thu 22 Aug 2019 02:34:59 PM PDT, Key ...", "source_rpm": "make-3.82-24.el7.src.rpm", "build_date": "Thu 08 Aug 2019 05:47:25 PM PDT", "build_host": "x86-01.bsys.centos.org", @@ -128,8 +138,8 @@ Examples: "packager": "CentOS BuildSystem ", "vendor": "CentOS", "url": "http://www.gnu.org/software/make/", - "summary": "A GNU tool which simplifies the build process for users", - "description": "A GNU tool for controlling the generation of executables and other..." + "summary": "A GNU tool which simplifies the build process for...", + "description": "A GNU tool for controlling the generation of exe..." }, { "name": "kbd-legacy", @@ -140,7 +150,7 @@ Examples: "group": "System Environment/Base", "size": "503608", "license": "GPLv2+", - "signature": "RSA/SHA256, Mon 12 Nov 2018 07:17:49 AM PST, Key ID 24c6a8a7f4a80eb5", + "signature": "RSA/SHA256, Mon 12 Nov 2018 07:17:49 AM PST, Key ...", "source_rpm": "kbd-1.15.5-15.el7.src.rpm", "build_date": "Tue 30 Oct 2018 03:40:00 PM PDT", "build_host": "x86-01.bsys.centos.org", @@ -149,7 +159,7 @@ Examples: "vendor": "CentOS", "url": "http://ftp.altlinux.org/pub/people/legion/kbd", "summary": "Legacy data for kbd package", - "description": "The kbd-legacy package contains original keymaps for kbd package..." + "description": "The kbd-legacy package contains original keymaps..." }, ... ] @@ -171,7 +181,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/sfdisk.md b/docs/parsers/sfdisk.md index c5eededc..70464fc1 100644 --- a/docs/parsers/sfdisk.md +++ b/docs/parsers/sfdisk.md @@ -22,6 +22,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('sfdisk', sfdisk_command_output) + + or + import jc.parsers.sfdisk result = jc.parsers.sfdisk.parse(sfdisk_command_output) @@ -53,7 +58,7 @@ Schema: "boot": boolean, "start": integer, "end": integer, - "size": string, # Note: will be integer when using deprecated -d sfdisk option + "size": string, # [0] "cyls": integer, "mib": integer, "blocks": integer, @@ -66,6 +71,8 @@ Schema: } ] + [0] will be integer when using deprecated -d sfdisk option + Examples: # sfdisk -l | jc --sfdisk -p @@ -75,7 +82,7 @@ Examples: "cylinders": 2610, "heads": 255, "sectors_per_track": 63, - "units": "cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0", + "units": "cylinders of 8225280 bytes, blocks of 1024 bytes, ...", "partitions": [ { "device": "/dev/sda1", @@ -140,7 +147,7 @@ Examples: "cylinders": "2610", "heads": "255", "sectors_per_track": "63", - "units": "cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0", + "units": "cylinders of 8225280 bytes, blocks of 1024 bytes, co...", "partitions": [ { "device": "/dev/sda1", @@ -215,7 +222,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/shadow.md b/docs/parsers/shadow.md index e937ab14..d3323439 100644 --- a/docs/parsers/shadow.md +++ b/docs/parsers/shadow.md @@ -9,6 +9,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('shadow', shadow_file_output) + + or + import jc.parsers.shadow result = jc.parsers.shadow.parse(shadow_file_output) @@ -116,7 +121,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ss.md b/docs/parsers/ss.md index 231055cf..a41d9a9f 100644 --- a/docs/parsers/ss.md +++ b/docs/parsers/ss.md @@ -3,7 +3,8 @@ # jc.parsers.ss jc - JSON CLI output utility `ss` command output parser -Extended information options like -e and -p are not supported and may cause parsing irregularities. +Extended information options like -e and -p are not supported and may cause +parsing irregularities. Usage (cli): @@ -15,12 +16,18 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ss', ss_command_output) + + or + import jc.parsers.ss result = jc.parsers.ss.parse(ss_command_output) Schema: - Information from https://www.cyberciti.biz/files/ss.html used to define field names + Information from https://www.cyberciti.biz/files/ss.html used to define + field names [ { @@ -293,7 +300,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/stat.md b/docs/parsers/stat.md index 05676277..1a1726c4 100644 --- a/docs/parsers/stat.md +++ b/docs/parsers/stat.md @@ -3,9 +3,11 @@ # jc.parsers.stat jc - JSON CLI output utility `stat` command output parser -The `xxx_epoch` calculated timestamp fields are naive (i.e. based on the local time of the system the parser is run on) +The `xxx_epoch` calculated timestamp fields are naive. (i.e. based on the +local time of the system the parser is run on) -The `xxx_epoch_utc` calculated timestamp fields are timezone-aware and are only available if the timezone field is UTC. +The `xxx_epoch_utc` calculated timestamp fields are timezone-aware and are +only available if the timezone field is UTC. Usage (cli): @@ -17,6 +19,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('stat', stat_command_output) + + or + import jc.parsers.stat result = jc.parsers.stat.parse(stat_command_output) @@ -183,7 +190,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/stat_s.md b/docs/parsers/stat_s.md index 3da8b142..9a442c21 100644 --- a/docs/parsers/stat_s.md +++ b/docs/parsers/stat_s.md @@ -5,9 +5,11 @@ jc - JSON CLI output utility `stat` command output streaming parser > This streaming parser outputs JSON Lines -The `xxx_epoch` calculated timestamp fields are naive (i.e. based on the local time of the system the parser is run on). +The `xxx_epoch` calculated timestamp fields are naive. (i.e. based on the +local time of the system the parser is run on). -The `xxx_epoch_utc` calculated timestamp fields are timezone-aware and are only available if the timezone field is UTC. +The `xxx_epoch_utc` calculated timestamp fields are timezone-aware and are +only available if the timezone field is UTC. Usage (cli): @@ -15,8 +17,17 @@ Usage (cli): Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('stat_s', stat_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.stat_s - result = jc.parsers.stat_s.parse(stat_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.stat_s.parse(stat_command_output.splitlines()) for item in result: # do something @@ -54,9 +65,12 @@ Schema: "rdev": integer, "block_size": integer, "unix_flags": string, - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error + "success": boolean, # false if error parsing "error": string, # exists if "success" is false "line": string # exists if "success" is false } @@ -65,10 +79,10 @@ Schema: Examples: $ stat | jc --stat-s - {"file":"(stdin)","unix_device":1027739696,"inode":1155,"flags":"crw--w----","links":1,"user":"kbrazil","group":"tty","rdev":268435456,"size":0,"access_time":"Jan 4 15:27:44 2022","modify_time":"Jan 4 15:27:44 2022","change_time":"Jan 4 15:27:44 2022","birth_time":"Dec 31 16:00:00 1969","block_size":131072,"blocks":0,"unix_flags":"0","access_time_epoch":1641338864,"access_time_epoch_utc":null,"modify_time_epoch":1641338864,"modify_time_epoch_utc":null,"change_time_epoch":1641338864,"change_time_epoch_utc":null,"birth_time_epoch":null,"birth_time_epoch_utc":null} + {"file":"(stdin)","unix_device":1027739696,"inode":1155,"flags":"cr...} $ stat | jc --stat-s -r - {"file":"(stdin)","unix_device":"1027739696","inode":"1155","flags":"crw--w----","links":"1","user":"kbrazil","group":"tty","rdev":"268435456","size":"0","access_time":"Jan 4 15:28:08 2022","modify_time":"Jan 4 15:28:08 2022","change_time":"Jan 4 15:28:08 2022","birth_time":"Dec 31 16:00:00 1969","block_size":"131072","blocks":"0","unix_flags":"0"} + {"file":"(stdin)","unix_device":"1027739696","inode":"1155","flag...} ## info @@ -86,8 +100,10 @@ Main text parsing generator function. Returns an iterator object. Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/docs/parsers/sysctl.md b/docs/parsers/sysctl.md index 7e51fb18..f324bd35 100644 --- a/docs/parsers/sysctl.md +++ b/docs/parsers/sysctl.md @@ -3,7 +3,10 @@ # jc.parsers.sysctl jc - JSON CLI output utility `sysctl -a` command output parser -Note: Since `sysctl` output is not easily parsable only a very simple key/value object will be output. An attempt is made to convert obvious integers and floats. If no conversion is desired, use the `-r` command-line argument or the `raw=True` argument in `parse()`. +Note: Since `sysctl` output is not easily parsable only a very simple + key/value object will be output. An attempt is made to convert obvious + integers and floats. If no conversion is desired, use the `-r` + command-line argument or the `raw=True` argument in `parse()`. Usage (cli): @@ -15,13 +18,18 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('sysctl', sysctl_command_output) + + or + import jc.parsers.sysctl result = jc.parsers.sysctl.parse(sysctl_command_output) Schema: { - "key1": string/integer/float, # best guess based on value + "key1": string/integer/float, # best guess based on value "key2": string/integer/float, "key3": string/integer/float } @@ -69,7 +77,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/systemctl.md b/docs/parsers/systemctl.md index 2c23c110..cb953574 100644 --- a/docs/parsers/systemctl.md +++ b/docs/parsers/systemctl.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('systemctl', systemctl_command_output) + + or + import jc.parsers.systemctl result = jc.parsers.systemctl.parse(systemctl_command_output) @@ -37,14 +42,14 @@ Examples: "load": "loaded", "active": "active", "sub": "waiting", - "description": "Arbitrary Executable File Formats File System Automount Point" + "description": "Arbitrary Executable File Formats File System ..." }, { "unit": "dev-block-8:2.device", "load": "loaded", "active": "active", "sub": "plugged", - "description": "LVM PV 3klkIj-w1qk-DkJi-0XBJ-y3o7-i2Ac-vHqWBM on /dev/sda2 2" + "description": "LVM PV 3klkIj-w1qk-DkJi-0XBJ-y3o7-i2Ac-vHqWBM o..." }, { "unit": "dev-cdrom.device", @@ -73,7 +78,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/systemctl_lj.md b/docs/parsers/systemctl_lj.md index cb38e495..f6fd1f7a 100644 --- a/docs/parsers/systemctl_lj.md +++ b/docs/parsers/systemctl_lj.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('systemctl_lj', systemctl_lj_command_output) + + or + import jc.parsers.systemctl_lj result = jc.parsers.systemctl_lj.parse(systemctl_lj_command_output) @@ -90,7 +95,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/systemctl_ls.md b/docs/parsers/systemctl_ls.md index e4f02318..ce0bbd1c 100644 --- a/docs/parsers/systemctl_ls.md +++ b/docs/parsers/systemctl_ls.md @@ -1,7 +1,8 @@ [Home](https://kellyjonbrazil.github.io/jc/) # jc.parsers.systemctl_ls -jc - JSON CLI output utility `systemctl list-sockets` command output parser +jc - JSON CLI output utility `systemctl list-sockets` command output +parser Usage (cli): @@ -13,6 +14,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('systemctl_ls', systemctl_ls_command_output) + + or + import jc.parsers.systemctl_ls result = jc.parsers.systemctl_ls.parse(systemctl_ls_command_output) @@ -65,7 +71,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/systemctl_luf.md b/docs/parsers/systemctl_luf.md index 477b2f14..c13e2480 100644 --- a/docs/parsers/systemctl_luf.md +++ b/docs/parsers/systemctl_luf.md @@ -1,7 +1,8 @@ [Home](https://kellyjonbrazil.github.io/jc/) # jc.parsers.systemctl_luf -jc - JSON CLI output utility `systemctl list-unit-files` command output parser +jc - JSON CLI output utility `systemctl list-unit-files` command output +parser Usage (cli): @@ -13,6 +14,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('systemctl_luf', systemctl_luf_command_output) + + or + import jc.parsers.systemctl_luf result = jc.parsers.systemctl_luf.parse(systemctl_luf_command_output) @@ -61,7 +67,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/systeminfo.md b/docs/parsers/systeminfo.md index a30216f5..d6af59be 100644 --- a/docs/parsers/systeminfo.md +++ b/docs/parsers/systeminfo.md @@ -5,9 +5,13 @@ jc - JSON CLI output utility `systeminfo` command output parser Blank or missing elements are set to `null`. -The `original_install_date_epoch` and `system_boot_time_epoch` calculated timestamp fields are naive (i.e. based on the local time of the system the parser is run on) +The `original_install_date_epoch` and `system_boot_time_epoch` calculated +timestamp fields are naive. (i.e. based on the local time of the system the +parser is run on) -The `original_install_date_epoch_utc` and `system_boot_time_epoch_utc` calculated timestamp fields are timezone-aware and are only available if the timezone field is UTC. +The `original_install_date_epoch_utc` and `system_boot_time_epoch_utc` +calculated timestamp fields are timezone-aware and are only available if +the timezone field is UTC. Usage (cli): @@ -15,6 +19,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('systeminfo', systeminfo_command_output) + + or + import jc.parsers.systeminfo result = jc.parsers.systeminfo.parse(systeminfo_command_output) @@ -31,11 +40,11 @@ Schema: "registered_organization": string, "product_id": string, "original_install_date": string, - "original_install_date_epoch": integer, # naive timestamp - "original_install_date_epoch_utc": integer, # timezone-aware timestamp + "original_install_date_epoch": integer, # [0] + "original_install_date_epoch_utc": integer, # [1] "system_boot_time": string, - "system_boot_time_epoch": integer, # naive timestamp - "system_boot_time_epoch_utc": integer, # timezone-aware timestamp + "system_boot_time_epoch": integer, # [0] + "system_boot_time_epoch_utc": integer, # [1] "system_manufacturer": string, "system_model": string, "system_type": string, @@ -80,6 +89,9 @@ Schema: } } + [0] naive timestamp + [1] timezone-aware timestamp + Examples: $ systeminfo | jc --systeminfo -p @@ -219,7 +231,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/time.md b/docs/parsers/time.md index 909d74ac..3a4c6bfa 100644 --- a/docs/parsers/time.md +++ b/docs/parsers/time.md @@ -3,18 +3,27 @@ # jc.parsers.time jc - JSON CLI output utility `/usr/bin/time` command output parser -Output from `/usr/bin/time` is sent to `STDERR`, so the `-o` option can be used to redirect the output to a file that can be read by `jc`. +Output from `/usr/bin/time` is sent to `STDERR`, so the `-o` option can be +used to redirect the output to a file that can be read by `jc`. -Alternatively, the output from `/usr/bin/time` can be redirected to `STDOUT` so `jc` can receive it. +Alternatively, the output from `/usr/bin/time` can be redirected to `STDOUT` +so `jc` can receive it. -Note: `/usr/bin/time` is similar but different from the Bash builtin `time` command. +Note: `/usr/bin/time` is similar but different from the Bash builtin + `time` command. Usage (cli): - $ /usr/bin/time -o timefile.out sleep 2.5; cat timefile.out | jc --time -p + $ /usr/bin/time -o timefile.out sleep 2; cat timefile.out | \ + jc --time -p Usage (module): + import jc + result = jc.parse('time', time_command_output) + + or + import jc.parsers.time result = jc.parsers.time.parse(time_command_output) @@ -39,8 +48,8 @@ Schema: "average_unshared_stack_size": integer, "average_shared_memory_size": integer, "maximum_resident_set_size": integer, - "block_input_operations": integer, # aka File system inputs - "block_output_operations": integer, # aka File system outputs + "block_input_operations": integer, # [0] + "block_output_operations": integer, # [1] "major_pagefaults": integer, "minor_pagefaults": integer, "swaps": integer, @@ -60,15 +69,19 @@ Schema: "exit_status": integer } + [0] aka File system inputs + [1] aka File system outputs + Examples: - $ /usr/bin/time --verbose -o timefile.out sleep 2.5; cat timefile.out | jc --time -p + $ /usr/bin/time --verbose -o timefile.out sleep 2; cat timefile.out | \ + jc --time -p { - "command_being_timed": "sleep 2.5", + "command_being_timed": "sleep 2", "user_time": 0.0, "system_time": 0.0, "cpu_percent": 0, - "elapsed_time": "0:02.50", + "elapsed_time": "0:02.00", "average_shared_text_size": 0, "average_unshared_data_size": 0, "average_stack_size": 0, @@ -94,13 +107,14 @@ Examples: "elapsed_time_total_seconds": 2.5 } - $ /usr/bin/time --verbose -o timefile.out sleep 2.5; cat timefile.out | jc --time -p -r + $ /usr/bin/time --verbose -o timefile.out sleep 2; cat timefile.out | \ + jc --time -p -r { - "command_being_timed": ""sleep 2.5"", + "command_being_timed": ""sleep 2"", "user_time": "0.00", "system_time": "0.00", "cpu_percent": "0", - "elapsed_time": "0:02.50", + "elapsed_time": "0:02.00", "average_shared_text_size": "0", "average_unshared_data_size": "0", "average_stack_size": "0", @@ -138,7 +152,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/timedatectl.md b/docs/parsers/timedatectl.md index e896bef5..2ee72708 100644 --- a/docs/parsers/timedatectl.md +++ b/docs/parsers/timedatectl.md @@ -3,7 +3,8 @@ # jc.parsers.timedatectl jc - JSON CLI output utility `timedatectl` command output parser -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the `universal_time` field is available. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the `universal_time` field is available. Usage (cli): @@ -15,6 +16,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('timedatectl', timedatectl_command_output) + + or + import jc.parsers.timedatectl result = jc.parsers.timedatectl.parse(timedatectl_command_output) @@ -23,7 +29,7 @@ Schema: { "local_time": string, "universal_time": string, - "epoch_utc": integer, # timezone-aware timestamp + "epoch_utc": integer, # timezone-aware "rtc_time": string, "time_zone": string, "ntp_enabled": boolean, @@ -78,7 +84,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/tracepath.md b/docs/parsers/tracepath.md index 3d621c8a..96297c54 100644 --- a/docs/parsers/tracepath.md +++ b/docs/parsers/tracepath.md @@ -15,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('tracepath', tracepath_command_output) + + or + import jc.parsers.tracepath result = jc.parsers.tracepath.parse(tracepath_command_output) @@ -146,7 +151,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/traceroute.md b/docs/parsers/traceroute.md index 5cedbc83..bf11c3c2 100644 --- a/docs/parsers/traceroute.md +++ b/docs/parsers/traceroute.md @@ -5,9 +5,12 @@ jc - JSON CLI output utility `traceroute` command output parser Supports `traceroute` and `traceroute6` output. -Note: On some operating systems you will need to redirect `STDERR` to `STDOUT` for destination info since the header line is sent to `STDERR`. A warning message will be printed to `STDERR` if the header row is not found. +Note: On some operating systems you will need to redirect `STDERR` to + `STDOUT` for destination info since the header line is sent to + `STDERR`. A warning message will be printed to `STDERR` if the + header row is not found. -e.g. `$ traceroute 8.8.8.8 2>&1 | jc --traceroute` + e.g. `$ traceroute 8.8.8.8 2>&1 | jc --traceroute` Usage (cli): @@ -19,6 +22,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('traceroute', traceroute_command_output) + + or + import jc.parsers.traceroute result = jc.parsers.traceroute.parse(traceroute_command_output) @@ -132,7 +140,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ufw.md b/docs/parsers/ufw.md index 1aee6e3d..2651b6b4 100644 --- a/docs/parsers/ufw.md +++ b/docs/parsers/ufw.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ufw', ufw_command_output) + + or + import jc.parsers.ufw result = jc.parsers.ufw.parse(ufw_command_output) @@ -43,7 +48,7 @@ Schema: "end": integer } ], - "to_service": string, # null if any to ports or port_ranges are set + "to_service": string, # [0] "from_ip": string, "from_ip_prefix": integer, "from_interface": string, @@ -57,12 +62,15 @@ Schema: "end": integer } ], - "from_service": string, # null if any from ports or port_ranges are set + "from_service": string, # [1] "comment": string # null if no comment } ] } + [0] null if any 'to' ports or port_ranges are set + [1] null if any 'from' ports or port_ranges are set + Examples: $ ufw status verbose | jc --ufw -p @@ -212,7 +220,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/ufw_appinfo.md b/docs/parsers/ufw_appinfo.md index 19250d97..85f46de4 100644 --- a/docs/parsers/ufw_appinfo.md +++ b/docs/parsers/ufw_appinfo.md @@ -1,11 +1,15 @@ [Home](https://kellyjonbrazil.github.io/jc/) # jc.parsers.ufw_appinfo -jc - JSON CLI output utility `ufw app info [application]` command output parser +jc - JSON CLI output utility `ufw app info [application]` command +output parser -Supports individual apps via `ufw app info [application]` and all apps list via `ufw app info all`. +Supports individual apps via `ufw app info [application]` and all apps list +via `ufw app info all`. -Because `ufw` application definitions allow overlapping ports and port ranges, this parser preserves that behavior, but also provides `normalized` lists and ranges that remove duplicate ports and merge overlapping ranges. +Because `ufw` application definitions allow overlapping ports and port +ranges, this parser preserves that behavior, but also provides `normalized` +lists and ranges that remove duplicate ports and merge overlapping ranges. Usage (cli): @@ -17,6 +21,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ufw_appinfo', ufw_appinfo_command_output) + + or + import jc.parsers.ufw_appinfo result = jc.parsers.ufw_appinfo.parse(ufw_appinfo_command_output) @@ -32,7 +41,7 @@ Schema: ], "tcp_ranges": [ { - "start": integer, # 'any' is converted to start/end: 0/65535 + "start": integer, # [0] "end": integer } ], @@ -41,31 +50,35 @@ Schema: ], "udp_ranges": [ { - "start": integer, # 'any' is converted to start/end: 0/65535 + "start": integer, # [0] "end": integer } ], "normalized_tcp_list": [ - integers # duplicates and overlapping are removed + integers # [1] ], "normalized_tcp_ranges": [ { - "start": integer, # 'any' is converted to start/end: 0/65535 - "end": integers # overlapping are merged + "start": integer, # [0] + "end": integers # [2] } ], "normalized_udp_list": [ - integers # duplicates and overlapping are removed + integers # [1] ], "normalized_udp_ranges": [ { - "start": integer, # 'any' is converted to start/end: 0/65535 - "end": integers # overlapping are merged + "start": integer, # [0] + "end": integers # [2] } ] } ] + [0] 'any' is converted to start/end: 0/65535 + [1] duplicates and overlapping are removed + [2] overlapping are merged + Examples: $ ufw app info MSN | jc --ufw-appinfo -p @@ -145,7 +158,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/uname.md b/docs/parsers/uname.md index 329711bc..f83c96ba 100644 --- a/docs/parsers/uname.md +++ b/docs/parsers/uname.md @@ -15,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('uname', uname_command_output) + + or + import jc.parsers.uname result = jc.parsers.uname.parse(uname_command_output) @@ -62,7 +67,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/universal.md b/docs/parsers/universal.md new file mode 100644 index 00000000..1b25f899 --- /dev/null +++ b/docs/parsers/universal.md @@ -0,0 +1,57 @@ + +# jc.parsers.universal +jc - JSON CLI output utility universal Parsers + +## simple_table_parse +```python +simple_table_parse(data) +``` + +Parse simple tables. The last column may contain data with spaces. + +Parameters: + + data: (list) Text data to parse that has been split into lines + via .splitlines(). Item 0 must be the header row. + Any spaces in header names should be changed to + underscore '_'. You should also ensure headers are + lowercase by using .lower(). + + Also, ensure there are no blank lines (list items) + in the data. + +Returns: + + List of Dictionaries + + +## sparse_table_parse +```python +sparse_table_parse(data, delim='\u2063') +``` + +Parse tables with missing column data or with spaces in column data. + +Parameters: + + data: (list) Text data to parse that has been split into lines + via .splitlines(). Item 0 must be the header row. + Any spaces in header names should be changed to + underscore '_'. You should also ensure headers are + lowercase by using .lower(). Do not change the + position of header names as the positions are used + to find the data. + + Also, ensure there are no blank lines (list items) + in the data. + + delim: (string) Delimiter to use. By default `u\2063` + (invisible separator) is used since it is unlikely + to ever be seen in terminal output. You can change + this for troubleshooting purposes or if there is a + delimiter conflict with your data. + +Returns: + + List of Dictionaries + diff --git a/docs/parsers/upower.md b/docs/parsers/upower.md index 2e4fc65f..5c1651f1 100644 --- a/docs/parsers/upower.md +++ b/docs/parsers/upower.md @@ -3,9 +3,11 @@ # jc.parsers.upower jc - JSON CLI output utility `upower` command output parser -The `updated_epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `updated_epoch` calculated timestamp field is naive. (i.e. based on the +local time of the system the parser is run on) -The `updated_epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `updated_epoch_utc` calculated timestamp field is timezone-aware and is +only available if the timezone field is UTC. Usage (cli): @@ -17,6 +19,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('upower', upower_command_output) + + or + import jc.parsers.upower result = jc.parsers.upower.parse(upower_command_output) @@ -29,8 +36,8 @@ Schema: "native_path": string, "power_supply": boolean, "updated": string, - "updated_epoch": integer, # null if date-time conversion fails - "updated_epoch_utc": integer, # null if date-time conversion fails + "updated_epoch": integer, # [0] + "updated_epoch_utc": integer, # [0] "updated_seconds_ago": integer, "has_history": boolean, "has_statistics": boolean, @@ -82,12 +89,14 @@ Schema: } ] + [0] null if date-time conversion fails + Examples: $ upower -i /org/freedesktop/UPower/devices/battery | jc --upower -p [ { - "native_path": "/sys/devices/LNXSYSTM:00/device:00/PNP0C0A:00/power_supply/BAT0", + "native_path": "/sys/devices/LNXSYSTM:00/device:00/PNP0C0A:00/p...", "vendor": "NOTEBOOK", "model": "BAT", "serial": "0001", @@ -146,7 +155,7 @@ Examples: $ upower -i /org/freedesktop/UPower/devices/battery | jc --upower -p -r [ { - "native_path": "/sys/devices/LNXSYSTM:00/device:00/PNP0C0A:00/power_supply/BAT0", + "native_path": "/sys/devices/LNXSYSTM:00/device:00/PNP0C0A:00/p...", "vendor": "NOTEBOOK", "model": "BAT", "serial": "0001", @@ -209,7 +218,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/uptime.md b/docs/parsers/uptime.md index e7cc2ed8..78d5e3e5 100644 --- a/docs/parsers/uptime.md +++ b/docs/parsers/uptime.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('uptime', uptime_command_output) + + or + import jc.parsers.uptime result = jc.parsers.uptime.parse(uptime_command_output) @@ -80,7 +85,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/vmstat.md b/docs/parsers/vmstat.md index fdf5792c..efa9a99c 100644 --- a/docs/parsers/vmstat.md +++ b/docs/parsers/vmstat.md @@ -5,9 +5,11 @@ jc - JSON CLI output utility `vmstat` command output parser Options supported: `-a`, `-w`, `-d`, `-t` -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the timezone field is UTC. Usage (cli): @@ -19,6 +21,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('vmstat', vmstat_command_output) + + or + import jc.parsers.vmstat result = jc.parsers.vmstat.parse(vmstat_command_output) @@ -58,11 +65,14 @@ Schema: "io_seconds": integer, "timestamp": string, "timezone": string, - "epoch": integer, # naive timestamp if -t flag is used - "epoch_utc": integer # aware timestamp if -t flag is used and UTC TZ + "epoch": integer, # [0] + "epoch_utc": integer # [1] } ] + [0] naive timestamp if -t flag is used + [1] aware timestamp if -t flag is used and UTC TZ + Examples: $ vmstat | jc --vmstat -p @@ -136,7 +146,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/vmstat_s.md b/docs/parsers/vmstat_s.md index e248f942..8c6532d7 100644 --- a/docs/parsers/vmstat_s.md +++ b/docs/parsers/vmstat_s.md @@ -7,76 +7,100 @@ jc - JSON CLI output utility `vmstat` command output streaming parser Options supported: `-a`, `-w`, `-d`, `-t` -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the timezone field is UTC. Usage (cli): $ vmstat | jc --vmstat-s -> Note: When piping `jc` converted `vmstat` output to other processes it may appear the output is hanging due to the OS pipe buffers. This is because `vmstat` output is too small to quickly fill up the buffer. Use the `-u` option to unbuffer the `jc` output if you would like immediate output. See the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) for more information. +> Note: When piping `jc` converted `vmstat` output to other processes it may +appear the output is hanging due to the OS pipe buffers. This is because +`vmstat` output is too small to quickly fill up the buffer. Use the `-u` +option to unbuffer the `jc` output if you would like immediate output. See +the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) +for more information. Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('vmstat_s', vmstat_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.vmstat_s - result = jc.parsers.vmstat_s.parse(vmstat_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.vmstat_s.parse(vmstat_command_output.splitlines()) for item in result: # do something Schema: { - "runnable_procs": integer, - "uninterruptible_sleeping_procs": integer, - "virtual_mem_used": integer, - "free_mem": integer, - "buffer_mem": integer, - "cache_mem": integer, - "inactive_mem": integer, - "active_mem": integer, - "swap_in": integer, - "swap_out": integer, - "blocks_in": integer, - "blocks_out": integer, - "interrupts": integer, - "context_switches": integer, - "user_time": integer, - "system_time": integer, - "idle_time": integer, - "io_wait_time": integer, - "stolen_time": integer, - "disk": string, - "total_reads": integer, - "merged_reads": integer, - "sectors_read": integer, - "reading_ms": integer, - "total_writes": integer, - "merged_writes": integer, - "sectors_written": integer, - "writing_ms": integer, - "current_io": integer, - "io_seconds": integer, - "timestamp": string, - "timezone": string, - "epoch": integer, # naive timestamp if -t flag is used - "epoch_utc": integer # aware timestamp if -t flag is used and UTC TZ - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + "runnable_procs": integer, + "uninterruptible_sleeping_procs": integer, + "virtual_mem_used": integer, + "free_mem": integer, + "buffer_mem": integer, + "cache_mem": integer, + "inactive_mem": integer, + "active_mem": integer, + "swap_in": integer, + "swap_out": integer, + "blocks_in": integer, + "blocks_out": integer, + "interrupts": integer, + "context_switches": integer, + "user_time": integer, + "system_time": integer, + "idle_time": integer, + "io_wait_time": integer, + "stolen_time": integer, + "disk": string, + "total_reads": integer, + "merged_reads": integer, + "sectors_read": integer, + "reading_ms": integer, + "total_writes": integer, + "merged_writes": integer, + "sectors_written": integer, + "writing_ms": integer, + "current_io": integer, + "io_seconds": integer, + "timestamp": string, + "timezone": string, + "epoch": integer, # [0] + "epoch_utc": integer # [1] + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error - "error": string, # exists if "success" is false - "line": string # exists if "success" is false + "success": boolean, # [2] + "error": string, # [3] + "line": string # [3] } } + [0] naive timestamp if -t flag is used + [1] aware timestamp if -t flag is used and UTC TZ + [2] false if error parsing + [3] exists if "success" is false + Examples: $ vmstat | jc --vmstat-s - {"runnable_procs":2,"uninterruptible_sleeping_procs":0,"virtual_mem_used":0,"free_mem":2794468,"buffer_mem":2108,"cache_mem":741208,"inactive_mem":null,"active_mem":null,"swap_in":0,"swap_out":0,"blocks_in":1,"blocks_out":3,"interrupts":29,"context_switches":57,"user_time":0,"system_time":0,"idle_time":99,"io_wait_time":0,"stolen_time":0,"timestamp":null,"timezone":null} + {"runnable_procs":2,"uninterruptible_sleeping_procs":0,"virtual_mem...} ... $ vmstat | jc --vmstat-s -r - {"runnable_procs":"2","uninterruptible_sleeping_procs":"0","virtual_mem_used":"0","free_mem":"2794468","buffer_mem":"2108","cache_mem":"741208","inactive_mem":null,"active_mem":null,"swap_in":"0","swap_out":"0","blocks_in":"1","blocks_out":"3","interrupts":"29","context_switches":"57","user_time":"0","system_time":"0","idle_time":"99","io_wait_time":"0","stolen_time":"0","timestamp":null,"timezone":null} + {"runnable_procs":"2","uninterruptible_sleeping_procs":"0","virtua...} ... @@ -95,8 +119,10 @@ Main text parsing generator function. Returns an iterator object. Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/docs/parsers/w.md b/docs/parsers/w.md index 630e942b..8f3a8b6b 100644 --- a/docs/parsers/w.md +++ b/docs/parsers/w.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('w', w_command_output) + + or + import jc.parsers.w result = jc.parsers.w.parse(w_command_output) @@ -118,7 +123,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/wc.md b/docs/parsers/wc.md index 1b5b5d38..b7251bb5 100644 --- a/docs/parsers/wc.md +++ b/docs/parsers/wc.md @@ -13,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('wc', wc_command_output) + + or + import jc.parsers.wc result = jc.parsers.wc.parse(wc_command_output) @@ -69,7 +74,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/who.md b/docs/parsers/who.md index fa954387..b8b6e211 100644 --- a/docs/parsers/who.md +++ b/docs/parsers/who.md @@ -5,7 +5,8 @@ jc - JSON CLI output utility `who` command output parser Accepts any of the following who options (or no options): `-aTH` -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) Usage (cli): @@ -17,6 +18,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('who', who_command_output) + + or + import jc.parsers.who result = jc.parsers.who.parse(who_command_output) @@ -29,7 +35,7 @@ Schema: "writeable_tty": string, "tty": string, "time": string, - "epoch": integer, # naive timestamp. null if time cannot be converted + "epoch": integer, # [0] "idle": string, "pid": integer, "from": string, @@ -37,6 +43,8 @@ Schema: } ] + [0] naive timestamp. null if time cannot be converted + Examples: $ who -a | jc --who -p @@ -147,7 +155,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/xml.md b/docs/parsers/xml.md index 2dddebc9..55f14bfa 100644 --- a/docs/parsers/xml.md +++ b/docs/parsers/xml.md @@ -9,6 +9,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('xml', xml_file_output) + + or + import jc.parsers.xml result = jc.parsers.xml.parse(xml_file_output) @@ -85,7 +90,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/yaml.md b/docs/parsers/yaml.md index a24deacb..9bb5fb20 100644 --- a/docs/parsers/yaml.md +++ b/docs/parsers/yaml.md @@ -9,6 +9,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('yaml', yaml_file_output) + + or + import jc.parsers.yaml result = jc.parsers.yaml.parse(yaml_file_output) @@ -99,7 +104,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/parsers/zipinfo.md b/docs/parsers/zipinfo.md index 525d00b5..1637a83b 100644 --- a/docs/parsers/zipinfo.md +++ b/docs/parsers/zipinfo.md @@ -18,6 +18,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('zipinfo', zipinfo_command_output) + + or + import jc.parsers.zipinfo result = jc.parsers.zipinfo.parse(zipinfo_command_output) @@ -94,7 +99,7 @@ Main text parsing function Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/docs/readme.md b/docs/readme.md index 0f5aa6b5..8a1485ac 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -4,72 +4,70 @@ JC - JSON CLI output utility * kellyjonbrazil@gmail.com -This package serializes the output of many standard unix command line tools to JSON format. +This package serializes the output of many standard unix command line tools +to dictionaries and lists of dictionaries. -For documentation on each parser, see the [documentation site](https://kellyjonbrazil.github.io/jc/). +## Interactive Documentation -CLI Example: + >>> help(jc) + >>> help(jc.lib) + >>> help(jc.util) + >>> jc.get_help('parser_module_name') - $ dig example.com | jc --dig -p - [ - { - "id": 2951, - "opcode": "QUERY", - "status": "NOERROR", - "flags": [ - "qr", - "rd", - "ra" - ], - "query_num": 1, - "answer_num": 1, - "authority_num": 0, - "additional_num": 1, - "opt_pseudosection": { - "edns": { - "version": 0, - "flags": [], - "udp": 4096 - } - }, - "question": { - "name": "example.com.", - "class": "IN", - "type": "A" - }, - "answer": [ - { - "name": "example.com.", - "class": "IN", - "type": "A", - "ttl": 39302, - "data": "93.184.216.34" - } - ], - "query_time": 49, - "server": "2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)", - "when": "Fri Apr 16 16:05:10 PDT 2021", - "rcvd": 56, - "when_epoch": 1618614310, - "when_epoch_utc": null - } - ] +## Online Documentation -Module Example: +### Latest: - >>> import jc.parsers.dig +https://github.com/kellyjonbrazil/jc/tree/master/docs + +### Specific Version + +Replace `{{full_version_number}}` - e.g. `1.17.7`: + +`https://github.com/kellyjonbrazil/jc/tree/v{{full_version_number}}/docs` + +Specific versions can also be selected by tag in the branch dropdown menu. + +## Usage Example: + + >>> import subprocess + >>> import jc >>> + >>> cmd_output = subprocess.check_output(['dig', 'example.com'], + text=True) + >>> data = jc.parse('dig', cmd_output) + >>> data + [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', ...}] + +Alternatively, you can bypass the high-level API and call the parser +modules directly: + >>> import subprocess >>> import jc.parsers.dig >>> - >>> cmd_output = subprocess.check_output(['dig', 'example.com'], text=True) + >>> cmd_output = subprocess.check_output(['dig', 'example.com'], + text=True) >>> data = jc.parsers.dig.parse(cmd_output) - >>> >>> data - [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', 'flags': ['qr', 'rd', 'ra'], 'query_num': 1, 'answer_num': - 1, 'authority_num': 0, 'additional_num': 1, 'opt_pseudosection': {'edns': {'version': 0, 'flags': [], 'udp': - 4096}}, 'question': {'name': 'example.com.', 'class': 'IN', 'type': 'A'}, 'answer': [{'name': 'example.com.', - 'class': 'IN', 'type': 'A', 'ttl': 29658, 'data': '93.184.216.34'}], 'query_time': 52, 'server': - '2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)', 'when': 'Fri Apr 16 16:13:00 PDT 2021', 'rcvd': 56, - 'when_epoch': 1618614780, 'when_epoch_utc': None}] + [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', ...}] + +## Available Functions + +Use `help(jc.lib)` for details: + + parse(parser_module_name: str, data: str | iterable) + High-level API to easily access the parser. This API will find both + built-in parsers and local plugin parsers. + + get_help(parser_module_name: str) + Convenience function to display the help screen for a parser using + its module name. + + parser_mod_list() + Get a list of all available parser module names to be used in + parse() and get_help(). + + plugin_parser_mod_list() + Get a list of plugin parser module names. This list is a subset of + parser_mod_list(). diff --git a/docs/utils.md b/docs/utils.md index 08938dcc..f79b5af2 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -7,9 +7,9 @@ jc - JSON CLI output utility utils warning_message(message_lines) ``` -Prints warning message for non-fatal issues. The first line is prepended with -'jc: Warning - ' and subsequent lines are indented. Wraps text as needed based -on the terminal width. +Prints warning message for non-fatal issues. The first line is +prepended with 'jc: Warning - ' and subsequent lines are indented. +Wraps text as needed based on the terminal width. Parameters: @@ -25,9 +25,9 @@ Returns: error_message(message_lines) ``` -Prints an error message for fatal issues. The first line is prepended with -'jc: Error - ' and subsequent lines are indented. Wraps text as needed based -on the terminal width. +Prints an error message for fatal issues. The first line is +prepended with 'jc: Error - ' and subsequent lines are indented. +Wraps text as needed based on the terminal width. Parameters: @@ -42,17 +42,19 @@ Returns: ```python compatibility(mod_name, compatible, quiet=False) ``` -Checks for the parser's compatibility with the running OS platform. + +Checks for the parser's compatibility with the running OS +platform. Parameters: - mod_name: (string) __name__ of the calling module + mod_name: (string) __name__ of the calling module - compatible: (list) sys.platform name(s) compatible with the parser - compatible options: - linux, darwin, cygwin, win32, aix, freebsd + compatible: (list) sys.platform name(s) compatible with + the parser. compatible options: + linux, darwin, cygwin, win32, aix, freebsd - quiet: (bool) supress compatibility message if True + quiet: (bool) supress compatibility message if True Returns: @@ -64,7 +66,8 @@ Returns: has_data(data) ``` -Checks if the input contains data. If there are any non-whitespace characters then return True, else return False +Checks if the input contains data. If there are any non-whitespace +characters then return True, else return False. Parameters: @@ -72,7 +75,8 @@ Parameters: Returns: - Boolean True if input string (data) contains non-whitespace characters, otherwise False + Boolean True if input string (data) contains non-whitespace + characters, otherwise False ## convert_to_int @@ -80,15 +84,16 @@ Returns: convert_to_int(value) ``` -Converts string and float input to int. Strips all non-numeric characters from strings. +Converts string and float input to int. Strips all non-numeric +characters from strings. Parameters: - value: (string/integer/float) Input value + value: (string/integer/float) Input value Returns: - integer/None Integer if successful conversion, otherwise None + integer/None Integer if successful conversion, otherwise None ## convert_to_float @@ -96,15 +101,16 @@ Returns: convert_to_float(value) ``` -Converts string and int input to float. Strips all non-numeric characters from strings. +Converts string and int input to float. Strips all non-numeric +characters from strings. Parameters: - value: (string) Input value + value: (string) Input value Returns: - float/None Float if successful conversion, otherwise None + float/None Float if successful conversion, otherwise None ## convert_to_bool @@ -112,7 +118,8 @@ Returns: convert_to_bool(value) ``` -Converts string, integer, or float input to boolean by checking for 'truthy' values +Converts string, integer, or float input to boolean by checking +for 'truthy' values. Parameters: @@ -120,7 +127,8 @@ Parameters: Returns: - True/False False unless a 'truthy' number or string is found ('y', 'yes', 'true', '1', 1, -1, etc.) + True/False False unless a 'truthy' number or string is found + ('y', 'yes', 'true', '1', 1, -1, etc.) ## stream_success @@ -133,8 +141,9 @@ Add `_jc_meta` object to output line if `ignore_exceptions=True` ```python stream_error(e, ignore_exceptions, line) ``` -Reraise the stream exception with annotation or print an error `_jc_meta` -field if `ignore_exceptions=True` + +Reraise the stream exception with annotation or print an error +`_jc_meta` field if `ignore_exceptions=True`. ## input_type_check @@ -160,16 +169,26 @@ Ensure each line is a string timestamp(datetime_string) ``` -Input a date-time text string of several formats and convert to a naive or timezone-aware epoch timestamp in UTC +Input a date-time text string of several formats and convert to a +naive or timezone-aware epoch timestamp in UTC. Parameters: - datetime_string: (str) a string representation of a date-time in several supported formats + datetime_string: (str) a string representation of a + date-time in several supported formats Attributes: - string (str) the input datetime string - format (int) the format rule that was used to decode the datetime string. None if conversion fails - naive (int) timestamp based on locally configured timezone. None if conversion fails - utc (int) aware timestamp only if UTC timezone detected in datetime string. None if conversion fails + string (str) the input datetime string + + format (int) the format rule that was used to + decode the datetime string. None if + conversion fails + + naive (int) timestamp based on locally configured + timezone. None if conversion fails + + utc (int) aware timestamp only if UTC timezone + detected in datetime string. None if + conversion fails diff --git a/jc/__init__.py b/jc/__init__.py index ba4ae139..40fb0817 100644 --- a/jc/__init__.py +++ b/jc/__init__.py @@ -2,75 +2,72 @@ * kellyjonbrazil@gmail.com -This package serializes the output of many standard unix command line tools to JSON format. +This package serializes the output of many standard unix command line tools +to dictionaries and lists of dictionaries. -For documentation on each parser, see the [documentation site](https://kellyjonbrazil.github.io/jc/). +## Interactive Documentation -CLI Example: + >>> help(jc) + >>> help(jc.lib) + >>> help(jc.util) + >>> jc.get_help('parser_module_name') - $ dig example.com | jc --dig -p - [ - { - "id": 2951, - "opcode": "QUERY", - "status": "NOERROR", - "flags": [ - "qr", - "rd", - "ra" - ], - "query_num": 1, - "answer_num": 1, - "authority_num": 0, - "additional_num": 1, - "opt_pseudosection": { - "edns": { - "version": 0, - "flags": [], - "udp": 4096 - } - }, - "question": { - "name": "example.com.", - "class": "IN", - "type": "A" - }, - "answer": [ - { - "name": "example.com.", - "class": "IN", - "type": "A", - "ttl": 39302, - "data": "93.184.216.34" - } - ], - "query_time": 49, - "server": "2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)", - "when": "Fri Apr 16 16:05:10 PDT 2021", - "rcvd": 56, - "when_epoch": 1618614310, - "when_epoch_utc": null - } - ] +## Online Documentation -Module Example: +### Latest: - >>> import jc.parsers.dig +https://github.com/kellyjonbrazil/jc/tree/master/docs + +### Specific Version + +Replace `{{full_version_number}}` - e.g. `1.17.7`: + +`https://github.com/kellyjonbrazil/jc/tree/v{{full_version_number}}/docs` + +Specific versions can also be selected by tag in the branch dropdown menu. + +## Usage Example: + + >>> import subprocess + >>> import jc >>> + >>> cmd_output = subprocess.check_output(['dig', 'example.com'], + text=True) + >>> data = jc.parse('dig', cmd_output) + >>> data + [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', ...}] + +Alternatively, you can bypass the high-level API and call the parser +modules directly: + >>> import subprocess >>> import jc.parsers.dig >>> - >>> cmd_output = subprocess.check_output(['dig', 'example.com'], text=True) + >>> cmd_output = subprocess.check_output(['dig', 'example.com'], + text=True) >>> data = jc.parsers.dig.parse(cmd_output) - >>> >>> data - [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', 'flags': ['qr', 'rd', 'ra'], 'query_num': 1, 'answer_num': - 1, 'authority_num': 0, 'additional_num': 1, 'opt_pseudosection': {'edns': {'version': 0, 'flags': [], 'udp': - 4096}}, 'question': {'name': 'example.com.', 'class': 'IN', 'type': 'A'}, 'answer': [{'name': 'example.com.', - 'class': 'IN', 'type': 'A', 'ttl': 29658, 'data': '93.184.216.34'}], 'query_time': 52, 'server': - '2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)', 'when': 'Fri Apr 16 16:13:00 PDT 2021', 'rcvd': 56, - 'when_epoch': 1618614780, 'when_epoch_utc': None}] -""" + [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', ...}] -name = 'jc' -__version__ = '1.17.7' +## Available Functions + +Use `help(jc.lib)` for details: + + parse(parser_module_name: str, data: str | iterable) + High-level API to easily access the parser. This API will find both + built-in parsers and local plugin parsers. + + get_help(parser_module_name: str) + Convenience function to display the help screen for a parser using + its module name. + + parser_mod_list() + Get a list of all available parser module names to be used in + parse() and get_help(). + + plugin_parser_mod_list() + Get a list of plugin parser module names. This list is a subset of + parser_mod_list(). +""" +from .lib import (__version__, parse, parser_mod_list, + plugin_parser_mod_list, get_help) diff --git a/jc/cli.py b/jc/cli.py index 750f7155..9798f7a8 100644 --- a/jc/cli.py +++ b/jc/cli.py @@ -4,19 +4,16 @@ JC cli module import sys import os -import os.path -import re import importlib import textwrap import signal import shlex import subprocess import json -import jc -from jc import appdirs -import jc.utils -import jc.tracebackplus -from jc.exceptions import LibraryNotInstalled, ParseError +from .lib import __version__, parsers, local_parsers +from . import utils +from . import tracebackplus +from .exceptions import LibraryNotInstalled, ParseError # make pygments import optional try: @@ -31,8 +28,11 @@ except Exception: PYGMENTS_INSTALLED = False +JC_ERROR_EXIT = 100 + + class info(): - version = jc.__version__ + version = __version__ description = 'JSON CLI output utility' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' @@ -41,114 +41,6 @@ class info(): license = 'MIT License' -__version__ = info.version - -parsers = [ - 'acpi', - 'airport', - 'airport-s', - 'arp', - 'blkid', - 'cksum', - 'crontab', - 'crontab-u', - 'csv', - 'csv-s', - 'date', - 'df', - 'dig', - 'dir', - 'dmidecode', - 'dpkg-l', - 'du', - 'env', - 'file', - 'finger', - 'free', - 'fstab', - 'group', - 'gshadow', - 'hash', - 'hashsum', - 'hciconfig', - 'history', - 'hosts', - 'id', - 'ifconfig', - 'ini', - 'iostat', - 'iostat-s', - 'iptables', - 'iw-scan', - 'jar-manifest', - 'jobs', - 'kv', - 'last', - 'ls', - 'ls-s', - 'lsblk', - 'lsmod', - 'lsof', - 'lsusb', - 'mount', - 'netstat', - 'ntpq', - 'passwd', - 'ping', - 'ping-s', - 'pip-list', - 'pip-show', - 'ps', - 'route', - 'rpm-qi', - 'sfdisk', - 'shadow', - 'ss', - 'stat', - 'stat-s', - 'sysctl', - 'systemctl', - 'systemctl-lj', - 'systemctl-ls', - 'systemctl-luf', - 'systeminfo', - 'time', - 'timedatectl', - 'tracepath', - 'traceroute', - 'ufw', - 'ufw-appinfo', - 'uname', - 'upower', - 'uptime', - 'vmstat', - 'vmstat-s', - 'w', - 'wc', - 'who', - 'xml', - 'yaml', - 'zipinfo' -] - -JC_ERROR_EXIT = 100 - - -# List of custom or override parsers. -# Allow any /jc/jcparsers/*.py -local_parsers = [] -data_dir = appdirs.user_data_dir('jc', 'jc') -local_parsers_dir = os.path.join(data_dir, 'jcparsers') -if os.path.isdir(local_parsers_dir): - sys.path.append(data_dir) - for name in os.listdir(local_parsers_dir): - if re.match(r'\w+\.py$', name) and os.path.isfile(os.path.join(local_parsers_dir, name)): - plugin_name = name[0:-3] - local_parsers.append(plugin_name) - if plugin_name not in parsers: - parsers.append(plugin_name) - - # We only support 2.3.0+, pygments changed color names in 2.4.0. # startswith is sufficient and avoids potential exceptions from split and int. if PYGMENTS_INSTALLED: @@ -226,7 +118,7 @@ def set_env_colors(env_colors=None): # if there is an issue with the env variable, just set all colors to default and move on if input_error: - jc.utils.warning_message(['Could not parse JC_COLORS environment variable']) + utils.warning_message(['Could not parse JC_COLORS environment variable']) color_list = ['default', 'default', 'default', 'default'] # Try the color set in the JC_COLORS env variable first. If it is set to default, then fall back to default colors @@ -543,7 +435,7 @@ def main(): version_info = 'v' in options if verbose_debug: - jc.tracebackplus.enable(context=11) + tracebackplus.enable(context=11) if not PYGMENTS_INSTALLED: mono = True @@ -578,29 +470,23 @@ def main(): if magic_stderr: print(magic_stderr[:-1], file=sys.stderr) - except FileNotFoundError: + except OSError as e: if debug: raise - jc.utils.error_message([f'"{run_command_str}" command could not be found. For details use the -d or -dd option.']) - sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) - - except OSError: - if debug: - raise - - jc.utils.error_message([f'"{run_command_str}" command could not be run due to too many open files. For details use the -d or -dd option.']) + error_msg = os.strerror(e.errno) + utils.error_message([f'"{run_command_str}" command could not be run: {error_msg}. For details use the -d or -dd option.']) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) except Exception: if debug: raise - jc.utils.error_message([f'"{run_command_str}" command could not be run. For details use the -d or -dd option.']) + utils.error_message([f'"{run_command_str}" command could not be run. For details use the -d or -dd option.']) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) elif run_command is not None: - jc.utils.error_message([f'"{run_command_str}" cannot be used with Magic syntax. Use "jc -h" for help.']) + utils.error_message([f'"{run_command_str}" cannot be used with Magic syntax. Use "jc -h" for help.']) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) # find the correct parser @@ -619,16 +505,16 @@ def main(): break if not found: - jc.utils.error_message(['Missing or incorrect arguments. Use "jc -h" for help.']) + utils.error_message(['Missing or incorrect arguments. Use "jc -h" for help.']) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) # check for input errors (pipe vs magic) if not sys.stdin.isatty() and magic_stdout: - jc.utils.error_message(['Piped data and Magic syntax used simultaneously. Use "jc -h" for help.']) + utils.error_message(['Piped data and Magic syntax used simultaneously. Use "jc -h" for help.']) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) elif sys.stdin.isatty() and magic_stdout is None: - jc.utils.error_message(['Missing piped data. Use "jc -h" for help.']) + utils.error_message(['Missing piped data. Use "jc -h" for help.']) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) # parse and print to stdout @@ -665,17 +551,17 @@ def main(): if debug: raise - jc.utils.error_message([f'Parser issue with {parser_name}:', - f'{e.__class__.__name__}: {e}', - 'For details use the -d or -dd option. Use "jc -h" for help.']) + utils.error_message([f'Parser issue with {parser_name}:', + f'{e.__class__.__name__}: {e}', + 'For details use the -d or -dd option. Use "jc -h" for help.']) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) except json.JSONDecodeError: if debug: raise - jc.utils.error_message(['There was an issue generating the JSON output.', - 'For details use the -d or -dd option.']) + utils.error_message(['There was an issue generating the JSON output.', + 'For details use the -d or -dd option.']) sys.exit(combined_exit_code(magic_exit_code, JC_ERROR_EXIT)) except Exception: @@ -686,7 +572,7 @@ def main(): if getattr(parser.info, 'streaming', None): streaming_msg = 'Use the -qq option to ignore streaming parser errors.' - jc.utils.error_message([ + utils.error_message([ f'{parser_name} parser could not parse the input data. Did you use the correct parser?', f'{streaming_msg}', 'For details use the -d or -dd option. Use "jc -h" for help.' diff --git a/jc/lib.py b/jc/lib.py new file mode 100644 index 00000000..33b87c85 --- /dev/null +++ b/jc/lib.py @@ -0,0 +1,209 @@ +"""jc - JSON CLI output utility +JC lib module +""" + +import sys +import os +import re +import importlib +from jc import appdirs + +__version__ = '1.18.0' + +parsers = [ + 'acpi', + 'airport', + 'airport-s', + 'arp', + 'blkid', + 'cksum', + 'crontab', + 'crontab-u', + 'csv', + 'csv-s', + 'date', + 'df', + 'dig', + 'dir', + 'dmidecode', + 'dpkg-l', + 'du', + 'env', + 'file', + 'finger', + 'free', + 'fstab', + 'group', + 'gshadow', + 'hash', + 'hashsum', + 'hciconfig', + 'history', + 'hosts', + 'id', + 'ifconfig', + 'ini', + 'iostat', + 'iostat-s', + 'iptables', + 'iw-scan', + 'jar-manifest', + 'jobs', + 'kv', + 'last', + 'ls', + 'ls-s', + 'lsblk', + 'lsmod', + 'lsof', + 'lsusb', + 'mount', + 'netstat', + 'ntpq', + 'passwd', + 'ping', + 'ping-s', + 'pip-list', + 'pip-show', + 'ps', + 'route', + 'rpm-qi', + 'sfdisk', + 'shadow', + 'ss', + 'stat', + 'stat-s', + 'sysctl', + 'systemctl', + 'systemctl-lj', + 'systemctl-ls', + 'systemctl-luf', + 'systeminfo', + 'time', + 'timedatectl', + 'tracepath', + 'traceroute', + 'ufw', + 'ufw-appinfo', + 'uname', + 'upower', + 'uptime', + 'vmstat', + 'vmstat-s', + 'w', + 'wc', + 'who', + 'xml', + 'yaml', + 'zipinfo' +] + +# Create the local_parsers list. This is a list of custom or +# override parsers from /jc/jcparsers/*.py. +# Once this list is created, extend the parsers list with it. +local_parsers = [] +data_dir = appdirs.user_data_dir('jc', 'jc') +local_parsers_dir = os.path.join(data_dir, 'jcparsers') +if os.path.isdir(local_parsers_dir): + sys.path.append(data_dir) + for name in os.listdir(local_parsers_dir): + if re.match(r'\w+\.py$', name) and os.path.isfile(os.path.join(local_parsers_dir, name)): + plugin_name = name[0:-3] + local_parsers.append(plugin_name) + if plugin_name not in parsers: + parsers.append(plugin_name) + del name + + +def _cliname_to_modname(parser_cli_name): + """Return real module name (dashes converted to underscores)""" + return parser_cli_name.replace('-', '_') + +def _modname_to_cliname(parser_mod_name): + """Return module's cli name (underscores converted to dashes)""" + return parser_mod_name.replace('_', '-') + +def _get_parser(parser_mod_name): + """Return the parser module object""" + parser_cli_name = _modname_to_cliname(parser_mod_name) + modpath = 'jcparsers.' if parser_cli_name in local_parsers else 'jc.parsers.' + return importlib.import_module(f'{modpath}{parser_mod_name}') + +def parse(parser_mod_name, data, + quiet=False, raw=False, ignore_exceptions=None, **kwargs): + """ + Parse the string data using the supplied parser module. + + This function provides a high-level API to simplify parser use. This + function will call built-in parsers and custom plugin parsers. + + Example: + + >>> import jc + >>> jc.parse('date', 'Tue Jan 18 10:23:07 PST 2022') + {'year': 2022, 'month': 'Jan', 'month_num': 1, 'day'...} + + To get a list of available parser module names, use `parser_mod_list()` + or `plugin_parser_mod_list()`. `plugin_parser_mod_list()` is a subset + of `parser_mod_list()`. + + You can also use the lower-level parser modules directly: + + >>> import jc.parsers.date + >>> jc.parsers.date.parse('Tue Jan 18 10:23:07 PST 2022') + + Though, accessing plugin parsers directly is a bit more cumbersome, so + this higher-level API is recommended. Here is how you can access plugin + parsers without this API: + + >>> import os + >>> import sys + >>> import jc.appdirs + >>> data_dir = jc.appdirs.user_data_dir('jc', 'jc') + >>> local_parsers_dir = os.path.join(data_dir, 'jcparsers') + >>> sys.path.append(local_parsers_dir) + >>> import my_custom_parser + >>> my_custom_parser.parse('command_data') + + Parameters: + + parser_mod_name: (string) name of the parser module + + data: (string or data to parse (string for normal + iterator) parsers, iterator of strings for + streaming parsers) + + raw: (boolean) output preprocessed JSON if True + + quiet: (boolean) suppress warning messages if True + + ignore_exceptions: (boolean) ignore parsing exceptions if True + (streaming parsers only) + + Returns: + + Standard Parsers: Dictionary or List of Dictionaries + Streaming Parsers: Generator Object containing Dictionaries + """ + jc_parser = _get_parser(parser_mod_name) + + if ignore_exceptions is not None: + return jc_parser.parse(data, quiet=quiet, raw=raw, + ignore_exceptions=ignore_exceptions, **kwargs) + + return jc_parser.parse(data, quiet=quiet, raw=raw, **kwargs) + +def parser_mod_list(): + """Returns a list of all available parser module names.""" + return [_cliname_to_modname(p) for p in parsers] + +def plugin_parser_mod_list(): + """ + Returns a list of plugin parser module names. This function is a + subset of `parser_mod_list()`. + """ + return [_cliname_to_modname(p) for p in local_parsers] + +def get_help(parser_mod_name): + """Show help screen for the selected parser.""" + help(_get_parser(parser_mod_name)) diff --git a/jc/parsers/acpi.py b/jc/parsers/acpi.py index 24225a55..4e9e41fe 100644 --- a/jc/parsers/acpi.py +++ b/jc/parsers/acpi.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('acpi', acpi_command_output) + + or + import jc.parsers.acpi result = jc.parsers.acpi.parse(acpi_command_output) @@ -231,8 +236,6 @@ class info(): description = '`acpi` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['acpi'] @@ -252,7 +255,8 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ - int_list = ['id', 'charge_percent', 'design_capacity_mah', 'last_full_capacity', 'last_full_capacity_percent'] + int_list = ['id', 'charge_percent', 'design_capacity_mah', 'last_full_capacity', + 'last_full_capacity_percent'] float_list = ['temperature'] for entry in proc_data: @@ -295,7 +299,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/airport.py b/jc/parsers/airport.py index 174ae851..715510f2 100644 --- a/jc/parsers/airport.py +++ b/jc/parsers/airport.py @@ -12,6 +12,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('airport', airport_command_output) + + or + import jc.parsers.airport result = jc.parsers.airport.parse(airport_command_output) @@ -84,9 +89,6 @@ class info(): description = '`airport -I` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['darwin'] magic_commands = ['airport -I'] @@ -123,7 +125,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: @@ -139,7 +141,9 @@ def parse(data, raw=False, quiet=False): for line in filter(None, data.splitlines()): linedata = line.split(':', maxsplit=1) - raw_output[linedata[0].strip().lower().replace(' ', '_').replace('.', '_')] = linedata[1].strip() + key = linedata[0].strip().lower().replace(' ', '_').replace('.', '_') + value = linedata[1].strip() + raw_output[key] = value if raw: return raw_output diff --git a/jc/parsers/airport_s.py b/jc/parsers/airport_s.py index cbf5203c..5648195c 100644 --- a/jc/parsers/airport_s.py +++ b/jc/parsers/airport_s.py @@ -12,6 +12,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('airport_s', airport_s_command_output) + + or + import jc.parsers.airport_s result = jc.parsers.airport_s.parse(airport_s_command_output) @@ -113,9 +118,6 @@ class info(): description = '`airport -s` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['darwin'] magic_commands = ['airport -s'] @@ -159,7 +161,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/arp.py b/jc/parsers/arp.py index 664ec6e0..2f4e8a58 100644 --- a/jc/parsers/arp.py +++ b/jc/parsers/arp.py @@ -12,6 +12,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('arp', arp_command_output) + + or + import jc.parsers.arp result = jc.parsers.arp.parse(arp_command_output) @@ -122,8 +127,6 @@ class info(): description = '`arp` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'aix', 'freebsd', 'darwin'] magic_commands = ['arp'] @@ -164,7 +167,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/blkid.py b/jc/parsers/blkid.py index 0a4af511..ce5df6be 100644 --- a/jc/parsers/blkid.py +++ b/jc/parsers/blkid.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('blkid', blkid_command_output) + + or + import jc.parsers.blkid result = jc.parsers.blkid.parse(blkid_command_output) @@ -125,9 +130,6 @@ class info(): description = '`blkid` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['blkid'] @@ -151,9 +153,10 @@ def _process(proc_data): if 'devname' in entry: entry['device'] = entry.pop('devname') - int_list = ['part_entry_number', 'part_entry_offset', 'part_entry_size', 'id_part_entry_number', - 'id_part_entry_offset', 'id_part_entry_size', 'minimum_io_size', 'physical_sector_size', - 'logical_sector_size', 'id_iolimit_minimum_io_size', 'id_iolimit_physical_sector_size', + int_list = ['part_entry_number', 'part_entry_offset', 'part_entry_size', + 'id_part_entry_number', 'id_part_entry_offset', 'id_part_entry_size', + 'minimum_io_size', 'physical_sector_size', 'logical_sector_size', + 'id_iolimit_minimum_io_size', 'id_iolimit_physical_sector_size', 'id_iolimit_logical_sector_size'] for key in entry: if key in int_list: @@ -169,7 +172,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/cksum.py b/jc/parsers/cksum.py index d1004fdc..02b812f6 100644 --- a/jc/parsers/cksum.py +++ b/jc/parsers/cksum.py @@ -14,6 +14,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('cksum', cksum_command_output) + + or + import jc.parsers.cksum result = jc.parsers.cksum.parse(cksum_command_output) @@ -58,8 +63,6 @@ class info(): description = '`cksum` and `sum` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] magic_commands = ['cksum', 'sum'] @@ -96,7 +99,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/crontab.py b/jc/parsers/crontab.py index c2a52397..1736671e 100644 --- a/jc/parsers/crontab.py +++ b/jc/parsers/crontab.py @@ -1,4 +1,5 @@ -"""jc - JSON CLI output utility `crontab -l` command output and crontab file parser +"""jc - JSON CLI output utility `crontab -l` command output and crontab +file parser Supports `crontab -l` command output and crontab files. @@ -12,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('crontab', crontab_output) + + or + import jc.parsers.crontab result = jc.parsers.crontab.parse(crontab_output) @@ -177,9 +183,6 @@ class info(): description = '`crontab` command and file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'aix', 'freebsd'] magic_commands = ['crontab'] @@ -220,7 +223,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/crontab_u.py b/jc/parsers/crontab_u.py index 93c0b6af..0cd10f88 100644 --- a/jc/parsers/crontab_u.py +++ b/jc/parsers/crontab_u.py @@ -1,6 +1,8 @@ -"""jc - JSON CLI output utility `crontab -l` command output and crontab file parser +"""jc - JSON CLI output utility `crontab -l` command output and crontab +file parser -This version of the `crontab -l` parser supports output that contains user information for processes. +This version of the `crontab -l` parser supports output that contains user +information for processes. Usage (cli): @@ -8,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('crontab_u', crontab_u_output) + + or + import jc.parsers.crontab_u result = jc.parsers.crontab_u.parse(crontab_u_output) @@ -52,7 +59,7 @@ Examples: "variables": [ { "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" + "value": "/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sb..." }, { "name": "SHELL", @@ -77,7 +84,7 @@ Examples: "*" ], "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.daily )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." }, { "minute": [ @@ -96,7 +103,7 @@ Examples: "7" ], "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.weekly )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." }, { "minute": [ @@ -115,7 +122,7 @@ Examples: "*" ], "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.monthly )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." } ] } @@ -125,7 +132,7 @@ Examples: "variables": [ { "name": "PATH", - "value": "/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" + "value": "/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/..." }, { "name": "SHELL", @@ -140,7 +147,7 @@ Examples: "month": "*", "day_of_week": "*", "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.daily )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." }, { "minute": "47", @@ -149,7 +156,7 @@ Examples: "month": "*", "day_of_week": "7", "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.weekly )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." }, { "minute": "52", @@ -158,7 +165,7 @@ Examples: "month": "*", "day_of_week": "*", "user": "root", - "command": "test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.monthly )" + "command": "test -x /usr/sbin/anacron || ( cd / && run-parts ..." } ] } @@ -173,9 +180,6 @@ class info(): description = '`crontab` file parser with user support' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'aix', 'freebsd'] @@ -215,7 +219,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/csv.py b/jc/parsers/csv.py index 539d961b..2910c832 100644 --- a/jc/parsers/csv.py +++ b/jc/parsers/csv.py @@ -1,6 +1,8 @@ """jc - JSON CLI output utility `csv` file parser -The `csv` parser will attempt to automatically detect the delimiter character. If the delimiter cannot be detected it will default to comma. The first row of the file must be a header row. +The `csv` parser will attempt to automatically detect the delimiter +character. If the delimiter cannot be detected it will default to comma. +The first row of the file must be a header row. Usage (cli): @@ -8,12 +10,18 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('csv', csv_output) + + or + import jc.parsers.csv result = jc.parsers.csv.parse(csv_output) Schema: - csv file converted to a Dictionary: https://docs.python.org/3/library/csv.html + csv file converted to a Dictionary: + https://docs.python.org/3/library/csv.html [ { @@ -25,7 +33,7 @@ Schema: Examples: $ cat homes.csv - "Sell", "List", "Living", "Rooms", "Beds", "Baths", "Age", "Acres", "Taxes" + "Sell", "List", "Living", "Rooms", "Beds", "Baths", "Age", "Acres"... 142, 160, 28, 10, 5, 3, 60, 0.28, 3167 175, 180, 18, 8, 4, 1, 12, 0.43, 4033 129, 132, 13, 6, 3, 1, 41, 0.33, 1471 @@ -80,8 +88,6 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Using the python standard csv library' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] @@ -98,7 +104,8 @@ def _process(proc_data): Returns: - List of Dictionaries. Each Dictionary represents a row in the csv file. + List of Dictionaries. Each Dictionary represents a row in the csv + file. """ # No further processing @@ -112,7 +119,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/csv_s.py b/jc/parsers/csv_s.py index f152bdc7..c692dde8 100644 --- a/jc/parsers/csv_s.py +++ b/jc/parsers/csv_s.py @@ -2,9 +2,12 @@ > This streaming parser outputs JSON Lines -The `csv` streaming parser will attempt to automatically detect the delimiter character. If the delimiter cannot be detected it will default to comma. The first row of the file must be a header row. +The `csv` streaming parser will attempt to automatically detect the +delimiter character. If the delimiter cannot be detected it will default +to comma. The first row of the file must be a header row. -Note: The first 100 rows are read into memory to enable delimiter detection, then the rest of the rows are loaded lazily. +Note: The first 100 rows are read into memory to enable delimiter detection, +then the rest of the rows are loaded lazily. Usage (cli): @@ -12,21 +15,34 @@ Usage (cli): Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('csv_s', csv_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.csv_s - result = jc.parsers.csv_s.parse(csv_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.csv_s.parse(csv_output.splitlines()) for item in result: # do something Schema: - csv file converted to a Dictionary: https://docs.python.org/3/library/csv.html + csv file converted to a Dictionary: + https://docs.python.org/3/library/csv.html { "column_name1": string, "column_name2": string, - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + + # below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error + "success": boolean, # false if error parsing "error": string, # exists if "success" is false "line": string # exists if "success" is false } @@ -35,16 +51,16 @@ Schema: Examples: $ cat homes.csv - "Sell", "List", "Living", "Rooms", "Beds", "Baths", "Age", "Acres", "Taxes" + "Sell", "List", "Living", "Rooms", "Beds", "Baths", "Age", "Acres"... 142, 160, 28, 10, 5, 3, 60, 0.28, 3167 175, 180, 18, 8, 4, 1, 12, 0.43, 4033 129, 132, 13, 6, 3, 1, 41, 0.33, 1471 ... $ cat homes.csv | jc --csv-s - {"Sell":"142","List":"160","Living":"28","Rooms":"10","Beds":"5","Baths":"3","Age":"60","Acres":"0.28","Taxes":"3167"} - {"Sell":"175","List":"180","Living":"18","Rooms":"8","Beds":"4","Baths":"1","Age":"12","Acres":"0.43","Taxes":"4033"} - {"Sell":"129","List":"132","Living":"13","Rooms":"6","Beds":"3","Baths":"1","Age":"41","Acres":"0.33","Taxes":"1471"} + {"Sell":"142","List":"160","Living":"28","Rooms":"10","Beds":"5"...} + {"Sell":"175","List":"180","Living":"18","Rooms":"8","Beds":"4"...} + {"Sell":"129","List":"132","Living":"13","Rooms":"6","Beds":"3"...} ... """ import itertools @@ -78,7 +94,8 @@ def _process(proc_data): Returns: - List of Dictionaries. Each Dictionary represents a row in the csv file. + List of Dictionaries. Each Dictionary represents a row in the csv + file. """ # No further processing return proc_data @@ -90,8 +107,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/jc/parsers/date.py b/jc/parsers/date.py index 1e6671c2..d09eed5d 100644 --- a/jc/parsers/date.py +++ b/jc/parsers/date.py @@ -1,8 +1,10 @@ """jc - JSON CLI output utility `date` command output parser -The `epoch` calculated timestamp field is naive. (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the timezone field is UTC. Usage (cli): @@ -14,6 +16,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('date', date_command_output) + + or + import jc.parsers.date result = jc.parsers.date.parse(date_command_output) @@ -32,15 +39,19 @@ Schema: "second": integer, "period": string, "timezone": string, - "utc_offset": string, # null if timezone field is not UTC + "utc_offset": string, # null if timezone field is not UTC "day_of_year": integer, "week_of_year": integer, "iso": string, - "epoch": integer, # naive timestamp - "epoch_utc": integer, # timezone-aware timestamp. Only available if timezone field is UTC - "timezone_aware": boolean # if true, all fields are correctly based on UTC + "epoch": integer, # [0] + "epoch_utc": integer, # [1] + "timezone_aware": boolean # [2] } + [0] naive timestamp + [1] timezone-aware timestamp. Only available if timezone field is UTC + [2] if true, all fields are correctly based on UTC + Examples: $ date | jc --date -p @@ -76,8 +87,6 @@ class info(): description = '`date` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['date'] @@ -108,7 +117,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: @@ -124,28 +133,33 @@ def parse(data, raw=False, quiet=False): # find the timezone no matter where it is in the string # from https://www.timeanddate.com/time/zones/ - tz_abbr = ['A', 'ACDT', 'ACST', 'ACT', 'ACWST', 'ADT', 'AEDT', 'AEST', 'AET', 'AFT', 'AKDT', 'AKST', 'ALMT', - 'AMST', 'AMT', 'ANAST', 'ANAT', 'AQTT', 'ART', 'AST', 'AT', 'AWDT', 'AWST', 'AZOST', 'AZOT', - 'AZST', 'AZT', 'AoE', 'B', 'BNT', 'BOT', 'BRST', 'BRT', 'BST', 'BTT', 'C', 'CAST', 'CAT', 'CCT', - 'CDT', 'CEST', 'CET', 'CHADT', 'CHAST', 'CHOST', 'CHOT', 'CHUT', 'CIDST', 'CIST', 'CKT', 'CLST', - 'CLT', 'COT', 'CST', 'CT', 'CVT', 'CXT', 'ChST', 'D', 'DAVT', 'DDUT', 'E', 'EASST', 'EAST', - 'EAT', 'ECT', 'EDT', 'EEST', 'EET', 'EGST', 'EGT', 'EST', 'ET', 'F', 'FET', 'FJST', 'FJT', 'FKST', - 'FKT', 'FNT', 'G', 'GALT', 'GAMT', 'GET', 'GFT', 'GILT', 'GMT', 'GST', 'GYT', 'H', 'HDT', 'HKT', - 'HOVST', 'HOVT', 'HST', 'I', 'ICT', 'IDT', 'IOT', 'IRDT', 'IRKST', 'IRKT', 'IRST', 'IST', 'JST', - 'K', 'KGT', 'KOST', 'KRAST', 'KRAT', 'KST', 'KUYT', 'L', 'LHDT', 'LHST', 'LINT', 'M', 'MAGST', - 'MAGT', 'MART', 'MAWT', 'MDT', 'MHT', 'MMT', 'MSD', 'MSK', 'MST', 'MT', 'MUT', 'MVT', 'MYT', 'N', - 'NCT', 'NDT', 'NFDT', 'NFT', 'NOVST', 'NOVT', 'NPT', 'NRT', 'NST', 'NUT', 'NZDT', 'NZST', 'O', - 'OMSST', 'OMST', 'ORAT', 'P', 'PDT', 'PET', 'PETST', 'PETT', 'PGT', 'PHOT', 'PHT', 'PKT', 'PMDT', - 'PMST', 'PONT', 'PST', 'PT', 'PWT', 'PYST', 'PYT', 'Q', 'QYZT', 'R', 'RET', 'ROTT', 'S', 'SAKT', - 'SAMT', 'SAST', 'SBT', 'SCT', 'SGT', 'SRET', 'SRT', 'SST', 'SYOT', 'T', 'TAHT', 'TFT', 'TJT', 'TKT', - 'TLT', 'TMT', 'TOST', 'TOT', 'TRT', 'TVT', 'U', 'ULAST', 'ULAT', 'UYST', 'UYT', 'UZT', 'V', 'VET', - 'VLAST', 'VLAT', 'VOST', 'VUT', 'W', 'WAKT', 'WARST', 'WAST', 'WAT', 'WEST', 'WET', 'WFT', 'WGST', - 'WGT', 'WIB', 'WIT', 'WITA', 'WST', 'WT', 'X', 'Y', 'YAKST', 'YAKT', 'YAPT', 'YEKST', 'YEKT', 'Z', - 'UTC', 'UTC-1200', 'UTC-1100', 'UTC-1000', 'UTC-0930', 'UTC-0900', 'UTC-0800', 'UTC-0700', 'UTC-0600', - 'UTC-0500', 'UTC-0400', 'UTC-0300', 'UTC-0230', 'UTC-0200', 'UTC-0100', 'UTC+0000', 'UTC-0000', - 'UTC+0100', 'UTC+0200', 'UTC+0300', 'UTC+0400', 'UTC+0430', 'UTC+0500', 'UTC+0530', 'UTC+0545', - 'UTC+0600', 'UTC+0630', 'UTC+0700', 'UTC+0800', 'UTC+0845', 'UTC+0900', 'UTC+1000', 'UTC+1030', - 'UTC+1100', 'UTC+1200', 'UTC+1300', 'UTC+1345', 'UTC+1400'] + tz_abbr = [ + 'A', 'ACDT', 'ACST', 'ACT', 'ACWST', 'ADT', 'AEDT', 'AEST', 'AET', 'AFT', 'AKDT', + 'AKST', 'ALMT', 'AMST', 'AMT', 'ANAST', 'ANAT', 'AQTT', 'ART', 'AST', 'AT', 'AWDT', + 'AWST', 'AZOST', 'AZOT', 'AZST', 'AZT', 'AoE', 'B', 'BNT', 'BOT', 'BRST', 'BRT', 'BST', + 'BTT', 'C', 'CAST', 'CAT', 'CCT', 'CDT', 'CEST', 'CET', 'CHADT', 'CHAST', 'CHOST', + 'CHOT', 'CHUT', 'CIDST', 'CIST', 'CKT', 'CLST', 'CLT', 'COT', 'CST', 'CT', 'CVT', 'CXT', + 'ChST', 'D', 'DAVT', 'DDUT', 'E', 'EASST', 'EAST', 'EAT', 'ECT', 'EDT', 'EEST', 'EET', + 'EGST', 'EGT', 'EST', 'ET', 'F', 'FET', 'FJST', 'FJT', 'FKST', 'FKT', 'FNT', 'G', + 'GALT', 'GAMT', 'GET', 'GFT', 'GILT', 'GMT', 'GST', 'GYT', 'H', 'HDT', 'HKT', 'HOVST', + 'HOVT', 'HST', 'I', 'ICT', 'IDT', 'IOT', 'IRDT', 'IRKST', 'IRKT', 'IRST', 'IST', 'JST', + 'K', 'KGT', 'KOST', 'KRAST', 'KRAT', 'KST', 'KUYT', 'L', 'LHDT', 'LHST', 'LINT', 'M', + 'MAGST', 'MAGT', 'MART', 'MAWT', 'MDT', 'MHT', 'MMT', 'MSD', 'MSK', 'MST', 'MT', 'MUT', + 'MVT', 'MYT', 'N', 'NCT', 'NDT', 'NFDT', 'NFT', 'NOVST', 'NOVT', 'NPT', 'NRT', 'NST', + 'NUT', 'NZDT', 'NZST', 'O', 'OMSST', 'OMST', 'ORAT', 'P', 'PDT', 'PET', 'PETST', 'PETT', + 'PGT', 'PHOT', 'PHT', 'PKT', 'PMDT', 'PMST', 'PONT', 'PST', 'PT', 'PWT', 'PYST', 'PYT', + 'Q', 'QYZT', 'R', 'RET', 'ROTT', 'S', 'SAKT', 'SAMT', 'SAST', 'SBT', 'SCT', 'SGT', + 'SRET', 'SRT', 'SST', 'SYOT', 'T', 'TAHT', 'TFT', 'TJT', 'TKT', 'TLT', 'TMT', 'TOST', + 'TOT', 'TRT', 'TVT', 'U', 'ULAST', 'ULAT', 'UYST', 'UYT', 'UZT', 'V', 'VET', 'VLAST', + 'VLAT', 'VOST', 'VUT', 'W', 'WAKT', 'WARST', 'WAST', 'WAT', 'WEST', 'WET', 'WFT', + 'WGST', 'WGT', 'WIB', 'WIT', 'WITA', 'WST', 'WT', 'X', 'Y', 'YAKST', 'YAKT', 'YAPT', + 'YEKST', 'YEKT', 'Z', 'UTC', 'UTC-1200', 'UTC-1100', 'UTC-1000', 'UTC-0930', 'UTC-0900', + 'UTC-0800', 'UTC-0700', 'UTC-0600', 'UTC-0500', 'UTC-0400', 'UTC-0300', 'UTC-0230', + 'UTC-0200', 'UTC-0100', 'UTC+0000', 'UTC-0000', 'UTC+0100', 'UTC+0200', 'UTC+0300', + 'UTC+0400', 'UTC+0430', 'UTC+0500', 'UTC+0530', 'UTC+0545', 'UTC+0600', 'UTC+0630', + 'UTC+0700', 'UTC+0800', 'UTC+0845', 'UTC+0900', 'UTC+1000', 'UTC+1030', 'UTC+1100', + 'UTC+1200', 'UTC+1300', 'UTC+1345', 'UTC+1400' + ] tz = None for term in data.replace('(', '').replace(')', '').split(): if term in tz_abbr: diff --git a/jc/parsers/df.py b/jc/parsers/df.py index edc2bfae..9b2e857b 100644 --- a/jc/parsers/df.py +++ b/jc/parsers/df.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('df', df_command_output) + + or + import jc.parsers.df result = jc.parsers.df.parse(df_command_output) @@ -103,8 +108,6 @@ class info(): description = '`df` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['df'] @@ -158,7 +161,8 @@ def _process(proc_data): entry['iused_percent'] = entry['iused_percent'].rstrip('%') # change used, available, use_percent, capacity_percent, ifree, iused, iused_percent to int - int_list = ['used', 'available', 'use_percent', 'capacity_percent', 'ifree', 'iused', 'iused_percent'] + int_list = ['used', 'available', 'use_percent', 'capacity_percent', 'ifree', + 'iused', 'iused_percent'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) @@ -167,7 +171,10 @@ def _process(proc_data): def _long_filesystem_hash(header, line): - """Returns truncated hash and value of the filesystem field if it is too long for the column""" + """ + Returns truncated hash and value of the filesystem field if it is too + long for the column. + """ filesystem_field = line.split()[0] # get length of filesystem column @@ -196,7 +203,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: @@ -233,7 +240,8 @@ def parse(data, raw=False, quiet=False): # parse the data raw_output = jc.parsers.universal.sparse_table_parse(fix_data) - # replace hash values with real values to fix long filesystem data in some older versions of df + # replace hash values with real values to fix long filesystem data + # in some older versions of df for item in raw_output: if 'filesystem' in item: if item['filesystem'] in filesystem_map: diff --git a/jc/parsers/dig.py b/jc/parsers/dig.py index b09dccca..b7d32d50 100644 --- a/jc/parsers/dig.py +++ b/jc/parsers/dig.py @@ -1,12 +1,15 @@ """jc - JSON CLI output utility `dig` command output parser Options supported: -- `+noall +answer` options are supported in cases where only the answer information is desired. +- `+noall +answer` options are supported in cases where only the answer + information is desired. - `+axfr` option is supported on its own -The `when_epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `when_epoch` calculated timestamp field is naive. (i.e. based on the +local time of the system the parser is run on) -The `when_epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `when_epoch_utc` calculated timestamp field is timezone-aware and is +only available if the timezone field is UTC. Usage (cli): @@ -18,6 +21,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('dig', dig_command_output) + + or + import jc.parsers.dig result = jc.parsers.dig.parse(dig_command_output) @@ -90,13 +98,16 @@ Schema: "query_time": integer, # in msec "server": string, "when": string, - "when_epoch": integer, # naive timestamp if when field is parsable, else null - "when_epoch_utc": integer, # timezone aware timestamp availabe for UTC, else null + "when_epoch": integer, # [0] + "when_epoch_utc": integer, # [1] "rcvd": integer "size": string } ] + [0] naive timestamp if "when" field is parsable, else null + [1] timezone aware timestamp availabe for UTC, else null + Examples: $ dig example.com | jc --dig -p @@ -320,8 +331,6 @@ class info(): description = '`dig` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'aix', 'freebsd', 'darwin', 'win32', 'cygwin'] magic_commands = ['dig'] @@ -342,8 +351,8 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ for entry in proc_data: - int_list = ['id', 'query_num', 'answer_num', 'authority_num', 'additional_num', 'rcvd', - 'query_size', 'query_time'] + int_list = ['id', 'query_num', 'answer_num', 'authority_num', 'additional_num', + 'rcvd', 'query_size', 'query_time'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) @@ -355,10 +364,12 @@ def _process(proc_data): if 'opt_pseudosection' in entry: if 'edns' in entry['opt_pseudosection']: if 'version' in entry['opt_pseudosection']['edns']: - entry['opt_pseudosection']['edns']['version'] = jc.utils.convert_to_int(entry['opt_pseudosection']['edns']['version']) + val = jc.utils.convert_to_int(entry['opt_pseudosection']['edns']['version']) + entry['opt_pseudosection']['edns']['version'] = val if 'udp' in entry['opt_pseudosection']['edns']: - entry['opt_pseudosection']['edns']['udp'] = jc.utils.convert_to_int(entry['opt_pseudosection']['edns']['udp']) + val = jc.utils.convert_to_int(entry['opt_pseudosection']['edns']['udp']) + entry['opt_pseudosection']['edns']['udp'] = val if 'answer' in entry: for ans in entry['answer']: @@ -521,7 +532,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/dir.py b/jc/parsers/dir.py index 9d5045e5..5c65750d 100644 --- a/jc/parsers/dir.py +++ b/jc/parsers/dir.py @@ -6,9 +6,11 @@ Options supported: - `/C, /-C` - `/S` -The "Magic" syntax is not supported since the `dir` command is a shell builtin. +The "Magic" syntax is not supported since the `dir` command is a shell +builtin. -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) Usage (cli): @@ -16,6 +18,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('dir', dir_command_output) + + or + import jc.parsers.dir result = jc.parsers.dir.parse(dir_command_output) @@ -123,8 +130,6 @@ class info(): description = '`dir` command parser' author = 'Rasheed Elsaleh' author_email = 'rasheed@rebelliondefense.com' - - # compatible options: win32 compatible = ['win32'] @@ -166,7 +171,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/dmidecode.py b/jc/parsers/dmidecode.py index e3e8ca39..f87e06bf 100644 --- a/jc/parsers/dmidecode.py +++ b/jc/parsers/dmidecode.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('dmidecode', dmidecode_command_output) + + or + import jc.parsers.dmidecode result = jc.parsers.dmidecode.parse(dmidecode_command_output) @@ -21,7 +26,7 @@ Schema: "type": integer, "bytes": integer, "description": string, - "values": { (null if empty) + "values": { # null if empty "lowercase_no_spaces_keys": string, "multiline_key_values": [ string, @@ -129,9 +134,6 @@ class info(): description = '`dmidecode` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['dmidecode'] @@ -170,7 +172,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/dpkg_l.py b/jc/parsers/dpkg_l.py index 3fb26974..260f1433 100644 --- a/jc/parsers/dpkg_l.py +++ b/jc/parsers/dpkg_l.py @@ -1,6 +1,7 @@ """jc - JSON CLI output utility `dpkg -l` command output parser -Set the `COLUMNS` environment variable to a large value to avoid field truncation. For example: +Set the `COLUMNS` environment variable to a large value to avoid field +truncation. For example: $ COLUMNS=500 dpkg -l | jc --dpkg-l @@ -14,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('dpkg_l', dpkg_command_output) + + or + import jc.parsers.dpkg_l result = jc.parsers.dpkg_l.parse(dpkg_command_output) @@ -69,7 +75,7 @@ Examples: "name": "acpid", "version": "1:2.0.28-1ubuntu1", "architecture": "amd64", - "description": "Advanced Configuration and Power Interface event daemon", + "description": "Advanced Configuration and Power Interface...", "desired": "remove", "status": "half installed" }, @@ -113,7 +119,7 @@ Examples: "name": "acpid", "version": "1:2.0.28-1ubuntu1", "architecture": "amd64", - "description": "Advanced Configuration and Power Interface event daemon" + "description": "Advanced Configuration and Power Interface..." }, { "codes": "pn", @@ -135,9 +141,6 @@ class info(): description = '`dpkg -l` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['dpkg -l'] @@ -210,7 +213,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/du.py b/jc/parsers/du.py index 37d4bb11..e91aaee0 100644 --- a/jc/parsers/du.py +++ b/jc/parsers/du.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('du', du_command_output) + + or + import jc.parsers.du result = jc.parsers.du.parse(du_command_output) @@ -32,23 +37,23 @@ Examples: }, { "size": 56, - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/_CodeSignature" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": 0, - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr/local/standalone" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": 0, - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr/local" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": 0, - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": 1008, - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/dfu" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, ... ] @@ -61,23 +66,23 @@ Examples: }, { "size": "56", - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/_CodeSignature" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": "0", - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr/local/standalone" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": "0", - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr/local" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": "0", - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/usr" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, { "size": "1008", - "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/Contents/Resources/Firmware/dfu" + "name": "/usr/standalone/firmware/iBridge1_1Customer.bundle/..." }, ... ] @@ -92,9 +97,6 @@ class info(): description = '`du` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'aix', 'freebsd'] magic_commands = ['du'] @@ -130,7 +132,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/env.py b/jc/parsers/env.py index ff56cc5e..d6155df3 100644 --- a/jc/parsers/env.py +++ b/jc/parsers/env.py @@ -1,6 +1,9 @@ """jc - JSON CLI output utility `env` and `printenv` command output parser -This parser will output a list of dictionaries each containing `name` and `value` keys. If you would like a simple dictionary output, then use the `-r` command-line option or the `raw=True` argument in the `parse()` function. +This parser will output a list of dictionaries each containing `name` and +`value` keys. If you would like a simple dictionary output, then use the +`-r` command-line option or the `raw=True` argument in the `parse()` +function. Usage (cli): @@ -12,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('env', env_command_output) + + or + import jc.parsers.env result = jc.parsers.env.parse(env_command_output) @@ -73,8 +81,6 @@ class info(): description = '`env` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] magic_commands = ['env', 'printenv'] @@ -113,7 +119,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/file.py b/jc/parsers/file.py index aeb844c7..d0cefe9f 100644 --- a/jc/parsers/file.py +++ b/jc/parsers/file.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('file', file_command_output) + + or + import jc.parsers.file result = jc.parsers.file.parse(file_command_output) @@ -18,7 +23,7 @@ Schema: [ { "filename": string, - "type ": string + "type": string } ] @@ -48,11 +53,11 @@ Examples: }, { "filename": "cd_catalog.xml", - "type": "XML 1.0 document text, ASCII text, with CRLF line terminators" + "type": "XML 1.0 document text, ASCII text, with CRLF line ..." }, { "filename": "centosserial.sh", - "type": "Bourne-Again shell script text executable, UTF-8 Unicode text" + "type": "Bourne-Again shell script text executable, UTF-8 ..." }, ... ] @@ -67,8 +72,6 @@ class info(): description = '`file` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'aix', 'freebsd', 'darwin'] magic_commands = ['file'] @@ -99,7 +102,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/finger.py b/jc/parsers/finger.py index b7721677..d2b45eca 100644 --- a/jc/parsers/finger.py +++ b/jc/parsers/finger.py @@ -12,6 +12,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('finger', finger_command_output) + + or + import jc.parsers.finger result = jc.parsers.finger.parse(finger_command_output) @@ -96,9 +101,6 @@ class info(): description = '`finger` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'freebsd'] magic_commands = ['finger'] @@ -161,7 +163,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: @@ -174,11 +176,12 @@ def parse(data, raw=False, quiet=False): raw_output = [] if jc.utils.has_data(data): - # Finger output is an abomination that is nearly unparsable. But there is a way: - # First find the location of the last character of 'Idle' in the table and cut - # all lines at that spot. Data before that spot can use the unviversal.sparse_table_parse function. - # All data after that spot can be run through regex to find the login datetime and possibly - # other fields. + # Finger output is an abomination that is nearly unparsable. But there + # is a way: + # First find the location of the last character of 'Idle' in the table + # and cut all lines at that spot. Data before that spot can use the + # unviversal.sparse_table_parse function. All data after that spot can + # be run through regex to find the login datetime and possibly other fields. data_lines = list(filter(None, data.splitlines())) sep_col = data_lines[0].find('Idle') + 4 diff --git a/jc/parsers/foo.py b/jc/parsers/foo.py index c97ac772..2743c05f 100644 --- a/jc/parsers/foo.py +++ b/jc/parsers/foo.py @@ -12,6 +12,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('foo', foo_command_output) + + or + import jc.parsers.foo result = jc.parsers.foo.parse(foo_command_output) @@ -67,7 +72,8 @@ def _process(proc_data): # process the data here # rebuild output for added semantic information - # use helper functions in jc.utils for int, float, bool conversions and timestamps + # use helper functions in jc.utils for int, float, bool + # conversions and timestamps return proc_data @@ -79,7 +85,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: @@ -94,9 +100,11 @@ def parse(data, raw=False, quiet=False): if jc.utils.has_data(data): for line in filter(None, data.splitlines()): - # + # parse the content here - # + # check out helper functions in jc.utils + # and jc.parsers.universal + pass return raw_output if raw else _process(raw_output) diff --git a/jc/parsers/foo_s.py b/jc/parsers/foo_s.py index 5444372c..9b29e337 100644 --- a/jc/parsers/foo_s.py +++ b/jc/parsers/foo_s.py @@ -10,8 +10,17 @@ Usage (cli): Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('foo_s', foo_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.foo_s - result = jc.parsers.foo_s.parse(foo_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.foo_s.parse(foo_command_output.splitlines()) for item in result: # do something @@ -19,9 +28,12 @@ Schema: { "foo": string, - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error + "success": boolean, # false if error parsing "error": string, # exists if "success" is false "line": string # exists if "success" is false } @@ -69,11 +81,11 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - # + # process the data here # rebuild output for added semantic information - # use helper functions in jc.utils for int, float, bool conversions and timestamps - # + # use helper functions in jc.utils for int, float, + # bool conversions and timestamps return proc_data @@ -84,8 +96,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True @@ -105,9 +119,9 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): try: jc.utils.streaming_line_input_type_check(line) - # - # parse the input here - # + # parse the content here + # check out helper functions in jc.utils + # and jc.parsers.universal if output_line: yield stream_success(output_line, ignore_exceptions) if raw else stream_success(_process(output_line), ignore_exceptions) diff --git a/jc/parsers/free.py b/jc/parsers/free.py index 84474f67..a7af04d5 100644 --- a/jc/parsers/free.py +++ b/jc/parsers/free.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('free', free_command_output) + + or + import jc.parsers.free result = jc.parsers.free.parse(free_command_output) @@ -77,8 +82,6 @@ class info(): description = '`free` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['free'] @@ -115,7 +118,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/fstab.py b/jc/parsers/fstab.py index 91e5df34..6d2689da 100644 --- a/jc/parsers/fstab.py +++ b/jc/parsers/fstab.py @@ -6,6 +6,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('fstab', fstab_command_output) + + or + import jc.parsers.fstab result = jc.parsers.fstab.parse(fstab_command_output) @@ -89,8 +94,6 @@ class info(): description = '`/etc/fstab` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'freebsd'] @@ -125,7 +128,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/group.py b/jc/parsers/group.py index 01cadbe1..bf9bef87 100644 --- a/jc/parsers/group.py +++ b/jc/parsers/group.py @@ -6,6 +6,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('group', group_file_output) + + or + import jc.parsers.group result = jc.parsers.group.parse(group_file_output) @@ -113,9 +118,6 @@ class info(): description = '`/etc/group` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'aix', 'freebsd'] @@ -153,7 +155,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/gshadow.py b/jc/parsers/gshadow.py index b260fcb2..d5df2b3b 100644 --- a/jc/parsers/gshadow.py +++ b/jc/parsers/gshadow.py @@ -6,6 +6,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('gshadow', gshadow_file_output) + + or + import jc.parsers.gshadow result = jc.parsers.gshadow.parse(gshadow_file_output) @@ -81,9 +86,6 @@ class info(): description = '`/etc/gshadow` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'aix', 'freebsd'] @@ -119,7 +121,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/hash.py b/jc/parsers/hash.py index 275d8495..1efa91c6 100644 --- a/jc/parsers/hash.py +++ b/jc/parsers/hash.py @@ -6,6 +6,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('hash', hash_command_output) + + or + import jc.parsers.hash result = jc.parsers.hash.parse(hash_command_output) @@ -42,8 +47,6 @@ class info(): description = '`hash` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] @@ -78,7 +81,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/hashsum.py b/jc/parsers/hashsum.py index 1582a794..3f3356eb 100644 --- a/jc/parsers/hashsum.py +++ b/jc/parsers/hashsum.py @@ -20,6 +20,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('hashsum', md5sum_command_output) + + or + import jc.parsers.hashsum result = jc.parsers.hashsum.parse(md5sum_command_output) @@ -73,10 +78,9 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Parses MD5 and SHA hash program output' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] - magic_commands = ['md5sum', 'md5', 'shasum', 'sha1sum', 'sha224sum', 'sha256sum', 'sha384sum', 'sha512sum'] + magic_commands = ['md5sum', 'md5', 'shasum', 'sha1sum', 'sha224sum', + 'sha256sum', 'sha384sum', 'sha512sum'] __version__ = info.version @@ -106,7 +110,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/hciconfig.py b/jc/parsers/hciconfig.py index 77e5cc3b..9aff32f9 100644 --- a/jc/parsers/hciconfig.py +++ b/jc/parsers/hciconfig.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('hciconfig', hciconfig_command_output) + + or + import jc.parsers.hciconfig result = jc.parsers.hciconfig.parse(hciconfig_command_output) @@ -321,9 +326,6 @@ class info(): description = '`hciconfig` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['hciconfig'] @@ -346,13 +348,17 @@ def _process(proc_data): for entry in proc_data: - int_list = ['acl_mtu', 'acl_mtu_packets', 'sco_mtu', 'sco_mtu_packets', 'rx_bytes', 'rx_acl', 'rx_sco', - 'rx_events', 'rx_errors', 'tx_bytes', 'tx_acl', 'tx_sco', 'tx_commands', 'tx_errors'] + int_list = ['acl_mtu', 'acl_mtu_packets', 'sco_mtu', 'sco_mtu_packets', + 'rx_bytes', 'rx_acl', 'rx_sco', 'rx_events', 'rx_errors', + 'tx_bytes', 'tx_acl', 'tx_sco', 'tx_commands', 'tx_errors'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) - if 'service_classes' in entry and len(entry['service_classes']) == 1 and 'Unspecified' in entry['service_classes']: + if ('service_classes' in entry and + len(entry['service_classes']) == 1 and + 'Unspecified' in entry['service_classes']): + entry['service_classes'] = None return proc_data @@ -365,7 +371,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/history.py b/jc/parsers/history.py index 43431cfe..2ed864a6 100644 --- a/jc/parsers/history.py +++ b/jc/parsers/history.py @@ -1,8 +1,12 @@ """jc - JSON CLI output utility `history` command output parser -This parser will output a list of dictionaries each containing `line` and `command` keys. If you would like a simple dictionary output, then use the `-r` command-line option or the `raw=True` argument in the `parse()` function. +This parser will output a list of dictionaries each containing `line` and +`command` keys. If you would like a simple dictionary output, then use the +`-r` command-line option or the `raw=True` argument in the `parse()` +function. -The "Magic" syntax is not supported since the `history` command is a shell builtin. +The "Magic" syntax is not supported since the `history` command is a shell +builtin. Usage (cli): @@ -10,6 +14,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('history', history_command_output) + + or + import jc.parsers.history result = jc.parsers.history.parse(history_command_output) @@ -64,8 +73,6 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Optimizations by https://github.com/philippeitis' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] @@ -101,7 +108,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/hosts.py b/jc/parsers/hosts.py index b0a0b10f..6f9a5071 100644 --- a/jc/parsers/hosts.py +++ b/jc/parsers/hosts.py @@ -6,6 +6,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('hosts', hosts_file_output) + + or + import jc.parsers.hosts result = jc.parsers.hosts.parse(hosts_file_output) @@ -78,8 +83,6 @@ class info(): description = '`/etc/hosts` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] @@ -110,7 +113,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/id.py b/jc/parsers/id.py index d824053b..3e14d467 100644 --- a/jc/parsers/id.py +++ b/jc/parsers/id.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('id', id_command_output) + + or + import jc.parsers.id result = jc.parsers.id.parse(id_command_output) @@ -109,9 +114,6 @@ class info(): description = '`id` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'aix', 'freebsd'] magic_commands = ['id'] @@ -154,7 +156,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/ifconfig.py b/jc/parsers/ifconfig.py index 15b8006d..8367671f 100644 --- a/jc/parsers/ifconfig.py +++ b/jc/parsers/ifconfig.py @@ -12,6 +12,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ifconfig', ifconfig_command_output) + + or + import jc.parsers.ifconfig result = jc.parsers.ifconfig.parse(ifconfig_command_output) @@ -193,8 +198,6 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Using ifconfig-parser from https://github.com/KnightWhoSayNi/ifconfig-parser' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'aix', 'freebsd', 'darwin'] magic_commands = ['ifconfig'] @@ -230,10 +233,12 @@ class _IfconfigParser(object): # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - attributes = ['name', 'type', 'mac_addr', 'ipv4_addr', 'ipv4_bcast', 'ipv4_mask', 'ipv6_addr', 'ipv6_mask', - 'ipv6_scope', 'state', 'mtu', 'metric', 'rx_packets', 'rx_errors', 'rx_dropped', 'rx_overruns', - 'rx_frame', 'tx_packets', 'tx_errors', 'tx_dropped', 'tx_overruns', 'tx_carrier', 'tx_collisions', - 'rx_bytes', 'tx_bytes'] + attributes = [ + 'name', 'type', 'mac_addr', 'ipv4_addr', 'ipv4_bcast', 'ipv4_mask', 'ipv6_addr', + 'ipv6_mask', 'ipv6_scope', 'state', 'mtu', 'metric', 'rx_packets', 'rx_errors', + 'rx_dropped', 'rx_overruns', 'rx_frame', 'tx_packets', 'tx_errors', 'tx_dropped', + 'tx_overruns', 'tx_carrier', 'tx_collisions', 'rx_bytes', 'tx_bytes' + ] def __init__(self, console_output): """ @@ -431,9 +436,11 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ for entry in proc_data: - int_list = ['flags', 'mtu', 'ipv6_mask', 'rx_packets', 'rx_bytes', 'rx_errors', 'rx_dropped', 'rx_overruns', - 'rx_frame', 'tx_packets', 'tx_bytes', 'tx_errors', 'tx_dropped', 'tx_overruns', 'tx_carrier', - 'tx_collisions', 'metric'] + int_list = [ + 'flags', 'mtu', 'ipv6_mask', 'rx_packets', 'rx_bytes', 'rx_errors', 'rx_dropped', + 'rx_overruns', 'rx_frame', 'tx_packets', 'tx_bytes', 'tx_errors', 'tx_dropped', + 'tx_overruns', 'tx_carrier', 'tx_collisions', 'metric' + ] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) @@ -467,7 +474,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/ini.py b/jc/parsers/ini.py index ac1c681d..23a9d15a 100644 --- a/jc/parsers/ini.py +++ b/jc/parsers/ini.py @@ -1,8 +1,12 @@ """jc - JSON CLI output utility `INI` file parser -Parses standard `INI` files and files containing simple key/value pairs. Delimiter can be `=` or `:`. Missing values are supported. Comment prefix can be `#` or `;`. Comments must be on their own line. +Parses standard `INI` files and files containing simple key/value pairs. +Delimiter can be `=` or `:`. Missing values are supported. Comment prefix +can be `#` or `;`. Comments must be on their own line. -Note: Values starting and ending with quotation marks will have the marks removed. If you would like to keep the quotation marks, use the `-r` command-line argument or the `raw=True` argument in `parse()`. +Note: Values starting and ending with quotation marks will have the marks +removed. If you would like to keep the quotation marks, use the `-r` +command-line argument or the `raw=True` argument in `parse()`. Usage (cli): @@ -10,16 +14,18 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ini', ini_file_output) + + or + import jc.parsers.ini result = jc.parsers.ini.parse(ini_file_output) Schema: - ini or key/value document converted to a dictionary - see configparser standard - library documentation for more details. - - Note: Values starting and ending with quotation marks will have the marks removed. - If you would like to keep the quotation marks, use the -r or raw=True argument. + ini or key/value document converted to a dictionary - see the + configparser standard library documentation for more details. { "key1": string, @@ -71,8 +77,6 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Using configparser from the standard library' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] @@ -103,8 +107,12 @@ def _process(proc_data): # simple key/value files with no headers else: - if proc_data[heading] is not None and proc_data[heading].startswith('"') and proc_data[heading].endswith('"'): + if (proc_data[heading] is not None and + proc_data[heading].startswith('"') and + proc_data[heading].endswith('"')): + proc_data[heading] = proc_data[heading].lstrip('"').rstrip('"') + elif proc_data[heading] is None: proc_data[heading] = '' @@ -118,7 +126,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/iostat.py b/jc/parsers/iostat.py index d33d5e93..6eaf3ca6 100644 --- a/jc/parsers/iostat.py +++ b/jc/parsers/iostat.py @@ -12,6 +12,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('iostat', iostat_command_output) + + or + import jc.parsers.iostat result = jc.parsers.iostat.parse(iostat_command_output) @@ -220,7 +225,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/iostat_s.py b/jc/parsers/iostat_s.py index e2fd3237..d6ac5b4b 100644 --- a/jc/parsers/iostat_s.py +++ b/jc/parsers/iostat_s.py @@ -10,8 +10,17 @@ Usage (cli): Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('iostat_s', iostat_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.iostat_s - result = jc.parsers.iostat_s.parse(iostat_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.iostat_s.parse(iostat_command_output.splitlines()) for item in result: # do something @@ -68,9 +77,12 @@ Schema: "percent_util": float, "percent_rrqm": float, "percent_wrqm": float, - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error + "success": boolean, # false if error parsing "error": string, # exists if "success" is false "line": string # exists if "success" is false } @@ -79,13 +91,13 @@ Schema: Examples: $ iostat | jc --iostat-s - {"percent_user":0.14,"percent_nice":0.0,"percent_system":0.16,"percent_iowait":0.0,"percent_steal":0.0,"percent_idle":99.7,"type":"cpu"} - {"device":"sda","tps":0.24,"kb_read_s":5.28,"kb_wrtn_s":1.1,"kb_read":203305,"kb_wrtn":42368,"type":"device"} + {"percent_user":0.14,"percent_nice":0.0,"percent_system":0.16,...} + {"device":"sda","tps":0.24,"kb_read_s":5.28,"kb_wrtn_s":1.1...} ... $ iostat | jc --iostat-s -r - {"percent_user":"0.14","percent_nice":"0.00","percent_system":"0.16","percent_iowait":"0.00","percent_steal":"0.00","percent_idle":"99.70","type":"cpu"} - {"device":"sda","tps":"0.24","kb_read_s":"5.28","kb_wrtn_s":"1.10","kb_read":"203305","kb_wrtn":"42368","type":"device"} + {"percent_user":"0.14","percent_nice":"0.00","percent_system":"0.16"...} + {"device":"sda","tps":"0.24","kb_read_s":"5.28","kb_wrtn_s":"1.10"...} ... """ import jc.utils @@ -153,8 +165,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/jc/parsers/iptables.py b/jc/parsers/iptables.py index ce217c10..a0e755b9 100644 --- a/jc/parsers/iptables.py +++ b/jc/parsers/iptables.py @@ -1,4 +1,4 @@ -"""jc - JSON CLI output utility `ipables` command output parser +"""jc - JSON CLI output utility `iptables` command output parser Supports `-vLn` and `--line-numbers` for all tables. @@ -12,6 +12,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('iptables', iptables_command_output) + + or + import jc.parsers.iptables result = jc.parsers.iptables.parse(iptables_command_output) @@ -167,8 +172,6 @@ class info(): description = '`iptables` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['iptables'] @@ -233,7 +236,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/iw_scan.py b/jc/parsers/iw_scan.py index ede62aca..3c984719 100644 --- a/jc/parsers/iw_scan.py +++ b/jc/parsers/iw_scan.py @@ -1,6 +1,7 @@ """jc - JSON CLI output utility `iw dev scan` command output parser -This parser is considered beta quality. Not all fields are parsed and there are not enough samples to test. +This parser is considered beta quality. Not all fields are parsed and there +are not enough samples to test. Usage (cli): @@ -12,14 +13,19 @@ Usage (cli): Usage (module): - import jc.parsers.iw-scan - result = jc.parsers.iw-scan.parse(iw-scan_command_output) + import jc + result = jc.parse('iw_scan', iw_scan_command_output) + + or + + import jc.parsers.iw_scan + result = jc.parsers.iw_scan.parse(iw_scan_command_output) Schema: [ { - "foo": string/integer/float, # best guess based on value + "foo": string/integer/float, # best guess based on value "bar": string/integer/float, "baz": string/integer/float } @@ -126,8 +132,6 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Enhancements by Philipp Schmitt (https://pschmitt.dev/)' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['iw dev'] @@ -286,7 +290,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: @@ -320,7 +324,9 @@ def parse(data, raw=False, quiet=False): split_line = line.split(':', maxsplit=1) if len(split_line) == 2: - split_line[0] = split_line[0].lower().replace('*', '').replace('(', '').replace(')', '').replace(',', '').replace('-', '_').strip().replace(' ', '_') + split_line[0] = split_line[0].lower().replace('*', '').replace('(', '')\ + .replace(')', '').replace(',', '').replace('-', '_')\ + .strip().replace(' ', '_') section[split_line[0]] = split_line[1].strip() continue diff --git a/jc/parsers/jar_manifest.py b/jc/parsers/jar_manifest.py index d625930a..e9afa424 100644 --- a/jc/parsers/jar_manifest.py +++ b/jc/parsers/jar_manifest.py @@ -6,6 +6,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('jar_manifest', jar_manifest_file_output) + + or + import jc.parsers.jar_manifest result = jc.parsers.jar_manifest.parse(jar_manifest_file_output) @@ -21,15 +26,16 @@ Schema: Examples: $ cat MANIFEST.MF | jc --jar-manifest -p - $ unzip -c apache-log4j-2.16.0-bin/log4j-core-2.16.0.jar META-INF/MANIFEST.MF | jc --jar-manifest -p - $ unzip -c 'apache-log4j-2.16.0-bin/*.jar' META-INF/MANIFEST.MF | jc --jar-manifest -p + $ unzip -c log4j-core-2.16.0.jar META-INF/MANIFEST.MF | \\ + jc --jar-manifest -p + $ unzip -c 'apache-log4j-2.16.0-bin/*.jar' META-INF/MANIFEST.MF | \\ + jc --jar-manifest -p $ cat MANIFEST.MF | jc --jar-manifest -p - [ { - "Import_Package": "com.conversantmedia.util.concurrent;resolution:=optional,com.fasterxml.jackson.annotation;version=\"[2.12,3)\";resolution:=optional,com.fasterxml.jackson.core;version=\"[2.12,3)\";resolution:=optional,com.fasterxml.jackson.core.type;version=\"[2.12,3)\";resolution:=optional,com.fasterxml.jackson.cor...", - "Export_Package": "org.apache.logging.log4j.core;uses:=\"org.apache.logging.log4j,org.apache.logging.log4j.core.config,org.apache.logging.log4j.core.impl,org.apache.logging.log4j.core.layout,org.apache.logging.log4j.core.time,org.apache.logging.log4j.message,org.apache.logging.log4j.spi,org.apache.logging.log4j.status...", + "Import_Package": "com.conversantmedia.util.concurrent;resoluti...", + "Export_Package": "org.apache.logging.log4j.core;uses:=\"org.ap...", "Manifest_Version": "1.0", "Bundle_License": "https://www.apache.org/licenses/LICENSE-2.0.txt", "Bundle_SymbolicName": "org.apache.logging.log4j.core", @@ -42,28 +48,28 @@ Examples: } ] - $ unzip -c 'apache-log4j-2.16.0-bin/*.jar' META-INF/MANIFEST.MF | jc --jar-manifest -p - + $ unzip -c 'apache-log4j-2.16.0-bin/*.jar' META-INF/MANIFEST.MF | \\ + jc --jar-manifest -p [ ... { - "Archive": "apache-log4j-2.16.0-bin/log4j-spring-boot-2.16.0-sources.jar", + "Archive": "apache-log4j-2.16.0-bin/log4j-spring-boot-2.16.0-so...", "Manifest_Version": "1.0", "Built_By": "matt", "Created_By": "Apache Maven 3.8.4", "Build_Jdk": "1.8.0_312" }, { - "Archive": "apache-log4j-2.16.0-bin/log4j-spring-boot-2.16.0-javadoc.jar", + "Archive": "apache-log4j-2.16.0-bin/log4j-spring-boot-2.16.0-ja...", "Manifest_Version": "1.0", "Built_By": "matt", "Created_By": "Apache Maven 3.8.4", "Build_Jdk": "1.8.0_312" }, { - "Bundle_SymbolicName": "org.apache.logging.log4j.spring-cloud-config-client.logging.log4j.core.util;version=\"[2.16,3)\",org.springframework.boot.autoconfigure.condition,org.springframework.cloud.context.environment,org.springframework.context,org.springframework.stereotype", - "Export_Package": "org.apache.logging.log4j.spring.cloud.config.controller;version=\"2.16.0\"ient", - "Archive": "apache-log4j-2.16.0-bin/log4j-spring-cloud-config-client-2.16.0.jar", + "Bundle_SymbolicName": "org.apache.logging.log4j.spring-cloud-c...", + "Export_Package": "org.apache.logging.log4j.spring.cloud.config...", + "Archive": "apache-log4j-2.16.0-bin/log4j-spring-cloud-config-c...", "Manifest_Version": "1.0", "Bundle_License": "https://www.apache.org/licenses/LICENSE-2.0.txt", ... @@ -110,7 +116,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/jobs.py b/jc/parsers/jobs.py index ad7bbd0a..cd7fd10b 100644 --- a/jc/parsers/jobs.py +++ b/jc/parsers/jobs.py @@ -2,7 +2,8 @@ Also supports the `-l` option. -The "Magic" syntax is not supported since the `jobs` command is a shell builtin. +The "Magic" syntax is not supported since the `jobs` command is a shell +builtin. Usage (cli): @@ -10,6 +11,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('jobs', jobs_command_output) + + or + import jc.parsers.jobs result = jc.parsers.jobs.parse(jobs_command_output) @@ -97,8 +103,6 @@ class info(): description = '`jobs` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] magic_commands = ['jobs'] @@ -134,7 +138,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/kv.py b/jc/parsers/kv.py index 0feec0fc..a179bff4 100644 --- a/jc/parsers/kv.py +++ b/jc/parsers/kv.py @@ -1,8 +1,12 @@ """jc - JSON CLI output utility `Key/Value` file parser -Supports files containing simple key/value pairs. Delimiter can be `=` or `:`. Missing values are supported. Comment prefix can be `#` or `;`. Comments must be on their own line. +Supports files containing simple key/value pairs. Delimiter can be `=` or +`:`. Missing values are supported. Comment prefix can be `#` or `;`. +Comments must be on their own line. -Note: Values starting and ending with quotation marks will have the marks removed. If you would like to keep the quotation marks, use the `-r` command-line argument or the `raw=True` argument in `parse()`. +Note: Values starting and ending with quotation marks will have the marks +removed. If you would like to keep the quotation marks, use the `-r` +command-line argument or the `raw=True` argument in `parse()`. Usage (cli): @@ -10,12 +14,18 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('kv', kv_file_output) + + or + import jc.parsers.kv result = jc.parsers.kv.parse(kv_file_output) Schema: - key/value document converted to a dictionary - see configparser standard library documentation for more details. + key/value document converted to a dictionary - see the + configparser standard library documentation for more details. { "key1": string, @@ -52,8 +62,6 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'This is a wrapper for the INI parser' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] @@ -69,7 +77,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/last.py b/jc/parsers/last.py index fb497bc3..ba6c7d16 100644 --- a/jc/parsers/last.py +++ b/jc/parsers/last.py @@ -2,7 +2,9 @@ Supports `-w` and `-F` options. -Calculated epoch time fields are naive (i.e. based on the local time of the system the parser is run on) since there is no timezone information in the `last` command output. +Calculated epoch time fields are naive (i.e. based on the local time of the +system the parser is run on) since there is no timezone information in the +`last` command output. Usage (cli): @@ -14,6 +16,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('last', last_command_output) + + or + import jc.parsers.last result = jc.parsers.last.parse(last_command_output) @@ -27,9 +34,9 @@ Schema: "login": string, "logout": string, "duration": string, - "login_epoch": integer, # (naive) available with last -F option - "logout_epoch": integer, # (naive) available with last -F option - "duration_seconds": integer # available with last -F option + "login_epoch": integer, # (naive) available w/last -F option + "logout_epoch": integer, # (naive) available w/last -F option + "duration_seconds": integer # available w/last -F option } ] @@ -109,8 +116,6 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Enhancements by https://github.com/zerolagtime' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'aix', 'freebsd'] magic_commands = ['last', 'lastb'] @@ -178,7 +183,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: @@ -198,7 +203,10 @@ def parse(data, raw=False, quiet=False): for entry in cleandata: output_line = {} - if entry.startswith('wtmp begins ') or entry.startswith('btmp begins ') or entry.startswith('utx.log begins '): + if (entry.startswith('wtmp begins ') or + entry.startswith('btmp begins ') or + entry.startswith('utx.log begins ')): + continue entry = entry.replace('system boot', 'system_boot') diff --git a/jc/parsers/ls.py b/jc/parsers/ls.py index 1ce7529b..e497c194 100644 --- a/jc/parsers/ls.py +++ b/jc/parsers/ls.py @@ -4,11 +4,17 @@ Options supported: - `lbaR1` - `--time-style=full-iso` -Note: The `-1`, `-l`, or `-b` option of `ls` should be used to correctly parse filenames that include newline characters. Since `ls` does not encode newlines in filenames when outputting to a pipe it will cause `jc` to see multiple files instead of a single file if `-1`, `-l`, or `-b` is not used. Alternatively, `vdir` can be used, which is the same as running `ls -lb`. +Note: The `-1`, `-l`, or `-b` option of `ls` should be used to correctly +parse filenames that include newline characters. Since `ls` does not encode +newlines in filenames when outputting to a pipe it will cause `jc` to see +multiple files instead of a single file if `-1`, `-l`, or `-b` is not used. +Alternatively, `vdir` can be used, which is the same as running `ls -lb`. -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the timezone field is UTC. Usage (cli): @@ -20,6 +26,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ls', ls_command_output) + + or + import jc.parsers.ls result = jc.parsers.ls.parse(ls_command_output) @@ -35,11 +46,15 @@ Schema: "group": string, "size": integer, "date": string, - "epoch": integer, # naive timestamp if date field exists and can be converted - "epoch_utc": integer # timezone aware timestamp if date field is in UTC and can be converted + "epoch": integer, # [0] + "epoch_utc": integer # [1] } ] + [0] naive timestamp if date field exists and can be converted. + [1] timezone aware timestamp if date field is in UTC and can + be converted. + Examples: $ ls /usr/bin | jc --ls -p @@ -111,8 +126,6 @@ class info(): description = '`ls` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] magic_commands = ['ls', 'vdir'] @@ -155,7 +168,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/ls_s.py b/jc/parsers/ls_s.py index 27117d27..7de75ce9 100644 --- a/jc/parsers/ls_s.py +++ b/jc/parsers/ls_s.py @@ -1,14 +1,19 @@ -"""jc - JSON CLI output utility `ls` and `vdir` command output streaming parser +"""jc - JSON CLI output utility `ls` and `vdir` command output streaming +parser > This streaming parser outputs JSON Lines -Requires the `-l` option to be used on `ls`. If there are newline characters in the filename, then make sure to use the `-b` option on `ls`. +Requires the `-l` option to be used on `ls`. If there are newline characters +in the filename, then make sure to use the `-b` option on `ls`. -The `jc` `-qq` option can be used to ignore parsing errors. (e.g. filenames with newline characters, but `-b` was not used) +The `jc` `-qq` option can be used to ignore parsing errors. (e.g. filenames +with newline characters, but `-b` was not used) -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive (i.e. based on the local +time of the system the parser is run on) -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the timezone field is UTC. Usage (cli): @@ -16,8 +21,17 @@ Usage (cli): Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('ls_s', ls_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.ls_s - result = jc.parsers.ls_s.parse(ls_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.ls_s.parse(ls_command_output.splitlines()) for item in result: # do something @@ -32,28 +46,35 @@ Schema: "group": string, "size": integer, "date": string, - "epoch": integer, # naive timestamp if date field exists and can be converted - "epoch_utc": integer, # timezone aware timestamp if date field is in UTC and can be converted - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + "epoch": integer, # [0] + "epoch_utc": integer, # [1] + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error + "success": boolean, # false if error parsing "error": string, # exists if "success" is false "line": string # exists if "success" is false } } + [0] naive timestamp if date field exists and can be converted. + [1] timezone aware timestamp if date field is in UTC and can + be converted + Examples: $ ls -l /usr/bin | jc --ls-s - {"filename":"2to3-","flags":"-rwxr-xr-x","links":4,"owner":"root","group":"wheel","size":925,"date":"Feb 22 2019"} - {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7","flags":"lrwxr-xr-x","links":1,"owner":"root","group":"wheel","size":74,"date":"May 4 2019"} - {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":1,"owner":"root","group":"wheel","size":55152,"date":"May 3 2019"} + {"filename":"2to3-","flags":"-rwxr-xr-x","links":4,"owner":"root","...} + {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/P...} + {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":1,...} ... $ ls -l /usr/bin | jc --ls-s -r - {"filename":"2to3-","flags":"-rwxr-xr-x","links":"4","owner":"root","group":"wheel","size":"925","date":"Feb 22 2019"} - {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/2to3-2.7","flags":"lrwxr-xr-x","links":"1","owner":"root","group":"wheel","size":"74","date":"May 4 2019"} - {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":"1","owner":"root","group":"wheel","size":"55152","date":"May 3 2019"} + {"filename":"2to3-","flags":"-rwxr-xr-x","links":"4","owner":"roo"..."} + {"filename":"2to3-2.7","link_to":"../../System/Library/Frameworks/P...} + {"filename":"AssetCacheLocatorUtil","flags":"-rwxr-xr-x","links":"1...} ... """ import re @@ -68,8 +89,6 @@ class info(): description = '`ls` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] streaming = True @@ -110,8 +129,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/jc/parsers/lsblk.py b/jc/parsers/lsblk.py index ca4f7184..45ddd844 100644 --- a/jc/parsers/lsblk.py +++ b/jc/parsers/lsblk.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('lsblk', lsblk_command_output) + + or + import jc.parsers.lsblk result = jc.parsers.lsblk.parse(lsblk_command_output) @@ -85,7 +90,10 @@ Examples: ... ] - $ lsblk -o +KNAME,FSTYPE,LABEL,UUID,PARTLABEL,PARTUUID,RA,MODEL,SERIAL,STATE,OWNER,GROUP,MODE,ALIGNMENT,MIN-IO,OPT-IO,PHY-SEC,LOG-SEC,ROTA,SCHED,RQ-SIZE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,WSAME,WWN,RAND,PKNAME,HCTL,TRAN,REV,VENDOR | jc --lsblk -p + $ lsblk -o +KNAME,FSTYPE,LABEL,UUID,PARTLABEL,PARTUUID,RA,MODEL,SERIAL,\\ + STATE,OWNER,GROUP,MODE,ALIGNMENT,MIN-IO,OPT-IO,PHY-SEC,LOG-SEC,ROTA,\\ + SCHED,RQ-SIZE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,WSAME,WWN,RAND,\\ + PKNAME,HCTL,TRAN,REV,VENDOR | jc --lsblk -p [ { "name": "sda", @@ -174,7 +182,10 @@ Examples: ... ] - $ lsblk -o +KNAME,FSTYPE,LABEL,UUID,PARTLABEL,PARTUUID,RA,MODEL,SERIAL,STATE,OWNER,GROUP,MODE,ALIGNMENT,MIN-IO,OPT-IO,PHY-SEC,LOG-SEC,ROTA,SCHED,RQ-SIZE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,WSAME,WWN,RAND,PKNAME,HCTL,TRAN,REV,VENDOR | jc --lsblk -p -r + $ lsblk -o +KNAME,FSTYPE,LABEL,UUID,PARTLABEL,PARTUUID,RA,MODEL,SERIAL,\\ + STATE,OWNER,GROUP,MODE,ALIGNMENT,MIN-IO,OPT-IO,PHY-SEC,LOG-SEC,ROTA,\\ + SCHED,RQ-SIZE,DISC-ALN,DISC-GRAN,DISC-MAX,DISC-ZERO,WSAME,WWN,RAND,\\ + PKNAME,HCTL,TRAN,REV,VENDOR | jc --lsblk -p -r [ { "name": "sda", @@ -273,8 +284,6 @@ class info(): description = '`lsblk` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['lsblk'] @@ -297,7 +306,8 @@ def _process(proc_data): for entry in proc_data: # boolean and integer changes bool_list = ['rm', 'ro', 'rota', 'disc_zero', 'rand'] - int_list = ['ra', 'alignment', 'min_io', 'opt_io', 'phy_sec', 'log_sec', 'rq_size', 'disc_aln'] + int_list = ['ra', 'alignment', 'min_io', 'opt_io', 'phy_sec', 'log_sec', + 'rq_size', 'disc_aln'] for key in entry: if key in bool_list: entry[key] = jc.utils.convert_to_bool(entry[key]) @@ -314,7 +324,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/lsmod.py b/jc/parsers/lsmod.py index cf20f683..26720e17 100644 --- a/jc/parsers/lsmod.py +++ b/jc/parsers/lsmod.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('lsmod', lsmod_command_output) + + or + import jc.parsers.lsmod result = jc.parsers.lsmod.parse(lsmod_command_output) @@ -130,8 +135,6 @@ class info(): description = '`lsmod` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['lsmod'] @@ -167,7 +170,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/lsof.py b/jc/parsers/lsof.py index 15bf2a5e..89c962d5 100644 --- a/jc/parsers/lsof.py +++ b/jc/parsers/lsof.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('lsof', lsof_command_output) + + or + import jc.parsers.lsof result = jc.parsers.lsof.parse(lsof_command_output) @@ -124,8 +129,6 @@ class info(): description = '`lsof` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['lsof'] @@ -161,7 +164,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/lsusb.py b/jc/parsers/lsusb.py index 966ab00d..87d2cc55 100644 --- a/jc/parsers/lsusb.py +++ b/jc/parsers/lsusb.py @@ -12,13 +12,19 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('lsusb', lsusb_command_output) + + or + import jc.parsers.lsusb result = jc.parsers.lsusb.parse(lsusb_command_output) Schema: - Note: object keynames are assigned directly from the lsusb output. - If there are duplicate names in a section, only the last one is converted. + Note: object keynames are assigned directly from the lsusb + output. If there are duplicate names in a section, only the + last one is converted. [ { @@ -808,7 +814,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/mount.py b/jc/parsers/mount.py index 85f427c3..25c7c48f 100644 --- a/jc/parsers/mount.py +++ b/jc/parsers/mount.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('mount', mount_command_output) + + or + import jc.parsers.mount result = jc.parsers.mount.parse(mount_command_output) @@ -79,8 +84,6 @@ class info(): description = '`mount` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['mount'] @@ -156,7 +159,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/netstat.py b/jc/parsers/netstat.py index 7a2e81c8..e3e9d3b5 100644 --- a/jc/parsers/netstat.py +++ b/jc/parsers/netstat.py @@ -2,7 +2,8 @@ Caveats: - Use of multiple `l` options is not supported on OSX (e.g. `netstat -rlll`) -- Use of the `A` option is not supported on OSX when using the `r` option (e.g. `netstat -rA`) +- Use of the `A` option is not supported on OSX when using the `r` option + (e.g. `netstat -rA`) Usage (cli): @@ -14,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('netstat', netstat_command_output) + + or + import jc.parsers.netstat result = jc.parsers.netstat.parse(netstat_command_output) @@ -358,8 +364,6 @@ class info(): description = '`netstat` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['netstat'] @@ -381,13 +385,14 @@ def _process(proc_data): """ for entry in proc_data: # integer and float conversions - int_list = ['recv_q', 'send_q', 'pid', 'refcnt', 'inode', 'unit', 'vendor', 'class', - 'osx_flags', 'subcla', 'pcbcount', 'rcvbuf', 'sndbuf', 'rxbytes', 'txbytes', - 'route_refs', 'use', 'mtu', 'mss', 'window', 'irtt', 'metric', 'ipkts', - 'ierrs', 'opkts', 'oerrs', 'coll', 'rx_ok', 'rx_err', 'rx_drp', 'rx_ovr', - 'tx_ok', 'tx_err', 'tx_drp', 'tx_ovr', 'idrop', 'ibytes', 'obytes', 'r_mbuf', - 's_mbuf', 'r_clus', 's_clus', 'r_hiwa', 's_hiwa', 'r_lowa', 's_lowa', 'r_bcnt', - 's_bcnt', 'r_bmax', 's_bmax', 'rexmit', 'ooorcv', '0_win'] + int_list = [ + 'recv_q', 'send_q', 'pid', 'refcnt', 'inode', 'unit', 'vendor', 'class', 'osx_flags', + 'subcla', 'pcbcount', 'rcvbuf', 'sndbuf', 'rxbytes', 'txbytes', 'route_refs', 'use', + 'mtu', 'mss', 'window', 'irtt', 'metric', 'ipkts', 'ierrs', 'opkts', 'oerrs', 'coll', + 'rx_ok', 'rx_err', 'rx_drp', 'rx_ovr', 'tx_ok', 'tx_err', 'tx_drp', 'tx_ovr', 'idrop', + 'ibytes', 'obytes', 'r_mbuf', 's_mbuf', 'r_clus', 's_clus', 'r_hiwa', 's_hiwa', + 'r_lowa', 's_lowa', 'r_bcnt', 's_bcnt', 'r_bmax', 's_bmax', 'rexmit', 'ooorcv', '0_win' + ] float_list = ['rexmt', 'persist', 'keep', '2msl', 'delack', 'rcvtime'] for key in entry: if key in int_list: @@ -416,7 +421,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/ntpq.py b/jc/parsers/ntpq.py index 06e7ddf9..f4eb7468 100644 --- a/jc/parsers/ntpq.py +++ b/jc/parsers/ntpq.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ntpq', ntpq_command_output) + + or + import jc.parsers.ntpq result = jc.parsers.ntpq.parse(ntpq_command_output) @@ -211,8 +216,6 @@ class info(): description = '`ntpq -p` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'freebsd'] magic_commands = ['ntpq'] @@ -257,7 +260,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/passwd.py b/jc/parsers/passwd.py index a6d42605..9be3b867 100644 --- a/jc/parsers/passwd.py +++ b/jc/parsers/passwd.py @@ -6,6 +6,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('passwd', passwd_file_output) + + or + import jc.parsers.passwd result = jc.parsers.passwd.parse(passwd_file_output) @@ -98,9 +103,6 @@ class info(): description = '`/etc/passwd` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'aix', 'freebsd'] @@ -135,7 +137,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/ping.py b/jc/parsers/ping.py index 10fcb54e..70e05465 100644 --- a/jc/parsers/ping.py +++ b/jc/parsers/ping.py @@ -4,7 +4,8 @@ Supports `ping` and `ping6` output. Usage (cli): - Note: Use the ping `-c` (count) option, otherwise data will not be piped to `jc`. + Note: Use the ping `-c` (count) option, otherwise data will not be + piped to `jc`. $ ping -c 3 1.2.3.4 | jc --ping @@ -14,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ping', ping_command_output) + + or + import jc.parsers.ping result = jc.parsers.ping.parse(ping_command_output) @@ -23,7 +29,7 @@ Schema: "source_ip": string, "destination_ip": string, "data_bytes": integer, - "pattern": string, # (null if not set) + "pattern": string, # null if not set "destination": string, "packets_transmitted": integer, "packets_received": integer, @@ -35,8 +41,8 @@ Schema: "round_trip_ms_stddev": float, "responses": [ { - "type": string, # 'reply', 'timeout', 'unparsable_line', etc. See `_error_type.type_map` for all options - "unparsed_line": string, # only if an 'unparsable_line' type + "type": string, # [0] + "unparsed_line": string, # [1] "timestamp": float, "bytes": integer, "response_ip": string, @@ -44,21 +50,26 @@ Schema: "ttl": integer, "time_ms": float, "duplicate": boolean, - "vr": integer, # hex value converted to decimal - "hl": integer, # hex value converted to decimal - "tos": integer, # hex value converted to decimal - "len": integer, # hex value converted to decimal - "id": integer, # hex value converted to decimal - "flg": integer, # hex value converted to decimal - "off": integer, # hex value converted to decimal - "pro": integer, # hex value converted to decimal - "cks": ingeger, # hex value converted to decimal + "vr": integer, # [2] + "hl": integer, # [2] + "tos": integer, # [2] + "len": integer, # [2] + "id": integer, # [2] + "flg": integer, # [2] + "off": integer, # [2] + "pro": integer, # [2] + "cks": ingeger, # [2] "src": string, "dst": string } ] } + [0] 'reply', 'timeout', 'unparsable_line', etc. See + `_error_type.type_map` for all options + [1] only if an 'unparsable_line' type + [2] hex value converted to decimal + Examples: $ ping -c 3 -p ff cnn.com | jc --ping -p @@ -162,8 +173,6 @@ class info(): description = '`ping` and `ping6` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['ping', 'ping6'] @@ -183,10 +192,14 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - int_list = ['data_bytes', 'packets_transmitted', 'packets_received', 'bytes', 'icmp_seq', 'ttl', - 'duplicates', 'vr', 'hl', 'tos', 'len', 'id', 'flg', 'off', 'pro', 'cks'] - float_list = ['packet_loss_percent', 'round_trip_ms_min', 'round_trip_ms_avg', 'round_trip_ms_max', - 'round_trip_ms_stddev', 'timestamp', 'time_ms'] + int_list = [ + 'data_bytes', 'packets_transmitted', 'packets_received', 'bytes', 'icmp_seq', 'ttl', + 'duplicates', 'vr', 'hl', 'tos', 'len', 'id', 'flg', 'off', 'pro', 'cks' + ] + float_list = [ + 'packet_loss_percent', 'round_trip_ms_min', 'round_trip_ms_avg', 'round_trip_ms_max', + 'round_trip_ms_stddev', 'timestamp', 'time_ms' + ] for key in proc_data: if key in int_list: @@ -609,7 +622,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/ping_s.py b/jc/parsers/ping_s.py index 3f3b7cdd..8536c3f5 100644 --- a/jc/parsers/ping_s.py +++ b/jc/parsers/ping_s.py @@ -8,59 +8,79 @@ Usage (cli): $ ping | jc --ping-s -> Note: When piping `jc` converted `ping` output to other processes it may appear the output is hanging due to the OS pipe buffers. This is because `ping` output is too small to quickly fill up the buffer. Use the `-u` option to unbuffer the `jc` output if you would like immediate output. See the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) for more information. +> Note: When piping `jc` converted `ping` output to other processes it may + appear the output is hanging due to the OS pipe buffers. This is because + `ping` output is too small to quickly fill up the buffer. Use the `-u` + option to unbuffer the `jc` output if you would like immediate output. + See the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) + for more information. Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('ping_s', ping_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.ping_s - result = jc.parsers.ping_s.parse(ping_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.ping_s.parse(ping_command_output.splitlines()) for item in result: # do something Schema: { - "type": string, # 'reply', 'timeout', 'summary', etc. See `_error_type.type_map` for all options. - "source_ip": string, - "destination_ip": string, - "sent_bytes": integer, - "pattern": string, # (null if not set) - "destination": string, - "timestamp": float, - "response_bytes": integer, - "response_ip": string, - "icmp_seq": integer, - "ttl": integer, - "time_ms": float, - "duplicate": boolean, - "packets_transmitted": integer, - "packets_received": integer, - "packet_loss_percent": float, - "duplicates": integer, - "round_trip_ms_min": float, - "round_trip_ms_avg": float, - "round_trip_ms_max": float, - "round_trip_ms_stddev": float, - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + "type": string, # [0] + "source_ip": string, + "destination_ip": string, + "sent_bytes": integer, + "pattern": string, # (null if not set) + "destination": string, + "timestamp": float, + "response_bytes": integer, + "response_ip": string, + "icmp_seq": integer, + "ttl": integer, + "time_ms": float, + "duplicate": boolean, + "packets_transmitted": integer, + "packets_received": integer, + "packet_loss_percent": float, + "duplicates": integer, + "round_trip_ms_min": float, + "round_trip_ms_avg": float, + "round_trip_ms_max": float, + "round_trip_ms_stddev": float, + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error - "error": string, # exists if "success" is false - "line": string # exists if "success" is false + "success": boolean, # false if error parsing + "error": string, # exists if "success" is false + "line": string # exists if "success" is false } } + [0] 'reply', 'timeout', 'summary', etc. See `_error_type.type_map` + for all options. + Examples: $ ping 1.1.1.1 | jc --ping-s - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"pattern":null,"response_bytes":64,"response_ip":"1.1.1.1","icmp_seq":0,"ttl":56,"time_ms":23.703} - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"pattern":null,"response_bytes":64,"response_ip":"1.1.1.1","icmp_seq":1,"ttl":56,"time_ms":22.862} - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"pattern":null,"response_bytes":64,"response_ip":"1.1.1.1","icmp_seq":2,"ttl":56,"time_ms":22.82} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"patte...} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"patte...} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":56,"patte...} ... $ ping 1.1.1.1 | jc --ping-s -r - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","pattern":null,"response_bytes":"64","response_ip":"1.1.1.1","icmp_seq":"0","ttl":"56","time_ms":"23.054"} - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","pattern":null,"response_bytes":"64","response_ip":"1.1.1.1","icmp_seq":"1","ttl":"56","time_ms":"24.739"} - {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","pattern":null,"response_bytes":"64","response_ip":"1.1.1.1","icmp_seq":"2","ttl":"56","time_ms":"23.232"} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","patte...} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","patte...} + {"type":"reply","destination_ip":"1.1.1.1","sent_bytes":"56","patte...} ... """ import string @@ -76,8 +96,6 @@ class info(): description = '`ping` and `ping6` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'freebsd'] streaming = True @@ -97,10 +115,14 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - int_list = ['sent_bytes', 'packets_transmitted', 'packets_received', 'response_bytes', 'icmp_seq', 'ttl', - 'duplicates', 'vr', 'hl', 'tos', 'len', 'id', 'flg', 'off', 'pro', 'cks'] - float_list = ['packet_loss_percent', 'round_trip_ms_min', 'round_trip_ms_avg', 'round_trip_ms_max', - 'round_trip_ms_stddev', 'timestamp', 'time_ms'] + int_list = [ + 'sent_bytes', 'packets_transmitted', 'packets_received', 'response_bytes', 'icmp_seq', + 'ttl', 'duplicates', 'vr', 'hl', 'tos', 'len', 'id', 'flg', 'off', 'pro', 'cks' + ] + float_list = [ + 'packet_loss_percent', 'round_trip_ms_min', 'round_trip_ms_avg', 'round_trip_ms_max', + 'round_trip_ms_stddev', 'timestamp', 'time_ms' + ] for key in proc_data: if key in int_list: @@ -453,8 +475,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/jc/parsers/pip_list.py b/jc/parsers/pip_list.py index c50b08e3..a429d468 100644 --- a/jc/parsers/pip_list.py +++ b/jc/parsers/pip_list.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('pip_list', pip_list_command_output) + + or + import jc.parsers.pip_list result = jc.parsers.pip_list.parse(pip_list_command_output) @@ -52,8 +57,6 @@ class info(): description = '`pip list` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] magic_commands = ['pip list', 'pip3 list'] @@ -84,7 +87,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/pip_show.py b/jc/parsers/pip_show.py index 926a4b2c..38142d6c 100644 --- a/jc/parsers/pip_show.py +++ b/jc/parsers/pip_show.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('pip_show', pip_show_command_output) + + or + import jc.parsers.pip_show result = jc.parsers.pip_show.parse(pip_show_command_output) @@ -69,8 +74,6 @@ class info(): description = '`pip show` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] magic_commands = ['pip show', 'pip3 show'] @@ -101,7 +104,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/ps.py b/jc/parsers/ps.py index 655c7ed3..99a68c6d 100644 --- a/jc/parsers/ps.py +++ b/jc/parsers/ps.py @@ -14,6 +14,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ps', ps_command_output) + + or + import jc.parsers.ps result = jc.parsers.ps.parse(ps_command_output) @@ -53,7 +58,7 @@ Examples: "stime": "Nov01", "tty": null, "time": "00:00:11", - "cmd": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "cmd": "/usr/lib/systemd/systemd --switched-root --system --dese..." }, { "uid": "root", @@ -88,7 +93,7 @@ Examples: "stime": "Nov01", "tty": "?", "time": "00:00:11", - "cmd": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "cmd": "/usr/lib/systemd/systemd --switched-root --system --dese..." }, { "uid": "root", @@ -126,7 +131,7 @@ Examples: "stat": "Ss", "start": "Nov09", "time": "0:08", - "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "command": "/usr/lib/systemd/systemd --switched-root --system --..." }, { "user": "root", @@ -170,7 +175,7 @@ Examples: "stat": "Ss", "start": "Nov09", "time": "0:08", - "command": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22" + "command": "/usr/lib/systemd/systemd --switched-root --system --..." }, { "user": "root", @@ -211,8 +216,6 @@ class info(): description = '`ps` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] magic_commands = ['ps'] @@ -269,7 +272,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/route.py b/jc/parsers/route.py index 0ad83826..3a5178bc 100644 --- a/jc/parsers/route.py +++ b/jc/parsers/route.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('route', route_command_output) + + or + import jc.parsers.route result = jc.parsers.route.parse(route_command_output) @@ -113,8 +118,6 @@ class info(): description = '`route` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['route'] @@ -173,7 +176,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: @@ -192,7 +195,9 @@ def parse(data, raw=False, quiet=False): # fixup header row for ipv6 if ' Next Hop ' in cleandata[0]: cleandata[0] = cleandata[0].replace(' If', ' Iface') - cleandata[0] = cleandata[0].replace(' Next Hop ', ' Next_Hop ').replace(' Flag ', ' Flags ').replace(' Met ', ' Metric ') + cleandata[0] = cleandata[0].replace(' Next Hop ', ' Next_Hop ')\ + .replace(' Flag ', ' Flags ')\ + .replace(' Met ', ' Metric ') cleandata[0] = cleandata[0].lower() raw_output = jc.parsers.universal.simple_table_parse(cleandata) diff --git a/jc/parsers/rpm_qi.py b/jc/parsers/rpm_qi.py index b5d526dd..a0e6e689 100644 --- a/jc/parsers/rpm_qi.py +++ b/jc/parsers/rpm_qi.py @@ -2,9 +2,11 @@ Works with `rpm -qi [package]` or `rpm -qia`. -The `..._epoch` calculated timestamp fields are naive (i.e. based on the local time of the system the parser is run on) +The `..._epoch` calculated timestamp fields are naive. (i.e. based on the +local time of the system the parser is run on) -The `..._epoch_utc` calculated timestamp fields are timezone-aware and is only available if the timezone field is UTC. +The `..._epoch_utc` calculated timestamp fields are timezone-aware and is +only available if the timezone field is UTC. Usage (cli): @@ -16,6 +18,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('rpm_qi', rpm_qi_command_output) + + or + import jc.parsers.rpm_qi result = jc.parsers.rpm_qi.parse(rpm_qi_command_output) @@ -29,16 +36,16 @@ Schema: "release": string, "architecture": string, "install_date": string, - "install_date_epoch": integer, # naive timestamp - "install_date_epoch_utc": integer, # Aware timestamp if timezone is UTC + "install_date_epoch": integer, # [0] + "install_date_epoch_utc": integer, # [1] "group": string, "size": integer, "license": string, "signature": string, "source_rpm": string, "build_date": string, - "build_epoch": integer, # naive timestamp - "build_epoch_utc": integer, # Aware timestamp if timezone is UTC + "build_epoch": integer, # [0] + "build_epoch_utc": integer, # [1] "build_host": string, "relocations": string, "packager": string, @@ -49,6 +56,9 @@ Schema: } ] + [0] naive timestamp + [1] Aware timestamp if timezone is UTC + Examples: $ rpm -qia | jc --rpm-qi -p @@ -63,7 +73,7 @@ Examples: "group": "Development/Tools", "size": 1160660, "license": "GPLv2+", - "signature": "RSA/SHA256, Thu 22 Aug 2019 02:34:59 PM PDT, Key ID 24c6a8a7f4a80eb5", + "signature": "RSA/SHA256, Thu 22 Aug 2019 02:34:59 PM PDT, Key ...", "source_rpm": "make-3.82-24.el7.src.rpm", "build_date": "Thu 08 Aug 2019 05:47:25 PM PDT", "build_host": "x86-01.bsys.centos.org", @@ -71,8 +81,8 @@ Examples: "packager": "CentOS BuildSystem ", "vendor": "CentOS", "url": "http://www.gnu.org/software/make/", - "summary": "A GNU tool which simplifies the build process for users", - "description": "A GNU tool for controlling the generation of executables and other...", + "summary": "A GNU tool which simplifies the build process for ...", + "description": "A GNU tool for controlling the generation of ex...", "build_epoch": 1565311645, "build_epoch_utc": null, "install_date_epoch": 1571242902, @@ -87,7 +97,7 @@ Examples: "group": "System Environment/Base", "size": 503608, "license": "GPLv2+", - "signature": "RSA/SHA256, Mon 12 Nov 2018 07:17:49 AM PST, Key ID 24c6a8a7f4a80eb5", + "signature": "RSA/SHA256, Mon 12 Nov 2018 07:17:49 AM PST, Key ...", "source_rpm": "kbd-1.15.5-15.el7.src.rpm", "build_date": "Tue 30 Oct 2018 03:40:00 PM PDT", "build_host": "x86-01.bsys.centos.org", @@ -96,7 +106,7 @@ Examples: "vendor": "CentOS", "url": "http://ftp.altlinux.org/pub/people/legion/kbd", "summary": "Legacy data for kbd package", - "description": "The kbd-legacy package contains original keymaps for kbd package...", + "description": "The kbd-legacy package contains original keymap...", "build_epoch": 1540939200, "build_epoch_utc": null, "install_date_epoch": 1565891588, @@ -117,7 +127,7 @@ Examples: "group": "Development/Tools", "size": "1160660", "license": "GPLv2+", - "signature": "RSA/SHA256, Thu 22 Aug 2019 02:34:59 PM PDT, Key ID 24c6a8a7f4a80eb5", + "signature": "RSA/SHA256, Thu 22 Aug 2019 02:34:59 PM PDT, Key ...", "source_rpm": "make-3.82-24.el7.src.rpm", "build_date": "Thu 08 Aug 2019 05:47:25 PM PDT", "build_host": "x86-01.bsys.centos.org", @@ -125,8 +135,8 @@ Examples: "packager": "CentOS BuildSystem ", "vendor": "CentOS", "url": "http://www.gnu.org/software/make/", - "summary": "A GNU tool which simplifies the build process for users", - "description": "A GNU tool for controlling the generation of executables and other..." + "summary": "A GNU tool which simplifies the build process for...", + "description": "A GNU tool for controlling the generation of exe..." }, { "name": "kbd-legacy", @@ -137,7 +147,7 @@ Examples: "group": "System Environment/Base", "size": "503608", "license": "GPLv2+", - "signature": "RSA/SHA256, Mon 12 Nov 2018 07:17:49 AM PST, Key ID 24c6a8a7f4a80eb5", + "signature": "RSA/SHA256, Mon 12 Nov 2018 07:17:49 AM PST, Key ...", "source_rpm": "kbd-1.15.5-15.el7.src.rpm", "build_date": "Tue 30 Oct 2018 03:40:00 PM PDT", "build_host": "x86-01.bsys.centos.org", @@ -146,7 +156,7 @@ Examples: "vendor": "CentOS", "url": "http://ftp.altlinux.org/pub/people/legion/kbd", "summary": "Legacy data for kbd package", - "description": "The kbd-legacy package contains original keymaps for kbd package..." + "description": "The kbd-legacy package contains original keymaps..." }, ... ] @@ -160,9 +170,6 @@ class info(): description = '`rpm -qi` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['rpm -qi', 'rpm -qia', 'rpm -qai'] @@ -209,7 +216,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/sfdisk.py b/jc/parsers/sfdisk.py index c368a9d4..d5f4ac88 100644 --- a/jc/parsers/sfdisk.py +++ b/jc/parsers/sfdisk.py @@ -19,6 +19,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('sfdisk', sfdisk_command_output) + + or + import jc.parsers.sfdisk result = jc.parsers.sfdisk.parse(sfdisk_command_output) @@ -50,7 +55,7 @@ Schema: "boot": boolean, "start": integer, "end": integer, - "size": string, # Note: will be integer when using deprecated -d sfdisk option + "size": string, # [0] "cyls": integer, "mib": integer, "blocks": integer, @@ -63,6 +68,8 @@ Schema: } ] + [0] will be integer when using deprecated -d sfdisk option + Examples: # sfdisk -l | jc --sfdisk -p @@ -72,7 +79,7 @@ Examples: "cylinders": 2610, "heads": 255, "sectors_per_track": 63, - "units": "cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0", + "units": "cylinders of 8225280 bytes, blocks of 1024 bytes, ...", "partitions": [ { "device": "/dev/sda1", @@ -137,7 +144,7 @@ Examples: "cylinders": "2610", "heads": "255", "sectors_per_track": "63", - "units": "cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0", + "units": "cylinders of 8225280 bytes, blocks of 1024 bytes, co...", "partitions": [ { "device": "/dev/sda1", @@ -224,9 +231,11 @@ def _process(proc_data): List of Dictionaries. Structured to conform to the schema. """ - int_list = ['cylinders', 'heads', 'sectors_per_track', 'start', 'end', 'cyls', 'mib', - 'blocks', 'sectors', 'bytes', 'logical_sector_size', 'physical_sector_size', - 'min_io_size', 'optimal_io_size', 'free_bytes', 'free_sectors'] + int_list = [ + 'cylinders', 'heads', 'sectors_per_track', 'start', 'end', 'cyls', 'mib', 'blocks', + 'sectors', 'bytes', 'logical_sector_size', 'physical_sector_size', 'min_io_size', + 'optimal_io_size', 'free_bytes', 'free_sectors' + ] bool_list = ['boot'] for entry in proc_data: @@ -258,7 +267,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/shadow.py b/jc/parsers/shadow.py index db51455c..00427a37 100644 --- a/jc/parsers/shadow.py +++ b/jc/parsers/shadow.py @@ -6,6 +6,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('shadow', shadow_file_output) + + or + import jc.parsers.shadow result = jc.parsers.shadow.parse(shadow_file_output) @@ -105,9 +110,6 @@ class info(): description = '`/etc/shadow` file parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'aix', 'freebsd'] @@ -142,7 +144,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/ss.py b/jc/parsers/ss.py index 5a4e2c72..0416f097 100644 --- a/jc/parsers/ss.py +++ b/jc/parsers/ss.py @@ -1,6 +1,7 @@ """jc - JSON CLI output utility `ss` command output parser -Extended information options like -e and -p are not supported and may cause parsing irregularities. +Extended information options like -e and -p are not supported and may cause +parsing irregularities. Usage (cli): @@ -12,12 +13,18 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ss', ss_command_output) + + or + import jc.parsers.ss result = jc.parsers.ss.parse(ss_command_output) Schema: - Information from https://www.cyberciti.biz/files/ss.html used to define field names + Information from https://www.cyberciti.biz/files/ss.html used to define + field names [ { @@ -283,8 +290,6 @@ class info(): description = '`ss` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['ss'] @@ -330,7 +335,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/stat.py b/jc/parsers/stat.py index 4fd69050..187c018e 100644 --- a/jc/parsers/stat.py +++ b/jc/parsers/stat.py @@ -1,8 +1,10 @@ """jc - JSON CLI output utility `stat` command output parser -The `xxx_epoch` calculated timestamp fields are naive (i.e. based on the local time of the system the parser is run on) +The `xxx_epoch` calculated timestamp fields are naive. (i.e. based on the +local time of the system the parser is run on) -The `xxx_epoch_utc` calculated timestamp fields are timezone-aware and are only available if the timezone field is UTC. +The `xxx_epoch_utc` calculated timestamp fields are timezone-aware and are +only available if the timezone field is UTC. Usage (cli): @@ -14,6 +16,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('stat', stat_command_output) + + or + import jc.parsers.stat result = jc.parsers.stat.parse(stat_command_output) @@ -173,8 +180,6 @@ class info(): description = '`stat` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['stat'] @@ -195,8 +200,8 @@ def _process(proc_data): List of Dictionaries. Structured data to conform to the schema. """ for entry in proc_data: - int_list = ['size', 'blocks', 'io_blocks', 'inode', 'links', 'uid', 'gid', 'unix_device', - 'rdev', 'block_size'] + int_list = ['size', 'blocks', 'io_blocks', 'inode', 'links', 'uid', 'gid', + 'unix_device', 'rdev', 'block_size'] for key in entry: if key in int_list: entry[key] = jc.utils.convert_to_int(entry[key]) @@ -222,7 +227,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/stat_s.py b/jc/parsers/stat_s.py index 55155991..ce7f6237 100644 --- a/jc/parsers/stat_s.py +++ b/jc/parsers/stat_s.py @@ -2,9 +2,11 @@ > This streaming parser outputs JSON Lines -The `xxx_epoch` calculated timestamp fields are naive (i.e. based on the local time of the system the parser is run on). +The `xxx_epoch` calculated timestamp fields are naive. (i.e. based on the +local time of the system the parser is run on). -The `xxx_epoch_utc` calculated timestamp fields are timezone-aware and are only available if the timezone field is UTC. +The `xxx_epoch_utc` calculated timestamp fields are timezone-aware and are +only available if the timezone field is UTC. Usage (cli): @@ -12,8 +14,17 @@ Usage (cli): Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('stat_s', stat_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.stat_s - result = jc.parsers.stat_s.parse(stat_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.stat_s.parse(stat_command_output.splitlines()) for item in result: # do something @@ -51,9 +62,12 @@ Schema: "rdev": integer, "block_size": integer, "unix_flags": string, - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error + "success": boolean, # false if error parsing "error": string, # exists if "success" is false "line": string # exists if "success" is false } @@ -62,10 +76,10 @@ Schema: Examples: $ stat | jc --stat-s - {"file":"(stdin)","unix_device":1027739696,"inode":1155,"flags":"crw--w----","links":1,"user":"kbrazil","group":"tty","rdev":268435456,"size":0,"access_time":"Jan 4 15:27:44 2022","modify_time":"Jan 4 15:27:44 2022","change_time":"Jan 4 15:27:44 2022","birth_time":"Dec 31 16:00:00 1969","block_size":131072,"blocks":0,"unix_flags":"0","access_time_epoch":1641338864,"access_time_epoch_utc":null,"modify_time_epoch":1641338864,"modify_time_epoch_utc":null,"change_time_epoch":1641338864,"change_time_epoch_utc":null,"birth_time_epoch":null,"birth_time_epoch_utc":null} + {"file":"(stdin)","unix_device":1027739696,"inode":1155,"flags":"cr...} $ stat | jc --stat-s -r - {"file":"(stdin)","unix_device":"1027739696","inode":"1155","flags":"crw--w----","links":"1","user":"kbrazil","group":"tty","rdev":"268435456","size":"0","access_time":"Jan 4 15:28:08 2022","modify_time":"Jan 4 15:28:08 2022","change_time":"Jan 4 15:28:08 2022","birth_time":"Dec 31 16:00:00 1969","block_size":"131072","blocks":"0","unix_flags":"0"} + {"file":"(stdin)","unix_device":"1027739696","inode":"1155","flag...} """ import shlex import jc.utils @@ -98,8 +112,8 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - int_list = ['size', 'blocks', 'io_blocks', 'inode', 'links', 'uid', 'gid', 'unix_device', - 'rdev', 'block_size'] + int_list = ['size', 'blocks', 'io_blocks', 'inode', 'links', 'uid', 'gid', + 'unix_device', 'rdev', 'block_size'] for key in proc_data: if key in int_list: proc_data[key] = jc.utils.convert_to_int(proc_data[key]) @@ -122,8 +136,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/jc/parsers/sysctl.py b/jc/parsers/sysctl.py index 0c3f63bd..388c0fb9 100644 --- a/jc/parsers/sysctl.py +++ b/jc/parsers/sysctl.py @@ -1,6 +1,9 @@ """jc - JSON CLI output utility `sysctl -a` command output parser -Note: Since `sysctl` output is not easily parsable only a very simple key/value object will be output. An attempt is made to convert obvious integers and floats. If no conversion is desired, use the `-r` command-line argument or the `raw=True` argument in `parse()`. +Note: Since `sysctl` output is not easily parsable only a very simple + key/value object will be output. An attempt is made to convert obvious + integers and floats. If no conversion is desired, use the `-r` + command-line argument or the `raw=True` argument in `parse()`. Usage (cli): @@ -12,13 +15,18 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('sysctl', sysctl_command_output) + + or + import jc.parsers.sysctl result = jc.parsers.sysctl.parse(sysctl_command_output) Schema: { - "key1": string/integer/float, # best guess based on value + "key1": string/integer/float, # best guess based on value "key2": string/integer/float, "key3": string/integer/float } @@ -58,9 +66,6 @@ class info(): description = '`sysctl` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['sysctl'] @@ -98,7 +103,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/systemctl.py b/jc/parsers/systemctl.py index 8a9956ad..58cbe8d0 100644 --- a/jc/parsers/systemctl.py +++ b/jc/parsers/systemctl.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('systemctl', systemctl_command_output) + + or + import jc.parsers.systemctl result = jc.parsers.systemctl.parse(systemctl_command_output) @@ -34,14 +39,14 @@ Examples: "load": "loaded", "active": "active", "sub": "waiting", - "description": "Arbitrary Executable File Formats File System Automount Point" + "description": "Arbitrary Executable File Formats File System ..." }, { "unit": "dev-block-8:2.device", "load": "loaded", "active": "active", "sub": "plugged", - "description": "LVM PV 3klkIj-w1qk-DkJi-0XBJ-y3o7-i2Ac-vHqWBM on /dev/sda2 2" + "description": "LVM PV 3klkIj-w1qk-DkJi-0XBJ-y3o7-i2Ac-vHqWBM o..." }, { "unit": "dev-cdrom.device", @@ -62,8 +67,6 @@ class info(): description = '`systemctl` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['systemctl'] @@ -94,7 +97,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/systemctl_lj.py b/jc/parsers/systemctl_lj.py index 98cbda5d..f078b20b 100644 --- a/jc/parsers/systemctl_lj.py +++ b/jc/parsers/systemctl_lj.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('systemctl_lj', systemctl_lj_command_output) + + or + import jc.parsers.systemctl_lj result = jc.parsers.systemctl_lj.parse(systemctl_lj_command_output) @@ -79,8 +84,6 @@ class info(): description = '`systemctl list-jobs` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['systemctl list-jobs'] @@ -116,7 +119,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/systemctl_ls.py b/jc/parsers/systemctl_ls.py index 42fbc1aa..0b7302c9 100644 --- a/jc/parsers/systemctl_ls.py +++ b/jc/parsers/systemctl_ls.py @@ -1,4 +1,5 @@ -"""jc - JSON CLI output utility `systemctl list-sockets` command output parser +"""jc - JSON CLI output utility `systemctl list-sockets` command output +parser Usage (cli): @@ -10,6 +11,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('systemctl_ls', systemctl_ls_command_output) + + or + import jc.parsers.systemctl_ls result = jc.parsers.systemctl_ls.parse(systemctl_ls_command_output) @@ -54,8 +60,6 @@ class info(): description = '`systemctl list-sockets` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['systemctl list-sockets'] @@ -86,7 +90,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/systemctl_luf.py b/jc/parsers/systemctl_luf.py index 69d34e55..9eb91035 100644 --- a/jc/parsers/systemctl_luf.py +++ b/jc/parsers/systemctl_luf.py @@ -1,4 +1,5 @@ -"""jc - JSON CLI output utility `systemctl list-unit-files` command output parser +"""jc - JSON CLI output utility `systemctl list-unit-files` command output +parser Usage (cli): @@ -10,6 +11,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('systemctl_luf', systemctl_luf_command_output) + + or + import jc.parsers.systemctl_luf result = jc.parsers.systemctl_luf.parse(systemctl_luf_command_output) @@ -50,8 +56,6 @@ class info(): description = '`systemctl list-unit-files` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['systemctl list-unit-files'] @@ -82,7 +86,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/systeminfo.py b/jc/parsers/systeminfo.py index f6bc2b3c..3cc03a2f 100644 --- a/jc/parsers/systeminfo.py +++ b/jc/parsers/systeminfo.py @@ -2,9 +2,13 @@ Blank or missing elements are set to `null`. -The `original_install_date_epoch` and `system_boot_time_epoch` calculated timestamp fields are naive (i.e. based on the local time of the system the parser is run on) +The `original_install_date_epoch` and `system_boot_time_epoch` calculated +timestamp fields are naive. (i.e. based on the local time of the system the +parser is run on) -The `original_install_date_epoch_utc` and `system_boot_time_epoch_utc` calculated timestamp fields are timezone-aware and are only available if the timezone field is UTC. +The `original_install_date_epoch_utc` and `system_boot_time_epoch_utc` +calculated timestamp fields are timezone-aware and are only available if +the timezone field is UTC. Usage (cli): @@ -12,6 +16,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('systeminfo', systeminfo_command_output) + + or + import jc.parsers.systeminfo result = jc.parsers.systeminfo.parse(systeminfo_command_output) @@ -28,11 +37,11 @@ Schema: "registered_organization": string, "product_id": string, "original_install_date": string, - "original_install_date_epoch": integer, # naive timestamp - "original_install_date_epoch_utc": integer, # timezone-aware timestamp + "original_install_date_epoch": integer, # [0] + "original_install_date_epoch_utc": integer, # [1] "system_boot_time": string, - "system_boot_time_epoch": integer, # naive timestamp - "system_boot_time_epoch_utc": integer, # timezone-aware timestamp + "system_boot_time_epoch": integer, # [0] + "system_boot_time_epoch_utc": integer, # [1] "system_manufacturer": string, "system_model": string, "system_type": string, @@ -77,6 +86,9 @@ Schema: } } + [0] naive timestamp + [1] timezone-aware timestamp + Examples: $ systeminfo | jc --systeminfo -p @@ -209,9 +221,6 @@ class info: description = "`systeminfo` command parser" author = "Jon Smith" author_email = "jon@rebelliondefense.com" - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ["win32"] magic_commands = ["systeminfo"] @@ -229,11 +238,12 @@ def _process(proc_data): Returns: - Dictionary. Some keys are optional. Example: a system without hyper-v capabilities - will not have a 'hyperv_requirements' key, and a system already running hyper-v - will have an empty "hyperv_requirements" object. + Dictionary. Structured data to conform to the schema. - Structured data to conform to the schema. + Some keys are optional. For example, a system without + hyper-v capabilities will not have a 'hyperv_requirements' key, and + a system already running hyper-v will have an empty + "hyperv_requirements" object. """ # convert empty strings to None/null for item in proc_data: @@ -296,7 +306,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/time.py b/jc/parsers/time.py index f5e594e1..c09b5c45 100644 --- a/jc/parsers/time.py +++ b/jc/parsers/time.py @@ -1,17 +1,26 @@ """jc - JSON CLI output utility `/usr/bin/time` command output parser -Output from `/usr/bin/time` is sent to `STDERR`, so the `-o` option can be used to redirect the output to a file that can be read by `jc`. +Output from `/usr/bin/time` is sent to `STDERR`, so the `-o` option can be +used to redirect the output to a file that can be read by `jc`. -Alternatively, the output from `/usr/bin/time` can be redirected to `STDOUT` so `jc` can receive it. +Alternatively, the output from `/usr/bin/time` can be redirected to `STDOUT` +so `jc` can receive it. -Note: `/usr/bin/time` is similar but different from the Bash builtin `time` command. +Note: `/usr/bin/time` is similar but different from the Bash builtin + `time` command. Usage (cli): - $ /usr/bin/time -o timefile.out sleep 2.5; cat timefile.out | jc --time -p + $ /usr/bin/time -o timefile.out sleep 2; cat timefile.out | \\ + jc --time -p Usage (module): + import jc + result = jc.parse('time', time_command_output) + + or + import jc.parsers.time result = jc.parsers.time.parse(time_command_output) @@ -36,8 +45,8 @@ Schema: "average_unshared_stack_size": integer, "average_shared_memory_size": integer, "maximum_resident_set_size": integer, - "block_input_operations": integer, # aka File system inputs - "block_output_operations": integer, # aka File system outputs + "block_input_operations": integer, # [0] + "block_output_operations": integer, # [1] "major_pagefaults": integer, "minor_pagefaults": integer, "swaps": integer, @@ -57,15 +66,19 @@ Schema: "exit_status": integer } + [0] aka File system inputs + [1] aka File system outputs + Examples: - $ /usr/bin/time --verbose -o timefile.out sleep 2.5; cat timefile.out | jc --time -p + $ /usr/bin/time --verbose -o timefile.out sleep 2; cat timefile.out | \\ + jc --time -p { - "command_being_timed": "sleep 2.5", + "command_being_timed": "sleep 2", "user_time": 0.0, "system_time": 0.0, "cpu_percent": 0, - "elapsed_time": "0:02.50", + "elapsed_time": "0:02.00", "average_shared_text_size": 0, "average_unshared_data_size": 0, "average_stack_size": 0, @@ -91,13 +104,14 @@ Examples: "elapsed_time_total_seconds": 2.5 } - $ /usr/bin/time --verbose -o timefile.out sleep 2.5; cat timefile.out | jc --time -p -r + $ /usr/bin/time --verbose -o timefile.out sleep 2; cat timefile.out | \\ + jc --time -p -r { - "command_being_timed": "\"sleep 2.5\"", + "command_being_timed": "\"sleep 2\"", "user_time": "0.00", "system_time": "0.00", "cpu_percent": "0", - "elapsed_time": "0:02.50", + "elapsed_time": "0:02.00", "average_shared_text_size": "0", "average_unshared_data_size": "0", "average_stack_size": "0", @@ -127,9 +141,6 @@ class info(): description = '`/usr/bin/time` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] @@ -168,12 +179,15 @@ def _process(proc_data): (proc_data['elapsed_time_centiseconds'] / 100) # convert ints and floats - int_list = ['cpu_percent', 'average_shared_text_size', 'average_unshared_data_size', 'average_unshared_stack_size', - 'average_shared_memory_size', 'maximum_resident_set_size', 'block_input_operations', - 'block_output_operations', 'major_pagefaults', 'minor_pagefaults', 'swaps', 'page_reclaims', - 'page_faults', 'messages_sent', 'messages_received', 'signals_received', 'voluntary_context_switches', - 'involuntary_context_switches', 'average_stack_size', 'average_total_size', 'average_resident_set_size', - 'signals_delivered', 'page_size', 'exit_status'] + int_list = [ + 'cpu_percent', 'average_shared_text_size', 'average_unshared_data_size', + 'average_unshared_stack_size', 'average_shared_memory_size', 'maximum_resident_set_size', + 'block_input_operations', 'block_output_operations', 'major_pagefaults', 'minor_pagefaults', + 'swaps', 'page_reclaims', 'page_faults', 'messages_sent', 'messages_received', + 'signals_received', 'voluntary_context_switches', 'involuntary_context_switches', + 'average_stack_size', 'average_total_size', 'average_resident_set_size', + 'signals_delivered', 'page_size', 'exit_status' + ] float_list = ['real_time', 'user_time', 'system_time'] for key in proc_data: if key in int_list: @@ -191,7 +205,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/timedatectl.py b/jc/parsers/timedatectl.py index 92f0d068..0d58c527 100644 --- a/jc/parsers/timedatectl.py +++ b/jc/parsers/timedatectl.py @@ -1,6 +1,7 @@ """jc - JSON CLI output utility `timedatectl` command output parser -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the `universal_time` field is available. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the `universal_time` field is available. Usage (cli): @@ -12,6 +13,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('timedatectl', timedatectl_command_output) + + or + import jc.parsers.timedatectl result = jc.parsers.timedatectl.parse(timedatectl_command_output) @@ -20,7 +26,7 @@ Schema: { "local_time": string, "universal_time": string, - "epoch_utc": integer, # timezone-aware timestamp + "epoch_utc": integer, # timezone-aware "rtc_time": string, "time_zone": string, "ntp_enabled": boolean, @@ -67,9 +73,6 @@ class info(): description = '`timedatectl status` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['timedatectl', 'timedatectl status'] @@ -109,7 +112,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/tracepath.py b/jc/parsers/tracepath.py index c600e68a..5a4458ed 100644 --- a/jc/parsers/tracepath.py +++ b/jc/parsers/tracepath.py @@ -12,6 +12,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('tracepath', tracepath_command_output) + + or + import jc.parsers.tracepath result = jc.parsers.tracepath.parse(tracepath_command_output) @@ -136,8 +141,6 @@ class info(): description = '`tracepath` and `tracepath6` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['tracepath', 'tracepath6'] @@ -185,7 +188,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/traceroute.py b/jc/parsers/traceroute.py index e78a066b..5df5e70a 100644 --- a/jc/parsers/traceroute.py +++ b/jc/parsers/traceroute.py @@ -2,9 +2,12 @@ Supports `traceroute` and `traceroute6` output. -Note: On some operating systems you will need to redirect `STDERR` to `STDOUT` for destination info since the header line is sent to `STDERR`. A warning message will be printed to `STDERR` if the header row is not found. +Note: On some operating systems you will need to redirect `STDERR` to + `STDOUT` for destination info since the header line is sent to + `STDERR`. A warning message will be printed to `STDERR` if the + header row is not found. -e.g. `$ traceroute 8.8.8.8 2>&1 | jc --traceroute` + e.g. `$ traceroute 8.8.8.8 2>&1 | jc --traceroute` Usage (cli): @@ -16,6 +19,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('traceroute', traceroute_command_output) + + or + import jc.parsers.traceroute result = jc.parsers.traceroute.parse(traceroute_command_output) @@ -124,8 +132,6 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Using the trparse library by Luis Benitez at https://github.com/lbenitez000/trparse' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['traceroute', 'traceroute6'] @@ -361,7 +367,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/ufw.py b/jc/parsers/ufw.py index 7e9b4f97..58bedd57 100644 --- a/jc/parsers/ufw.py +++ b/jc/parsers/ufw.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ufw', ufw_command_output) + + or + import jc.parsers.ufw result = jc.parsers.ufw.parse(ufw_command_output) @@ -40,7 +45,7 @@ Schema: "end": integer } ], - "to_service": string, # null if any to ports or port_ranges are set + "to_service": string, # [0] "from_ip": string, "from_ip_prefix": integer, "from_interface": string, @@ -54,12 +59,15 @@ Schema: "end": integer } ], - "from_service": string, # null if any from ports or port_ranges are set + "from_service": string, # [1] "comment": string # null if no comment } ] } + [0] null if any 'to' ports or port_ranges are set + [1] null if any 'from' ports or port_ranges are set + Examples: $ ufw status verbose | jc --ufw -p @@ -387,7 +395,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/ufw_appinfo.py b/jc/parsers/ufw_appinfo.py index 522a53ca..ec3e78ef 100644 --- a/jc/parsers/ufw_appinfo.py +++ b/jc/parsers/ufw_appinfo.py @@ -1,8 +1,12 @@ -"""jc - JSON CLI output utility `ufw app info [application]` command output parser +"""jc - JSON CLI output utility `ufw app info [application]` command +output parser -Supports individual apps via `ufw app info [application]` and all apps list via `ufw app info all`. +Supports individual apps via `ufw app info [application]` and all apps list +via `ufw app info all`. -Because `ufw` application definitions allow overlapping ports and port ranges, this parser preserves that behavior, but also provides `normalized` lists and ranges that remove duplicate ports and merge overlapping ranges. +Because `ufw` application definitions allow overlapping ports and port +ranges, this parser preserves that behavior, but also provides `normalized` +lists and ranges that remove duplicate ports and merge overlapping ranges. Usage (cli): @@ -14,6 +18,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('ufw_appinfo', ufw_appinfo_command_output) + + or + import jc.parsers.ufw_appinfo result = jc.parsers.ufw_appinfo.parse(ufw_appinfo_command_output) @@ -29,7 +38,7 @@ Schema: ], "tcp_ranges": [ { - "start": integer, # 'any' is converted to start/end: 0/65535 + "start": integer, # [0] "end": integer } ], @@ -38,31 +47,35 @@ Schema: ], "udp_ranges": [ { - "start": integer, # 'any' is converted to start/end: 0/65535 + "start": integer, # [0] "end": integer } ], "normalized_tcp_list": [ - integers # duplicates and overlapping are removed + integers # [1] ], "normalized_tcp_ranges": [ { - "start": integer, # 'any' is converted to start/end: 0/65535 - "end": integers # overlapping are merged + "start": integer, # [0] + "end": integers # [2] } ], "normalized_udp_list": [ - integers # duplicates and overlapping are removed + integers # [1] ], "normalized_udp_ranges": [ { - "start": integer, # 'any' is converted to start/end: 0/65535 - "end": integers # overlapping are merged + "start": integer, # [0] + "end": integers # [2] } ] } ] + [0] 'any' is converted to start/end: 0/65535 + [1] duplicates and overlapping are removed + [2] overlapping are merged + Examples: $ ufw app info MSN | jc --ufw-appinfo -p @@ -258,7 +271,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/uname.py b/jc/parsers/uname.py index 96ae4ba7..3f8538a6 100644 --- a/jc/parsers/uname.py +++ b/jc/parsers/uname.py @@ -12,6 +12,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('uname', uname_command_output) + + or + import jc.parsers.uname result = jc.parsers.uname.parse(uname_command_output) @@ -52,8 +57,6 @@ class info(): description = '`uname -a` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'freebsd'] magic_commands = ['uname'] @@ -84,7 +87,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: @@ -125,7 +128,7 @@ def parse(data, raw=False, quiet=False): fixup.insert(-1, 'unknown') fixup.insert(-1, 'unknown') data = ' '.join(fixup) - + parsed_line = data.split(maxsplit=3) if len(parsed_line) < 3: diff --git a/jc/parsers/universal.py b/jc/parsers/universal.py index c7137daf..9d1bfa64 100644 --- a/jc/parsers/universal.py +++ b/jc/parsers/universal.py @@ -6,24 +6,25 @@ import string def simple_table_parse(data): """ - Parse simple tables. The last column may contain data with spaces - - code adapted from Conor Heine at: - https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 + Parse simple tables. The last column may contain data with spaces. Parameters: - data: (list) Text data to parse that has been split into lines via .splitlines(). - Item 0 must be the header row. Any spaces in header names should be - changed to underscore '_'. You should also ensure headers are - lowercase by using .lower(). + data: (list) Text data to parse that has been split into lines + via .splitlines(). Item 0 must be the header row. + Any spaces in header names should be changed to + underscore '_'. You should also ensure headers are + lowercase by using .lower(). - Also, ensure there are no blank lines (list items) in the data. + Also, ensure there are no blank lines (list items) + in the data. Returns: - List of Dictionaries raw structured data + List of Dictionaries """ + # code adapted from Conor Heine at: + # https://gist.github.com/cahna/43a1a3ff4d075bcd71f9d7120037a501 headers = [h for h in ' '.join(data[0].strip().split()).split() if h] raw_data = map(lambda s: s.strip().split(None, len(headers) - 1), data[1:]) raw_output = [dict(zip(headers, r)) for r in raw_data] @@ -37,22 +38,26 @@ def sparse_table_parse(data, delim='\u2063'): Parameters: - data: (list) Text data to parse that has been split into lines via .splitlines(). - Item 0 must be the header row. Any spaces in header names should be - changed to underscore '_'. You should also ensure headers are - lowercase by using .lower(). Do not change the position of header - names as the positions are used to find the data. + data: (list) Text data to parse that has been split into lines + via .splitlines(). Item 0 must be the header row. + Any spaces in header names should be changed to + underscore '_'. You should also ensure headers are + lowercase by using .lower(). Do not change the + position of header names as the positions are used + to find the data. - Also, ensure there are no blank lines (list items) in the data. + Also, ensure there are no blank lines (list items) + in the data. - delim: (string) Delimiter to use. By default 'u\2063' (invisible separator) is used - since this is unlikely to ever be seen in terminal output. You can - change this for troubleshooting purposes or if there is a delimiter - conflict with your data. + delim: (string) Delimiter to use. By default `u\\2063` + (invisible separator) is used since it is unlikely + to ever be seen in terminal output. You can change + this for troubleshooting purposes or if there is a + delimiter conflict with your data. Returns: - List of Dictionaries raw structured data + List of Dictionaries """ output = [] header_text = data.pop(0) diff --git a/jc/parsers/upower.py b/jc/parsers/upower.py index 400443ec..0a3df60d 100644 --- a/jc/parsers/upower.py +++ b/jc/parsers/upower.py @@ -1,8 +1,10 @@ """jc - JSON CLI output utility `upower` command output parser -The `updated_epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `updated_epoch` calculated timestamp field is naive. (i.e. based on the +local time of the system the parser is run on) -The `updated_epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `updated_epoch_utc` calculated timestamp field is timezone-aware and is +only available if the timezone field is UTC. Usage (cli): @@ -14,6 +16,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('upower', upower_command_output) + + or + import jc.parsers.upower result = jc.parsers.upower.parse(upower_command_output) @@ -26,8 +33,8 @@ Schema: "native_path": string, "power_supply": boolean, "updated": string, - "updated_epoch": integer, # null if date-time conversion fails - "updated_epoch_utc": integer, # null if date-time conversion fails + "updated_epoch": integer, # [0] + "updated_epoch_utc": integer, # [0] "updated_seconds_ago": integer, "has_history": boolean, "has_statistics": boolean, @@ -79,12 +86,14 @@ Schema: } ] + [0] null if date-time conversion fails + Examples: $ upower -i /org/freedesktop/UPower/devices/battery | jc --upower -p [ { - "native_path": "/sys/devices/LNXSYSTM:00/device:00/PNP0C0A:00/power_supply/BAT0", + "native_path": "/sys/devices/LNXSYSTM:00/device:00/PNP0C0A:00/p...", "vendor": "NOTEBOOK", "model": "BAT", "serial": "0001", @@ -143,7 +152,7 @@ Examples: $ upower -i /org/freedesktop/UPower/devices/battery | jc --upower -p -r [ { - "native_path": "/sys/devices/LNXSYSTM:00/device:00/PNP0C0A:00/power_supply/BAT0", + "native_path": "/sys/devices/LNXSYSTM:00/device:00/PNP0C0A:00/p...", "vendor": "NOTEBOOK", "model": "BAT", "serial": "0001", @@ -198,9 +207,6 @@ class info(): description = '`upower` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['upower'] @@ -233,7 +239,10 @@ def _process(proc_data): entry['updated_epoch_utc'] = ts.utc # top level boolean conversions - bool_list = ['power_supply', 'has_history', 'has_statistics', 'on_battery', 'lid_is_closed', 'lid_is_present'] + bool_list = [ + 'power_supply', 'has_history', 'has_statistics', 'on_battery', 'lid_is_closed', + 'lid_is_present' + ] for key in entry: if key in bool_list: entry[key] = jc.utils.convert_to_bool(entry[key]) @@ -316,7 +325,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: @@ -369,7 +378,9 @@ def parse(data, raw=False, quiet=False): # general detail lines if line.startswith(' ') and ':' in line: - key = line.split(':', maxsplit=1)[0].strip().lower().replace('-', '_').replace(' ', '_').replace('(', '').replace(')', '') + key = line.split(':', maxsplit=1)[0].strip().lower().replace('-', '_')\ + .replace(' ', '_').replace('(', '')\ + .replace(')', '') val = line.split(':', maxsplit=1)[1].strip() device_obj['detail'][key] = val continue @@ -388,7 +399,9 @@ def parse(data, raw=False, quiet=False): # top level lines if line.startswith(' ') and ':' in line: - key = line.split(':', maxsplit=1)[0].strip().lower().replace('-', '_').replace(' ', '_').replace('(', '').replace(')', '') + key = line.split(':', maxsplit=1)[0].strip().lower().replace('-', '_')\ + .replace(' ', '_').replace('(', '')\ + .replace(')', '') val = line.split(':', maxsplit=1)[1].strip() device_obj[key] = val continue diff --git a/jc/parsers/uptime.py b/jc/parsers/uptime.py index 810815a2..9b433dcc 100644 --- a/jc/parsers/uptime.py +++ b/jc/parsers/uptime.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('uptime', uptime_command_output) + + or + import jc.parsers.uptime result = jc.parsers.uptime.parse(uptime_command_output) @@ -69,8 +74,6 @@ class info(): description = '`uptime` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] magic_commands = ['uptime'] @@ -148,7 +151,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/vmstat.py b/jc/parsers/vmstat.py index c05caa27..a456e0e4 100644 --- a/jc/parsers/vmstat.py +++ b/jc/parsers/vmstat.py @@ -2,9 +2,11 @@ Options supported: `-a`, `-w`, `-d`, `-t` -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the timezone field is UTC. Usage (cli): @@ -16,6 +18,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('vmstat', vmstat_command_output) + + or + import jc.parsers.vmstat result = jc.parsers.vmstat.parse(vmstat_command_output) @@ -55,11 +62,14 @@ Schema: "io_seconds": integer, "timestamp": string, "timezone": string, - "epoch": integer, # naive timestamp if -t flag is used - "epoch_utc": integer # aware timestamp if -t flag is used and UTC TZ + "epoch": integer, # [0] + "epoch_utc": integer # [1] } ] + [0] naive timestamp if -t flag is used + [1] aware timestamp if -t flag is used and UTC TZ + Examples: $ vmstat | jc --vmstat -p @@ -125,9 +135,6 @@ class info(): description = '`vmstat` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] magic_commands = ['vmstat'] @@ -148,11 +155,13 @@ def _process(proc_data): List of Dictionaries. Structured to conform to the schema. """ - int_list = ['runnable_procs', 'uninterruptible_sleeping_procs', 'virtual_mem_used', 'free_mem', 'buffer_mem', - 'cache_mem', 'inactive_mem', 'active_mem', 'swap_in', 'swap_out', 'blocks_in', 'blocks_out', - 'interrupts', 'context_switches', 'user_time', 'system_time', 'idle_time', 'io_wait_time', - 'stolen_time', 'total_reads', 'merged_reads', 'sectors_read', 'reading_ms', 'total_writes', - 'merged_writes', 'sectors_written', 'writing_ms', 'current_io', 'io_seconds'] + int_list = [ + 'runnable_procs', 'uninterruptible_sleeping_procs', 'virtual_mem_used', 'free_mem', + 'buffer_mem', 'cache_mem', 'inactive_mem', 'active_mem', 'swap_in', 'swap_out', 'blocks_in', + 'blocks_out', 'interrupts', 'context_switches', 'user_time', 'system_time', 'idle_time', + 'io_wait_time', 'stolen_time', 'total_reads', 'merged_reads', 'sectors_read', 'reading_ms', + 'total_writes', 'merged_writes', 'sectors_written', 'writing_ms', 'current_io', 'io_seconds' + ] for entry in proc_data: for key in entry: @@ -174,7 +183,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/vmstat_s.py b/jc/parsers/vmstat_s.py index 044c79a0..0e0a319c 100644 --- a/jc/parsers/vmstat_s.py +++ b/jc/parsers/vmstat_s.py @@ -4,76 +4,100 @@ Options supported: `-a`, `-w`, `-d`, `-t` -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) -The `epoch_utc` calculated timestamp field is timezone-aware and is only available if the timezone field is UTC. +The `epoch_utc` calculated timestamp field is timezone-aware and is only +available if the timezone field is UTC. Usage (cli): $ vmstat | jc --vmstat-s -> Note: When piping `jc` converted `vmstat` output to other processes it may appear the output is hanging due to the OS pipe buffers. This is because `vmstat` output is too small to quickly fill up the buffer. Use the `-u` option to unbuffer the `jc` output if you would like immediate output. See the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) for more information. +> Note: When piping `jc` converted `vmstat` output to other processes it may +appear the output is hanging due to the OS pipe buffers. This is because +`vmstat` output is too small to quickly fill up the buffer. Use the `-u` +option to unbuffer the `jc` output if you would like immediate output. See +the [readme](https://github.com/kellyjonbrazil/jc/tree/master#unbuffering-output) +for more information. Usage (module): + import jc + # result is an iterable object (generator) + result = jc.parse('vmstat_s', vmstat_command_output.splitlines()) + for item in result: + # do something + + or + import jc.parsers.vmstat_s - result = jc.parsers.vmstat_s.parse(vmstat_command_output.splitlines()) # result is an iterable object + # result is an iterable object (generator) + result = jc.parsers.vmstat_s.parse(vmstat_command_output.splitlines()) for item in result: # do something Schema: { - "runnable_procs": integer, - "uninterruptible_sleeping_procs": integer, - "virtual_mem_used": integer, - "free_mem": integer, - "buffer_mem": integer, - "cache_mem": integer, - "inactive_mem": integer, - "active_mem": integer, - "swap_in": integer, - "swap_out": integer, - "blocks_in": integer, - "blocks_out": integer, - "interrupts": integer, - "context_switches": integer, - "user_time": integer, - "system_time": integer, - "idle_time": integer, - "io_wait_time": integer, - "stolen_time": integer, - "disk": string, - "total_reads": integer, - "merged_reads": integer, - "sectors_read": integer, - "reading_ms": integer, - "total_writes": integer, - "merged_writes": integer, - "sectors_written": integer, - "writing_ms": integer, - "current_io": integer, - "io_seconds": integer, - "timestamp": string, - "timezone": string, - "epoch": integer, # naive timestamp if -t flag is used - "epoch_utc": integer # aware timestamp if -t flag is used and UTC TZ - "_jc_meta": # This object only exists if using -qq or ignore_exceptions=True + "runnable_procs": integer, + "uninterruptible_sleeping_procs": integer, + "virtual_mem_used": integer, + "free_mem": integer, + "buffer_mem": integer, + "cache_mem": integer, + "inactive_mem": integer, + "active_mem": integer, + "swap_in": integer, + "swap_out": integer, + "blocks_in": integer, + "blocks_out": integer, + "interrupts": integer, + "context_switches": integer, + "user_time": integer, + "system_time": integer, + "idle_time": integer, + "io_wait_time": integer, + "stolen_time": integer, + "disk": string, + "total_reads": integer, + "merged_reads": integer, + "sectors_read": integer, + "reading_ms": integer, + "total_writes": integer, + "merged_writes": integer, + "sectors_written": integer, + "writing_ms": integer, + "current_io": integer, + "io_seconds": integer, + "timestamp": string, + "timezone": string, + "epoch": integer, # [0] + "epoch_utc": integer # [1] + + # Below object only exists if using -qq or ignore_exceptions=True + + "_jc_meta": { - "success": boolean, # true if successfully parsed, false if error - "error": string, # exists if "success" is false - "line": string # exists if "success" is false + "success": boolean, # [2] + "error": string, # [3] + "line": string # [3] } } + [0] naive timestamp if -t flag is used + [1] aware timestamp if -t flag is used and UTC TZ + [2] false if error parsing + [3] exists if "success" is false + Examples: $ vmstat | jc --vmstat-s - {"runnable_procs":2,"uninterruptible_sleeping_procs":0,"virtual_mem_used":0,"free_mem":2794468,"buffer_mem":2108,"cache_mem":741208,"inactive_mem":null,"active_mem":null,"swap_in":0,"swap_out":0,"blocks_in":1,"blocks_out":3,"interrupts":29,"context_switches":57,"user_time":0,"system_time":0,"idle_time":99,"io_wait_time":0,"stolen_time":0,"timestamp":null,"timezone":null} + {"runnable_procs":2,"uninterruptible_sleeping_procs":0,"virtual_mem...} ... $ vmstat | jc --vmstat-s -r - {"runnable_procs":"2","uninterruptible_sleeping_procs":"0","virtual_mem_used":"0","free_mem":"2794468","buffer_mem":"2108","cache_mem":"741208","inactive_mem":null,"active_mem":null,"swap_in":"0","swap_out":"0","blocks_in":"1","blocks_out":"3","interrupts":"29","context_switches":"57","user_time":"0","system_time":"0","idle_time":"99","io_wait_time":"0","stolen_time":"0","timestamp":null,"timezone":null} + {"runnable_procs":"2","uninterruptible_sleeping_procs":"0","virtua...} ... """ import jc.utils @@ -87,8 +111,6 @@ class info(): description = '`vmstat` command streaming parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux'] streaming = True @@ -108,11 +130,13 @@ def _process(proc_data): Dictionary. Structured data to conform to the schema. """ - int_list = ['runnable_procs', 'uninterruptible_sleeping_procs', 'virtual_mem_used', 'free_mem', 'buffer_mem', - 'cache_mem', 'inactive_mem', 'active_mem', 'swap_in', 'swap_out', 'blocks_in', 'blocks_out', - 'interrupts', 'context_switches', 'user_time', 'system_time', 'idle_time', 'io_wait_time', - 'stolen_time', 'total_reads', 'merged_reads', 'sectors_read', 'reading_ms', 'total_writes', - 'merged_writes', 'sectors_written', 'writing_ms', 'current_io', 'io_seconds'] + int_list = [ + 'runnable_procs', 'uninterruptible_sleeping_procs', 'virtual_mem_used', 'free_mem', + 'buffer_mem', 'cache_mem', 'inactive_mem', 'active_mem', 'swap_in', 'swap_out', 'blocks_in', + 'blocks_out', 'interrupts', 'context_switches', 'user_time', 'system_time', 'idle_time', + 'io_wait_time', 'stolen_time', 'total_reads', 'merged_reads', 'sectors_read', 'reading_ms', + 'total_writes', 'merged_writes', 'sectors_written', 'writing_ms', 'current_io', 'io_seconds' + ] for key in proc_data: if key in int_list: @@ -132,8 +156,10 @@ def parse(data, raw=False, quiet=False, ignore_exceptions=False): Parameters: - data: (iterable) line-based text data to parse (e.g. sys.stdin or str.splitlines()) - raw: (boolean) output preprocessed JSON if True + data: (iterable) line-based text data to parse + (e.g. sys.stdin or str.splitlines()) + + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True ignore_exceptions: (boolean) ignore parsing exceptions if True diff --git a/jc/parsers/w.py b/jc/parsers/w.py index e0ff5ec5..386bd066 100644 --- a/jc/parsers/w.py +++ b/jc/parsers/w.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('w', w_command_output) + + or + import jc.parsers.w result = jc.parsers.w.parse(w_command_output) @@ -108,8 +113,6 @@ class info(): description = '`w` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] magic_commands = ['w'] @@ -146,7 +149,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/wc.py b/jc/parsers/wc.py index 5f672ff7..ce10b46e 100644 --- a/jc/parsers/wc.py +++ b/jc/parsers/wc.py @@ -10,6 +10,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('wc', wc_command_output) + + or + import jc.parsers.wc result = jc.parsers.wc.parse(wc_command_output) @@ -58,8 +63,6 @@ class info(): description = '`wc` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] magic_commands = ['wc'] @@ -96,7 +99,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/who.py b/jc/parsers/who.py index 3df28d9f..541df5d1 100644 --- a/jc/parsers/who.py +++ b/jc/parsers/who.py @@ -2,7 +2,8 @@ Accepts any of the following who options (or no options): `-aTH` -The `epoch` calculated timestamp field is naive (i.e. based on the local time of the system the parser is run on) +The `epoch` calculated timestamp field is naive. (i.e. based on the local +time of the system the parser is run on) Usage (cli): @@ -14,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('who', who_command_output) + + or + import jc.parsers.who result = jc.parsers.who.parse(who_command_output) @@ -26,7 +32,7 @@ Schema: "writeable_tty": string, "tty": string, "time": string, - "epoch": integer, # naive timestamp. null if time cannot be converted + "epoch": integer, # [0] "idle": string, "pid": integer, "from": string, @@ -34,6 +40,8 @@ Schema: } ] + [0] naive timestamp. null if time cannot be converted + Examples: $ who -a | jc --who -p @@ -137,9 +145,6 @@ class info(): description = '`who` command parser' author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' - # details = 'enter any other details here' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'aix', 'freebsd'] magic_commands = ['who'] @@ -179,7 +184,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/xml.py b/jc/parsers/xml.py index 544d4d80..3100f586 100644 --- a/jc/parsers/xml.py +++ b/jc/parsers/xml.py @@ -6,6 +6,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('xml', xml_file_output) + + or + import jc.parsers.xml result = jc.parsers.xml.parse(xml_file_output) @@ -76,8 +81,6 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Using the xmltodict library at https://github.com/martinblech/xmltodict' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] @@ -108,7 +111,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/yaml.py b/jc/parsers/yaml.py index ba0b7a59..f8c085cd 100644 --- a/jc/parsers/yaml.py +++ b/jc/parsers/yaml.py @@ -6,6 +6,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('yaml', yaml_file_output) + + or + import jc.parsers.yaml result = jc.parsers.yaml.parse(yaml_file_output) @@ -90,8 +95,6 @@ class info(): author = 'Kelly Brazil' author_email = 'kellyjonbrazil@gmail.com' details = 'Using the ruamel.yaml library at https://pypi.org/project/ruamel.yaml' - - # compatible options: linux, darwin, cygwin, win32, aix, freebsd compatible = ['linux', 'darwin', 'cygwin', 'win32', 'aix', 'freebsd'] @@ -122,7 +125,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/parsers/zipinfo.py b/jc/parsers/zipinfo.py index dab8aab6..7627659d 100644 --- a/jc/parsers/zipinfo.py +++ b/jc/parsers/zipinfo.py @@ -15,6 +15,11 @@ Usage (cli): Usage (module): + import jc + result = jc.parse('zipinfo', zipinfo_command_output) + + or + import jc.parsers.zipinfo result = jc.parsers.zipinfo.parse(zipinfo_command_output) @@ -129,7 +134,7 @@ def parse(data, raw=False, quiet=False): Parameters: data: (string) text data to parse - raw: (boolean) output preprocessed JSON if True + raw: (boolean) unprocessed output if True quiet: (boolean) suppress warning messages if True Returns: diff --git a/jc/utils.py b/jc/utils.py index f4b8a85e..b0a467ae 100644 --- a/jc/utils.py +++ b/jc/utils.py @@ -9,9 +9,9 @@ from textwrap import TextWrapper def warning_message(message_lines): """ - Prints warning message for non-fatal issues. The first line is prepended with - 'jc: Warning - ' and subsequent lines are indented. Wraps text as needed based - on the terminal width. + Prints warning message for non-fatal issues. The first line is + prepended with 'jc: Warning - ' and subsequent lines are indented. + Wraps text as needed based on the terminal width. Parameters: @@ -45,9 +45,9 @@ def warning_message(message_lines): def error_message(message_lines): """ - Prints an error message for fatal issues. The first line is prepended with - 'jc: Error - ' and subsequent lines are indented. Wraps text as needed based - on the terminal width. + Prints an error message for fatal issues. The first line is + prepended with 'jc: Error - ' and subsequent lines are indented. + Wraps text as needed based on the terminal width. Parameters: @@ -76,17 +76,19 @@ def error_message(message_lines): def compatibility(mod_name, compatible, quiet=False): - """Checks for the parser's compatibility with the running OS platform. + """ + Checks for the parser's compatibility with the running OS + platform. Parameters: - mod_name: (string) __name__ of the calling module + mod_name: (string) __name__ of the calling module - compatible: (list) sys.platform name(s) compatible with the parser - compatible options: - linux, darwin, cygwin, win32, aix, freebsd + compatible: (list) sys.platform name(s) compatible with + the parser. compatible options: + linux, darwin, cygwin, win32, aix, freebsd - quiet: (bool) supress compatibility message if True + quiet: (bool) supress compatibility message if True Returns: @@ -109,7 +111,8 @@ def compatibility(mod_name, compatible, quiet=False): def has_data(data): """ - Checks if the input contains data. If there are any non-whitespace characters then return True, else return False + Checks if the input contains data. If there are any non-whitespace + characters then return True, else return False. Parameters: @@ -117,22 +120,24 @@ def has_data(data): Returns: - Boolean True if input string (data) contains non-whitespace characters, otherwise False + Boolean True if input string (data) contains non-whitespace + characters, otherwise False """ return bool(data and not data.isspace()) def convert_to_int(value): """ - Converts string and float input to int. Strips all non-numeric characters from strings. + Converts string and float input to int. Strips all non-numeric + characters from strings. Parameters: - value: (string/integer/float) Input value + value: (string/integer/float) Input value Returns: - integer/None Integer if successful conversion, otherwise None + integer/None Integer if successful conversion, otherwise None """ if isinstance(value, str): str_val = re.sub(r'[^0-9\-\.]', '', value) @@ -153,15 +158,16 @@ def convert_to_int(value): def convert_to_float(value): """ - Converts string and int input to float. Strips all non-numeric characters from strings. + Converts string and int input to float. Strips all non-numeric + characters from strings. Parameters: - value: (string) Input value + value: (string) Input value Returns: - float/None Float if successful conversion, otherwise None + float/None Float if successful conversion, otherwise None """ if isinstance(value, str): try: @@ -178,7 +184,8 @@ def convert_to_float(value): def convert_to_bool(value): """ - Converts string, integer, or float input to boolean by checking for 'truthy' values + Converts string, integer, or float input to boolean by checking + for 'truthy' values. Parameters: @@ -186,7 +193,8 @@ def convert_to_bool(value): Returns: - True/False False unless a 'truthy' number or string is found ('y', 'yes', 'true', '1', 1, -1, etc.) + True/False False unless a 'truthy' number or string is found + ('y', 'yes', 'true', '1', 1, -1, etc.) """ # if number, then bool it # if string, try to convert to float @@ -221,8 +229,9 @@ def stream_success(output_line, ignore_exceptions): def stream_error(e, ignore_exceptions, line): - """Reraise the stream exception with annotation or print an error `_jc_meta` - field if `ignore_exceptions=True` + """ + Reraise the stream exception with annotation or print an error + `_jc_meta` field if `ignore_exceptions=True`. """ if not ignore_exceptions: e.args = (str(e) + '... Use the ignore_exceptions option (-qq) to ignore streaming parser errors.',) @@ -258,18 +267,28 @@ def streaming_line_input_type_check(line): class timestamp: """ - Input a date-time text string of several formats and convert to a naive or timezone-aware epoch timestamp in UTC + Input a date-time text string of several formats and convert to a + naive or timezone-aware epoch timestamp in UTC. Parameters: - datetime_string: (str) a string representation of a date-time in several supported formats + datetime_string: (str) a string representation of a + date-time in several supported formats Attributes: - string (str) the input datetime string - format (int) the format rule that was used to decode the datetime string. None if conversion fails - naive (int) timestamp based on locally configured timezone. None if conversion fails - utc (int) aware timestamp only if UTC timezone detected in datetime string. None if conversion fails + string (str) the input datetime string + + format (int) the format rule that was used to + decode the datetime string. None if + conversion fails + + naive (int) timestamp based on locally configured + timezone. None if conversion fails + + utc (int) aware timestamp only if UTC timezone + detected in datetime string. None if + conversion fails """ def __init__(self, datetime_string): @@ -284,27 +303,42 @@ class timestamp: def _parse(self): """ - Input a date-time text string of several formats and convert to a naive or timezone-aware epoch timestamp in UTC + Input a date-time text string of several formats and convert to + a naive or timezone-aware epoch timestamp in UTC. Parameters: - data: (string) a string representation of a date-time in several supported formats + data: (string) a string representation of a date-time + in several supported formats Returns: Dictionary A Dictionary of the following format: { - "format": integer, # for debugging purposes. None if conversion fails - "timestamp_naive": integer, # timestamp based on locally configured timezone. None if conversion fails - "timestamp_utc": integer # aware timestamp only if UTC timezone detected. None if conversion fails + # for debugging purposes. None if conversion fails + "format": integer, + + # timestamp based on locally configured timezone. + # None if conversion fails. + "timestamp_naive": integer, + + # aware timestamp only if UTC timezone detected. + # None if conversion fails. + "timestamp_utc": integer } - The format integer denotes which date_time format conversion succeeded. - The timestamp_naive integer is the converted date-time string to a naive epoch timestamp. - The timestamp_utc integer is the converted date-time string to an aware epoch timestamp - in the UTC timezone. If an aware conversion cannot be performed (e.g. the UTC timezone - is not found in the date-time string), then this field will be None. + The `format` integer denotes which date_time format + conversion succeeded. + + The `timestamp_naive` integer is the converted date-time + string to a naive epoch timestamp. + + The `timestamp_utc` integer is the converted date-time + string to an aware epoch timestamp in the UTC timezone. If + an aware conversion cannot be performed (e.g. the UTC + timezone is not found in the date-time string), then this + field will be None. If the conversion completely fails, all fields will be None. """ diff --git a/man/jc.1 b/man/jc.1 index 5589fc6a..d4e607f1 100644 --- a/man/jc.1 +++ b/man/jc.1 @@ -1,4 +1,4 @@ -.TH jc 1 2022-01-14 1.17.7 "JSON CLI output utility" +.TH jc 1 2022-01-21 1.18.0 "JSON CLI output utility" .SH NAME jc \- JSONifies the output of many CLI tools and file-types .SH SYNOPSIS diff --git a/setup.py b/setup.py index 3d5d6d1f..af4c6760 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md', 'r') as f: setuptools.setup( name='jc', - version='1.17.7', + version='1.18.0', author='Kelly Brazil', author_email='kellyjonbrazil@gmail.com', description='Converts the output of popular command-line tools and file-types to JSON.', diff --git a/templates/readme_template b/templates/readme_template index 18c6ec32..a7918715 100644 --- a/templates/readme_template +++ b/templates/readme_template @@ -32,10 +32,10 @@ $ jc dig example.com | jq -r '.[].answer[].data' The `jc` parsers can also be used as python modules. In this case the output will be a python dictionary, or list of dictionaries, instead of JSON: ```python >>> import subprocess ->>> import jc.parsers.dig ->>> +>>> import jc +>>> >>> cmd_output = subprocess.check_output(['dig', 'example.com'], text=True) ->>> data = jc.parsers.dig.parse(cmd_output) +>>> data = jc.parse('dig', cmd_output) >>> >>> data [{'id': 64612, 'opcode': 'QUERY', 'status': 'NOERROR', 'flags': ['qr', 'rd', 'ra'], 'query_num': 1, 'answer_num': @@ -45,6 +45,9 @@ The `jc` parsers can also be used as python modules. In this case the output wil '2600:1700:bab0:d40::1#53(2600:1700:bab0:d40::1)', 'when': 'Fri Apr 16 16:13:00 PDT 2021', 'rcvd': 56, 'when_epoch': 1618614780, 'when_epoch_utc': None}] ``` + +> For `jc` Python package documentation, use `help('jc')`, `help('jc.lib')`, or see the [online documentation](https://github.com/kellyjonbrazil/jc/tree/master/docs). + Two representations of the data are available. The default representation uses a strict schema per parser and converts known numbers to int/float JSON values. Certain known values of `None` are converted to JSON `null`, known boolean values are converted, and, in some cases, additional semantic context fields are added. To access the raw, pre-processed JSON, use the `-r` cli option or the `raw=True` function parameter in `parse()`. @@ -206,9 +209,9 @@ Streaming parsers accept any iterable object and return a generator iterator obj To use the generator object in your code, simply loop through it or use the [next()](https://docs.python.org/3/library/functions.html#next) builtin function: ```python -import jc.parsers.ls_s +import jc -result = jc.parsers.ls_s.parse(ls_command_output.splitlines()) +result = jc.parse('ls_s', ls_command_output.splitlines()) for item in result: print(item["filename"]) ``` diff --git a/tests/test_jc.py b/tests/test_jc.py new file mode 100644 index 00000000..60fce850 --- /dev/null +++ b/tests/test_jc.py @@ -0,0 +1,41 @@ +import unittest +from typing import Generator +import jc + + +class MyTests(unittest.TestCase): + def test_jc_parse_csv(self): + data = { + '': [], + 'a,b,c\n1,2,3': [{'a':'1', 'b':'2', 'c':'3'}] + } + + for test_data, expected_output in data.items(): + self.assertEqual(jc.parse('csv', test_data), expected_output) + + def test_jc_parse_csv_s_is_generator(self): + self.assertIsInstance(jc.parse('csv_s', 'a,b,c\n1,2,3'), Generator) + + def test_jc_parse_kv(self): + data = { + '': {}, + 'a=1\nb=2\nc=3': {'a':'1', 'b':'2', 'c':'3'} + } + + for test_data, expected_output in data.items(): + self.assertEqual(jc.parse('kv', test_data), expected_output) + + def test_jc_parser_mod_list_is_list(self): + self.assertIsInstance(jc.parser_mod_list(), list) + + def test_jc_parser_mod_list_contains_csv(self): + self.assertTrue('csv' in jc.parser_mod_list()) + + def test_jc_parser_mod_list_length(self): + self.assertGreaterEqual(len(jc.parser_mod_list()), 80) + + def test_jc_plugin_parser_mod_list_is_list(self): + self.assertIsInstance(jc.plugin_parser_mod_list(), list) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/test_lib.py b/tests/test_lib.py new file mode 100644 index 00000000..2714b811 --- /dev/null +++ b/tests/test_lib.py @@ -0,0 +1,51 @@ +import unittest +from typing import Generator +import jc.lib + + +class MyTests(unittest.TestCase): + def test_lib_parse_csv(self): + data = { + '': [], + 'a,b,c\n1,2,3': [{'a':'1', 'b':'2', 'c':'3'}] + } + + for test_data, expected_output in data.items(): + self.assertEqual(jc.lib.parse('csv', test_data), expected_output) + + def test_lib_parse_csv_s_is_generator(self): + self.assertIsInstance(jc.lib.parse('csv_s', 'a,b,c\n1,2,3'), Generator) + + def test_lib_parse_kv(self): + data = { + '': {}, + 'a=1\nb=2\nc=3': {'a':'1', 'b':'2', 'c':'3'} + } + + for test_data, expected_output in data.items(): + self.assertEqual(jc.lib.parse('kv', test_data), expected_output) + + def test_lib_parser_mod_list_is_list(self): + self.assertIsInstance(jc.lib.parser_mod_list(), list) + + def test_lib_parser_mod_list_contains_csv(self): + self.assertTrue('csv' in jc.lib.parser_mod_list()) + + def test_lib_parser_mod_list_length(self): + self.assertGreaterEqual(len(jc.lib.parser_mod_list()), 80) + + def test_lib_plugin_parser_mod_list_is_list(self): + self.assertIsInstance(jc.lib.plugin_parser_mod_list(), list) + + def test_lib_plugin_parser_mod_list_length_is_zero(self): + """Ensure there are no plugin parsers present during test/build.""" + self.assertEqual(len(jc.lib.plugin_parser_mod_list()), 0) + + def test_lib_cliname_to_modname(self): + self.assertEqual(jc.lib._cliname_to_modname('module-name'), 'module_name') + + def test_lib_modname_to_cliname(self): + self.assertEqual(jc.lib._modname_to_cliname('module_name'), 'module-name') + +if __name__ == '__main__': + unittest.main() \ No newline at end of file