python/jc
1
0
Fork 0
mirror of https://github.com/kellyjonbrazil/jc.git synced 2025-06-17 00:07:37 +02:00
Code Issues Releases Activity

New doc generation script

Browse Source
This commit is contained in:
Kelly Brazil
2024-03-14 21:44:37 -07:00
parent 401b7b31b8
commit 37061227d9
13 changed files with 502 additions and 530 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
doc2md.py Executable file
View File

@ -0,0 +1,92 @@
#!/usr/bin/env python3
"""
Convert parser doc string to markdown
"""
import sys
import importlib
from inspect import isfunction, signature, cleandoc
import yapf
import os
import sys
ignore_lib_functions = [
'cast',
'wraps',
'lru_cache',
'namedtuple'
]
sys.path.append(os.getcwd() + '/jc')
mod_path = sys.argv[1]
mod_name = mod_path.split('.')[-1]
module = importlib.import_module(f'{mod_path}')
######## HEADER ########
header = f'''[Home](https://kellyjonbrazil.github.io/jc/)
<a id="{mod_path}"></a>
# {mod_path}
'''
summary = module.__doc__
functions = []
for attribute in dir(module):
if isfunction(getattr(module, attribute)) \
and not getattr(module, attribute).__name__.startswith('_'):
if 'jc.parsers.' in mod_path and not 'universal' in mod_path:
if attribute == 'parse':
functions.append(attribute)
else:
if not attribute in ignore_lib_functions:
functions.append(attribute)
######## TABLE OF CONTENTS ########
toc = f'# Table of Contents\n\n*[{mod_path}](#{mod_path})\n'
for api in functions:
toc = f'{toc} *[{api}](#{mod_path}.{api})\n'
######## API DOCS ########
api_docs = ''
for api in functions:
api_function = getattr(module, api)
this_header = f'<a id="{mod_path}.{api}"></a>\n\n### {api}\n'
this_sig = str(signature(api_function))
formatted_sig = yapf.yapf_api.FormatCode(f'def {api_function.__name__}{this_sig}:\n pass' )
formatted_sig = formatted_sig[0].split(':\n pass')[0]
this_name_and_sig = f'{this_header}\n```python\n{formatted_sig}\n```'
this_doc = cleandoc(api_function.__doc__)
api_docs = api_docs + this_name_and_sig + '\n\n' + this_doc + '\n\n'
######## FOOTER ########
footer = ''
if 'jc.parsers.' in mod_path and not 'universal' in mod_path:
footer = '### Parser Information\n'
comp = ', '.join(module.info.compatible)
ver = module.info.version
author = module.info.author
author_email = module.info.author_email
slurpable = 'slurpable' in module.info.tags
footer = footer + f'Compatibility: {comp}\n\n'
footer = footer + f'Source: [`jc/parsers/{mod_name}.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/{mod_name}.py)\n\n'
if slurpable:
footer = footer + 'This parser can be used with the `--slurp` command-line option.\n\n'
footer = footer + f'Version {ver} by {author} ({author_email})\n'
if 'jc.parsers.' in mod_path and not 'universal' in mod_path:
final_doc = header + '\n' + summary + '\n\n' + api_docs + '\n' + footer
elif mod_path == 'jc':
final_doc = header + '\n' + summary
else:
final_doc = header + '\n' + toc + '\n' + summary + '\n\n' + api_docs
print(final_doc)

110
docgen.sh
View File

@ -1,107 +1,34 @@
#!/bin/bash #!/bin/bash
# Generate docs.md # Generate markdown document files (*.md)
# requires pydoc-markdown 4.6.1
# Requires the yapf python library
# use ./docgen all to generate all docs # use ./docgen all to generate all docs
readme_config=$(cat <<'EOF'
{
"processors": [
{
"type": "filter"
},
{
"type": "pydocmd"
}
],
"renderer": {
"type": "markdown",
"header_level_by_type": {
"Module": 1,
"Class": 3,
"Method": 3,
"Function": 3,
"Variable": 3
}
}
}
EOF
)
toc_config=$(cat <<'EOF'
{
"processors": [
{
"type": "filter"
},
{
"type": "pydocmd"
}
],
"renderer": {
"type": "markdown",
"render_toc": true,
"header_level_by_type": {
"Module": 1,
"Class": 3,
"Method": 3,
"Function": 3,
"Variable": 3
}
}
}
EOF
)
parser_config=$(cat <<'EOF'
{
"processors": [
{
"type": "filter",
"expression": "not name == \"info\" and not name.startswith(\"_\") and default()"
},
{
"type": "pydocmd"
}
],
"renderer": {
"type": "markdown",
"header_level_by_type": {
"Module": 1,
"Class": 3,
"Method": 3,
"Function": 3,
"Variable": 3
}
}
}
EOF
)
cd jc cd jc
( (
echo Building docs for: package echo Building docs for: package
pydoc-markdown -m jc "${readme_config}" > ../docs/readme.md; echo "+++ package docs complete" ../doc2md.py jc > ../docs/readme.md && echo "+++ package docs complete" || echo "*** PACKAGE DOCS FAILED"
) & ) &
( (
echo Building docs for: lib echo Building docs for: lib
pydoc-markdown -m jc.lib "${toc_config}" > ../docs/lib.md; echo "+++ lib docs complete" ../doc2md.py jc.lib > ../docs/lib.md && echo "+++ lib docs complete" || echo "*** LIB DOCS FAILED"
) & ) &
( (
echo Building docs for: utils echo Building docs for: utils
pydoc-markdown -m jc.utils "${toc_config}" > ../docs/utils.md; echo "+++ utils docs complete" ../doc2md.py jc.utils > ../docs/utils.md && echo "+++ utils docs complete" || echo "*** UTILS DOCS FAILED"
) & ) &
( (
echo Building docs for: streaming echo Building docs for: streaming
pydoc-markdown -m jc.streaming "${toc_config}" > ../docs/streaming.md; echo "+++ streaming docs complete" ../doc2md.py jc.streaming > ../docs/streaming.md && echo "+++ streaming docs complete" || echo "*** STREAMING DOCS FAILED"
) & ) &
( (
echo Building docs for: universal parser echo Building docs for: universal parser
pydoc-markdown -m jc.parsers.universal "${toc_config}" > ../docs/parsers/universal.md; echo "+++ universal parser docs complete" ../doc2md.py jc.parsers.universal > ../docs/parsers/universal.md && echo "+++ universal parser docs complete" || echo "*** UNIVERSAL PARSER DOCS FAILED"
) & ) &
# a bit of inception here... jc is being used to help # a bit of inception here... jc is being used to help
@ -119,27 +46,8 @@ for parser in "${parsers[@]}"; do
parser_name=$(jq -r '.name' <<< "$parser") parser_name=$(jq -r '.name' <<< "$parser")
{ {
if [[ $1 == "all" ]] || ! git diff --quiet --exit-code HEAD~5 -- "parsers/${parser_name}.py"; then if [[ $1 == "all" ]] || ! git diff --quiet --exit-code HEAD~5 -- "parsers/${parser_name}.py"; then
compatible=$(jq -r '.compatible | join(", ")' <<< "$parser")
version=$(jq -r '.version' <<< "$parser")
author=$(jq -r '.author' <<< "$parser")
author_email=$(jq -r '.author_email' <<< "$parser")
echo "Building docs for: ${parser_name}" echo "Building docs for: ${parser_name}"
echo "[Home](https://kellyjonbrazil.github.io/jc/)" > ../docs/parsers/"${parser_name}".md ../doc2md.py jc.parsers."${parser_name}" > ../docs/parsers/"${parser_name}".md && echo "+++ ${parser_name} docs complete" || echo "*** ${parser_name} DOCS FAILED"
pydoc-markdown -m jc.parsers."${parser_name}" "${parser_config}" >> ../docs/parsers/"${parser_name}".md
echo "### Parser Information" >> ../docs/parsers/"${parser_name}".md
echo "Compatibility: ${compatible}" >> ../docs/parsers/"${parser_name}".md
echo >> ../docs/parsers/"${parser_name}".md
echo "Source: [\`jc/parsers/${parser_name}.py\`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/${parser_name}.py)" >> ../docs/parsers/"${parser_name}".md
echo >> ../docs/parsers/"${parser_name}".md
if $(jq -e '.tags | contains(["slurpable"])' <<< "$parser"); then
echo "This parser can be used with the \`--slurp\` command-line option." >> ../docs/parsers/"${parser_name}".md
echo >> ../docs/parsers/"${parser_name}".md
fi
echo "Version ${version} by ${author} (${author_email})" >> ../docs/parsers/"${parser_name}".md
echo "+++ ${parser_name} docs complete"
fi fi
} & } &
done done

211
docs/lib.md
View File

@ -1,29 +1,67 @@
# Table of Contents [Home](https://kellyjonbrazil.github.io/jc/)
* [jc.lib](#jc.lib)
* [get\_parser](#jc.lib.get_parser)
* [parse](#jc.lib.parse)
* [parser\_mod\_list](#jc.lib.parser_mod_list)
* [plugin\_parser\_mod\_list](#jc.lib.plugin_parser_mod_list)
* [standard\_parser\_mod\_list](#jc.lib.standard_parser_mod_list)
* [streaming\_parser\_mod\_list](#jc.lib.streaming_parser_mod_list)
* [slurpable\_parser\_mod\_list](#jc.lib.slurpable_parser_mod_list)
* [parser\_info](#jc.lib.parser_info)
* [all\_parser\_info](#jc.lib.all_parser_info)
* [get\_help](#jc.lib.get_help)
<a id="jc.lib"></a> <a id="jc.lib"></a>
# jc.lib # jc.lib
# Table of Contents
*[jc.lib](#jc.lib)
*[all_parser_info](#jc.lib.all_parser_info)
*[get_help](#jc.lib.get_help)
*[get_parser](#jc.lib.get_parser)
*[parse](#jc.lib.parse)
*[parser_info](#jc.lib.parser_info)
*[parser_mod_list](#jc.lib.parser_mod_list)
*[plugin_parser_mod_list](#jc.lib.plugin_parser_mod_list)
*[slurpable_parser_mod_list](#jc.lib.slurpable_parser_mod_list)
*[standard_parser_mod_list](#jc.lib.standard_parser_mod_list)
*[streaming_parser_mod_list](#jc.lib.streaming_parser_mod_list)
jc - JSON Convert lib module jc - JSON Convert lib module
<a id="jc.lib.all_parser_info"></a>
### all_parser_info
```python
def all_parser_info(
documentation: bool = False,
show_hidden: bool = False,
show_deprecated: bool = False) -> List[jc.jc_types.ParserInfoType]
```
Returns a list of dictionaries that includes metadata for all parser
modules. By default only non-hidden, non-deprecated parsers are
returned.
Parameters:
documentation: (boolean) include parser docstrings if True
show_hidden: (boolean) also show parsers marked as hidden
in their info metadata.
show_deprecated: (boolean) also show parsers marked as
deprecated in their info metadata.
<a id="jc.lib.get_help"></a>
### get_help
```python
def get_help(parser_mod_name: Union[str, module]) -> None
```
Show help screen for the selected parser.
This function will accept **module_name**, **cli-name**, and
**--argument-name** variants of the module name string as well as a
parser module object.
<a id="jc.lib.get_parser"></a> <a id="jc.lib.get_parser"></a>
### get\_parser ### get_parser
```python ```python
def get_parser(parser_mod_name: Union[str, ModuleType]) -> ModuleType def get_parser(parser_mod_name: Union[str, module]) -> module
``` ```
Return the parser module object and check that the module is a valid Return the parser module object and check that the module is a valid
@ -56,13 +94,13 @@ Raises:
```python ```python
def parse( def parse(
parser_mod_name: Union[str, ModuleType], parser_mod_name: Union[str, module],
data: Union[str, bytes, Iterable[str]], data: Union[str, bytes, Iterable[str]],
quiet: bool = False, quiet: bool = False,
raw: bool = False, raw: bool = False,
ignore_exceptions: Optional[bool] = None, ignore_exceptions: Optional[bool] = None,
**kwargs **kwargs
) -> Union[JSONDictType, List[JSONDictType], Iterator[JSONDictType]] ) -> Union[Dict[str, Any], List[Dict[str, Any]], Iterator[Dict[str, Any]]]
``` ```
Parse the data (string or bytes) using the supplied parser (string or Parse the data (string or bytes) using the supplied parser (string or
@ -152,73 +190,13 @@ Returns:
Standard Parsers: Dictionary or List of Dictionaries Standard Parsers: Dictionary or List of Dictionaries
Streaming Parsers: Generator Object containing Dictionaries Streaming Parsers: Generator Object containing Dictionaries
<a id="jc.lib.parser_mod_list"></a>
### parser\_mod\_list
```python
def parser_mod_list(show_hidden: bool = False,
show_deprecated: bool = False) -> List[str]
```
Returns a list of all available parser module names.
<a id="jc.lib.plugin_parser_mod_list"></a>
### plugin\_parser\_mod\_list
```python
def plugin_parser_mod_list(show_hidden: bool = False,
show_deprecated: bool = False) -> List[str]
```
Returns a list of plugin parser module names. This function is a
subset of `parser_mod_list()`.
<a id="jc.lib.standard_parser_mod_list"></a>
### standard\_parser\_mod\_list
```python
def standard_parser_mod_list(show_hidden: bool = False,
show_deprecated: bool = False) -> List[str]
```
Returns a list of standard parser module names. This function is a
subset of `parser_mod_list()` and does not contain any streaming
parsers.
<a id="jc.lib.streaming_parser_mod_list"></a>
### streaming\_parser\_mod\_list
```python
def streaming_parser_mod_list(show_hidden: bool = False,
show_deprecated: bool = False) -> List[str]
```
Returns a list of streaming parser module names. This function is a
subset of `parser_mod_list()`.
<a id="jc.lib.slurpable_parser_mod_list"></a>
### slurpable\_parser\_mod\_list
```python
def slurpable_parser_mod_list(show_hidden: bool = False,
show_deprecated: bool = False) -> List[str]
```
Returns a list of slurpable parser module names. This function is a
subset of `parser_mod_list()`.
<a id="jc.lib.parser_info"></a> <a id="jc.lib.parser_info"></a>
### parser\_info ### parser_info
```python ```python
def parser_info(parser_mod_name: Union[str, ModuleType], def parser_info(parser_mod_name: Union[str, module],
documentation: bool = False) -> ParserInfoType documentation: bool = False) -> jc.jc_types.ParserInfoType
``` ```
Returns a dictionary that includes the parser module metadata. Returns a dictionary that includes the parser module metadata.
@ -233,39 +211,64 @@ Parameters:
documentation: (boolean) include parser docstring if True documentation: (boolean) include parser docstring if True
<a id="jc.lib.all_parser_info"></a> <a id="jc.lib.parser_mod_list"></a>
### all\_parser\_info ### parser_mod_list
```python ```python
def all_parser_info(documentation: bool = False, def parser_mod_list(show_hidden: bool = False,
show_hidden: bool = False, show_deprecated: bool = False) -> List[str]
show_deprecated: bool = False) -> List[ParserInfoType]
``` ```
Returns a list of dictionaries that includes metadata for all parser Returns a list of all available parser module names.
modules. By default only non-hidden, non-deprecated parsers are
returned.
Parameters: <a id="jc.lib.plugin_parser_mod_list"></a>
documentation: (boolean) include parser docstrings if True ### plugin_parser_mod_list
show_hidden: (boolean) also show parsers marked as hidden
in their info metadata.
show_deprecated: (boolean) also show parsers marked as
deprecated in their info metadata.
<a id="jc.lib.get_help"></a>
### get\_help
```python ```python
def get_help(parser_mod_name: Union[str, ModuleType]) -> None def plugin_parser_mod_list(show_hidden: bool = False,
show_deprecated: bool = False) -> List[str]
``` ```
Show help screen for the selected parser. Returns a list of plugin parser module names. This function is a
subset of `parser_mod_list()`.
<a id="jc.lib.slurpable_parser_mod_list"></a>
### slurpable_parser_mod_list
```python
def slurpable_parser_mod_list(show_hidden: bool = False,
show_deprecated: bool = False) -> List[str]
```
Returns a list of slurpable parser module names. This function is a
subset of `parser_mod_list()`.
<a id="jc.lib.standard_parser_mod_list"></a>
### standard_parser_mod_list
```python
def standard_parser_mod_list(show_hidden: bool = False,
show_deprecated: bool = False) -> List[str]
```
Returns a list of standard parser module names. This function is a
subset of `parser_mod_list()` and does not contain any streaming
parsers.
<a id="jc.lib.streaming_parser_mod_list"></a>
### streaming_parser_mod_list
```python
def streaming_parser_mod_list(show_hidden: bool = False,
show_deprecated: bool = False) -> List[str]
```
Returns a list of streaming parser module names. This function is a
subset of `parser_mod_list()`.
This function will accept **module_name**, **cli-name**, and
**--argument-name** variants of the module name string as well as a
parser module object.

41
docs/parsers/apt_get_sqq.md
View File

@ -1,7 +1,7 @@
[Home](https://kellyjonbrazil.github.io/jc/) [Home](https://kellyjonbrazil.github.io/jc/)
<a id="jc.parsers.apt_get_sqq"></a> <a id="jc.parsers.apt_get_sqq"></a>
# jc.parsers.apt\_get\_sqq # jc.parsers.apt_get_sqq
jc - JSON Convert `apt-get -sqq` command output parser jc - JSON Convert `apt-get -sqq` command output parser
@ -28,7 +28,7 @@ Schema:
"package": string, "package": string,
"broken": string/null, "broken": string/null,
"proposed_pkg_ver": string, "proposed_pkg_ver": string,
"existing_src": string/null, "existing_pkg_ver": string/null,
"architecture": string "architecture": string
} }
] ]
@ -42,7 +42,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": "1.19.7", "broken": "1.19.7",
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -50,7 +50,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": null, "broken": null,
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -58,7 +58,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": "1.19.7", "broken": "1.19.7",
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -66,7 +66,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": "1.19.7", "broken": "1.19.7",
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -74,7 +74,7 @@ Examples:
"package": "base-files", "package": "base-files",
"broken": "10.3+deb10u4", "broken": "10.3+deb10u4",
"proposed_pkg_ver": "10.3+deb10u13 Debian:10.13/oldstable", "proposed_pkg_ver": "10.3+deb10u13 Debian:10.13/oldstable",
"existing_src": null, "existing_pkg_ver": null,
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -82,7 +82,7 @@ Examples:
"package": "base-files", "package": "base-files",
"broken": null, "broken": null,
"proposed_pkg_ver": "10.3+deb10u13 Debian:10.13/oldstable", "proposed_pkg_ver": "10.3+deb10u13 Debian:10.13/oldstable",
"existing_src": null, "existing_pkg_ver": null,
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -90,7 +90,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": "1.19.7", "broken": "1.19.7",
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -98,7 +98,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": null, "broken": null,
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
} }
] ]
@ -110,7 +110,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": "1.19.7", "broken": "1.19.7",
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -118,7 +118,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": null, "broken": null,
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -126,7 +126,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": "1.19.7", "broken": "1.19.7",
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -134,7 +134,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": "1.19.7", "broken": "1.19.7",
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -142,7 +142,7 @@ Examples:
"package": "base-files", "package": "base-files",
"broken": "10.3+deb10u4", "broken": "10.3+deb10u4",
"proposed_pkg_ver": "10.3+deb10u13 Debian:10.13/oldstable", "proposed_pkg_ver": "10.3+deb10u13 Debian:10.13/oldstable",
"existing_src": null, "existing_pkg_ver": null,
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -150,7 +150,7 @@ Examples:
"package": "base-files", "package": "base-files",
"broken": null, "broken": null,
"proposed_pkg_ver": "10.3+deb10u13 Debian:10.13/oldstable", "proposed_pkg_ver": "10.3+deb10u13 Debian:10.13/oldstable",
"existing_src": null, "existing_pkg_ver": null,
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -158,7 +158,7 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": "1.19.7", "broken": "1.19.7",
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
}, },
{ {
@ -166,11 +166,12 @@ Examples:
"package": "dpkg", "package": "dpkg",
"broken": null, "broken": null,
"proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable", "proposed_pkg_ver": "1.19.8 Debian:10.13/oldstable",
"existing_src": "Debian-Security:10/oldstable", "existing_pkg_ver": "Debian-Security:10/oldstable",
"architecture": "amd64" "architecture": "amd64"
} }
] ]
<a id="jc.parsers.apt_get_sqq.parse"></a> <a id="jc.parsers.apt_get_sqq.parse"></a>
### parse ### parse
@ -178,7 +179,7 @@ Examples:
```python ```python
def parse(data: str, def parse(data: str,
raw: bool = False, raw: bool = False,
quiet: bool = False) -> List[JSONDictType] quiet: bool = False) -> List[Dict[str, Any]]
``` ```
Main text parsing function Main text parsing function
@ -193,9 +194,11 @@ Returns:
List of Dictionaries. Raw or processed structured data. List of Dictionaries. Raw or processed structured data.
### Parser Information ### Parser Information
Compatibility: linux Compatibility: linux
Source: [`jc/parsers/apt_get_sqq.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/apt_get_sqq.py) Source: [`jc/parsers/apt_get_sqq.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/apt_get_sqq.py)
Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

3
docs/parsers/dir.md
View File

@ -120,6 +120,7 @@ Examples:
... ...
] ]
<a id="jc.parsers.dir.parse"></a> <a id="jc.parsers.dir.parse"></a>
### parse ### parse
@ -140,9 +141,11 @@ Returns:
List of Dictionaries. Raw or processed structured data. List of Dictionaries. Raw or processed structured data.
### Parser Information ### Parser Information
Compatibility: win32 Compatibility: win32
Source: [`jc/parsers/dir.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/dir.py) Source: [`jc/parsers/dir.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/dir.py)
Version 1.6 by Rasheed Elsaleh (rasheed@rebelliondefense.com) Version 1.6 by Rasheed Elsaleh (rasheed@rebelliondefense.com)

5
docs/parsers/ethtool.md
View File

@ -166,12 +166,13 @@ Examples:
"br_margin_min": "0%" "br_margin_min": "0%"
} }
<a id="jc.parsers.ethtool.parse"></a> <a id="jc.parsers.ethtool.parse"></a>
### parse ### parse
```python ```python
def parse(data: str, raw: bool = False, quiet: bool = False) -> JSONDictType def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict[str, Any]
``` ```
Main text parsing function Main text parsing function
@ -186,9 +187,11 @@ Returns:
List of Dictionaries. Raw or processed structured data. List of Dictionaries. Raw or processed structured data.
### Parser Information ### Parser Information
Compatibility: linux Compatibility: linux
Source: [`jc/parsers/ethtool.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/ethtool.py) Source: [`jc/parsers/ethtool.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/ethtool.py)
Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com) Version 1.0 by Kelly Brazil (kellyjonbrazil@gmail.com)

20
docs/parsers/universal.md
View File

@ -1,18 +1,19 @@
# Table of Contents [Home](https://kellyjonbrazil.github.io/jc/)
* [jc.parsers.universal](#jc.parsers.universal)
* [simple\_table\_parse](#jc.parsers.universal.simple_table_parse)
* [sparse\_table\_parse](#jc.parsers.universal.sparse_table_parse)
<a id="jc.parsers.universal"></a> <a id="jc.parsers.universal"></a>
# jc.parsers.universal # jc.parsers.universal
# Table of Contents
*[jc.parsers.universal](#jc.parsers.universal)
*[simple_table_parse](#jc.parsers.universal.simple_table_parse)
*[sparse_table_parse](#jc.parsers.universal.sparse_table_parse)
jc - JSON Convert universal parsers jc - JSON Convert universal parsers
<a id="jc.parsers.universal.simple_table_parse"></a> <a id="jc.parsers.universal.simple_table_parse"></a>
### simple\_table\_parse ### simple_table_parse
```python ```python
def simple_table_parse(data: Iterable[str]) -> List[Dict] def simple_table_parse(data: Iterable[str]) -> List[Dict]
@ -50,7 +51,7 @@ Returns:
<a id="jc.parsers.universal.sparse_table_parse"></a> <a id="jc.parsers.universal.sparse_table_parse"></a>
### sparse\_table\_parse ### sparse_table_parse
```python ```python
def sparse_table_parse(data: Iterable[str], def sparse_table_parse(data: Iterable[str],
@ -86,7 +87,7 @@ Parameters:
Also, ensure there are no blank line items. Also, ensure there are no blank line items.
delim: (string) Delimiter to use. By default `u\\2063` delim: (string) Delimiter to use. By default `u\2063`
(invisible separator) is used since it is unlikely (invisible separator) is used since it is unlikely
to ever be seen in terminal output. You can change to ever be seen in terminal output. You can change
this for troubleshooting purposes or if there is a this for troubleshooting purposes or if there is a
@ -96,3 +97,4 @@ Returns:
List of Dictionaries List of Dictionaries

7
docs/parsers/ver.md
View File

@ -86,12 +86,13 @@ Examples:
"strict": false "strict": false
} }
<a id="jc.parsers.ver.parse"></a> <a id="jc.parsers.ver.parse"></a>
### parse ### parse
```python ```python
def parse(data: str, raw: bool = False, quiet: bool = False) -> JSONDictType def parse(data: str, raw: bool = False, quiet: bool = False) -> Dict[str, Any]
``` ```
Main text parsing function Main text parsing function
@ -106,6 +107,7 @@ Returns:
List of Dictionaries. Raw or processed structured data. List of Dictionaries. Raw or processed structured data.
### Parser Information ### Parser Information
Compatibility: linux, darwin, cygwin, win32, aix, freebsd Compatibility: linux, darwin, cygwin, win32, aix, freebsd
@ -113,4 +115,5 @@ Source: [`jc/parsers/ver.py`](https://github.com/kellyjonbrazil/jc/blob/master/j
This parser can be used with the `--slurp` command-line option. This parser can be used with the `--slurp` command-line option.
Version 1.1 by Kelly Brazil (kellyjonbrazil@gmail.com) Version 1.2 by Kelly Brazil (kellyjonbrazil@gmail.com)

1
docs/readme.md
View File

@ -1,3 +1,4 @@
[Home](https://kellyjonbrazil.github.io/jc/)
<a id="jc"></a> <a id="jc"></a>
# jc # jc

112
docs/streaming.md
View File

@ -1,67 +1,26 @@
# Table of Contents [Home](https://kellyjonbrazil.github.io/jc/)
* [jc.streaming](#jc.streaming)
* [streaming\_input\_type\_check](#jc.streaming.streaming_input_type_check)
* [streaming\_line\_input\_type\_check](#jc.streaming.streaming_line_input_type_check)
* [stream\_success](#jc.streaming.stream_success)
* [stream\_error](#jc.streaming.stream_error)
* [add\_jc\_meta](#jc.streaming.add_jc_meta)
* [raise\_or\_yield](#jc.streaming.raise_or_yield)
<a id="jc.streaming"></a> <a id="jc.streaming"></a>
# jc.streaming # jc.streaming
# Table of Contents
*[jc.streaming](#jc.streaming)
*[add_jc_meta](#jc.streaming.add_jc_meta)
*[raise_or_yield](#jc.streaming.raise_or_yield)
*[stream_error](#jc.streaming.stream_error)
*[stream_success](#jc.streaming.stream_success)
*[streaming_input_type_check](#jc.streaming.streaming_input_type_check)
*[streaming_line_input_type_check](#jc.streaming.streaming_line_input_type_check)
jc - JSON Convert streaming utils jc - JSON Convert streaming utils
<a id="jc.streaming.streaming_input_type_check"></a>
### streaming\_input\_type\_check
```python
def streaming_input_type_check(data: Iterable[Union[str, bytes]]) -> None
```
Ensure input data is an iterable, but not a string or bytes. Raises
`TypeError` if not.
<a id="jc.streaming.streaming_line_input_type_check"></a>
### streaming\_line\_input\_type\_check
```python
def streaming_line_input_type_check(line: str) -> None
```
Ensure each line is a string. Raises `TypeError` if not.
<a id="jc.streaming.stream_success"></a>
### stream\_success
```python
def stream_success(output_line: JSONDictType,
ignore_exceptions: bool) -> JSONDictType
```
Add `_jc_meta` object to output line if `ignore_exceptions=True`
<a id="jc.streaming.stream_error"></a>
### stream\_error
```python
def stream_error(e: BaseException, line: str) -> JSONDictType
```
Return an error `_jc_meta` field.
<a id="jc.streaming.add_jc_meta"></a> <a id="jc.streaming.add_jc_meta"></a>
### add\_jc\_meta ### add_jc_meta
```python ```python
def add_jc_meta(func: F) -> F def add_jc_meta(func: ~F) -> ~F
``` ```
Decorator for streaming parsers to add stream_success and stream_error Decorator for streaming parsers to add stream_success and stream_error
@ -103,7 +62,7 @@ In all cases above:
<a id="jc.streaming.raise_or_yield"></a> <a id="jc.streaming.raise_or_yield"></a>
### raise\_or\_yield ### raise_or_yield
```python ```python
def raise_or_yield(ignore_exceptions: bool, e: BaseException, def raise_or_yield(ignore_exceptions: bool, e: BaseException,
@ -114,3 +73,46 @@ Return the exception object and line string if ignore_exceptions is
True. Otherwise, re-raise the exception from the exception object with True. Otherwise, re-raise the exception from the exception object with
an annotation. an annotation.
<a id="jc.streaming.stream_error"></a>
### stream_error
```python
def stream_error(e: BaseException, line: str) -> Dict[str, Any]
```
Return an error `_jc_meta` field.
<a id="jc.streaming.stream_success"></a>
### stream_success
```python
def stream_success(output_line: Dict[str, Any],
ignore_exceptions: bool) -> Dict[str, Any]
```
Add `_jc_meta` object to output line if `ignore_exceptions=True`
<a id="jc.streaming.streaming_input_type_check"></a>
### streaming_input_type_check
```python
def streaming_input_type_check(data: Iterable[Union[str, bytes]]) -> None
```
Ensure input data is an iterable, but not a string or bytes. Raises
`TypeError` if not.
<a id="jc.streaming.streaming_line_input_type_check"></a>
### streaming_line_input_type_check
```python
def streaming_line_input_type_check(line: str) -> None
```
Ensure each line is a string. Raises `TypeError` if not.

410
docs/utils.md
View File

@ -1,78 +1,27 @@
# Table of Contents [Home](https://kellyjonbrazil.github.io/jc/)
* [jc.utils](#jc.utils)
* [warning\_message](#jc.utils.warning_message)
* [error\_message](#jc.utils.error_message)
* [is\_compatible](#jc.utils.is_compatible)
* [compatibility](#jc.utils.compatibility)
* [has\_data](#jc.utils.has_data)
* [remove\_quotes](#jc.utils.remove_quotes)
* [normalize\_key](#jc.utils.normalize_key)
* [convert\_to\_int](#jc.utils.convert_to_int)
* [convert\_to\_float](#jc.utils.convert_to_float)
* [convert\_to\_bool](#jc.utils.convert_to_bool)
* [convert\_size\_to\_int](#jc.utils.convert_size_to_int)
* [input\_type\_check](#jc.utils.input_type_check)
* [line\_slice](#jc.utils.line_slice)
* [timestamp](#jc.utils.timestamp)
* [\_\_init\_\_](#jc.utils.timestamp.__init__)
<a id="jc.utils"></a> <a id="jc.utils"></a>
# jc.utils # jc.utils
# Table of Contents
*[jc.utils](#jc.utils)
*[compatibility](#jc.utils.compatibility)
*[convert_size_to_int](#jc.utils.convert_size_to_int)
*[convert_to_bool](#jc.utils.convert_to_bool)
*[convert_to_float](#jc.utils.convert_to_float)
*[convert_to_int](#jc.utils.convert_to_int)
*[error_message](#jc.utils.error_message)
*[has_data](#jc.utils.has_data)
*[input_type_check](#jc.utils.input_type_check)
*[is_compatible](#jc.utils.is_compatible)
*[line_slice](#jc.utils.line_slice)
*[normalize_key](#jc.utils.normalize_key)
*[remove_quotes](#jc.utils.remove_quotes)
*[warning_message](#jc.utils.warning_message)
jc - JSON Convert utils jc - JSON Convert utils
<a id="jc.utils.warning_message"></a>
### warning\_message
```python
def warning_message(message_lines: List[str]) -> None
```
Prints warning message to `STDERR` for non-fatal issues. The first line
is prepended with 'jc: Warning - ' and subsequent lines are indented.
Wraps text as needed based on the terminal width.
Parameters:
message: (list) list of string lines
Returns:
None - just prints output to STDERR
<a id="jc.utils.error_message"></a>
### error\_message
```python
def error_message(message_lines: List[str]) -> None
```
Prints an error message to `STDERR` for fatal issues. The first line is
prepended with 'jc: Error - ' and subsequent lines are indented.
Wraps text as needed based on the terminal width.
Parameters:
message: (list) list of string lines
Returns:
None - just prints output to STDERR
<a id="jc.utils.is_compatible"></a>
### is\_compatible
```python
def is_compatible(compatible: List[str]) -> bool
```
Returns True if the parser is compatible with the running OS platform.
<a id="jc.utils.compatibility"></a> <a id="jc.utils.compatibility"></a>
### compatibility ### compatibility
@ -101,135 +50,9 @@ Returns:
None - just prints output to STDERR None - just prints output to STDERR
<a id="jc.utils.has_data"></a>
### has\_data
```python
def has_data(data: Union[str, bytes]) -> bool
```
Checks if the string input contains data. If there are any
non-whitespace characters then return `True`, else return `False`.
For bytes, returns True if there is any data.
Parameters:
data: (string, bytes) input to check whether it contains data
Returns:
Boolean True if input string (data) contains non-whitespace
characters, otherwise False. For bytes data, returns
True if there is any data, otherwise False.
<a id="jc.utils.remove_quotes"></a>
### remove\_quotes
```python
def remove_quotes(data: str) -> str
```
Remove single or double quotes surrounding a string. If no quotes are
found then the string is returned unmodified.
Parameters:
data: (string) Input value
Returns:
string
<a id="jc.utils.normalize_key"></a>
### normalize\_key
```python
def normalize_key(data: str) -> str
```
Normalize a key name by shifting to lower-case and converting special
characters to underscores.
Special characters are defined as `space` and the following:
!"#$%&'()*+,-./:;<=>?@[\]^`{|}~
This is a lossy algorithm. Repeating and trailing underscores are
removed.
Parameters:
data: (string) Input value
Returns:
string
<a id="jc.utils.convert_to_int"></a>
### convert\_to\_int
```python
def convert_to_int(value: object) -> Optional[int]
```
Converts string and float input to int. Strips all non-numeric
characters from strings.
Parameters:
value: (string/float) Input value
Returns:
integer/None Integer if successful conversion, otherwise None
<a id="jc.utils.convert_to_float"></a>
### convert\_to\_float
```python
def convert_to_float(value: object) -> Optional[float]
```
Converts string and int input to float. Strips all non-numeric
characters from strings.
Parameters:
value: (string/integer) Input value
Returns:
float/None Float if successful conversion, otherwise None
<a id="jc.utils.convert_to_bool"></a>
### convert\_to\_bool
```python
def convert_to_bool(value: object) -> bool
```
Converts string, integer, or float input to boolean by checking
for 'truthy' values.
Parameters:
value: (string/integer/float) Input value
Returns:
True/False False unless a 'truthy' number or string is found
('y', 'yes', 'true', '1', 1, -1, etc.)
<a id="jc.utils.convert_size_to_int"></a> <a id="jc.utils.convert_size_to_int"></a>
### convert\_size\_to\_int ### convert_size_to_int
```python ```python
def convert_size_to_int(size: str, binary: bool = False) -> Optional[int] def convert_size_to_int(size: str, binary: bool = False) -> Optional[int]
@ -269,9 +92,110 @@ gigabytes, terabytes and petabytes. Some examples:
>>> convert_size_to_int('1.5 GB', binary=True) >>> convert_size_to_int('1.5 GB', binary=True)
1610612736 1610612736
<a id="jc.utils.convert_to_bool"></a>
### convert_to_bool
```python
def convert_to_bool(value: object) -> bool
```
Converts string, integer, or float input to boolean by checking
for 'truthy' values.
Parameters:
value: (string/integer/float) Input value
Returns:
True/False False unless a 'truthy' number or string is found
('y', 'yes', 'true', '1', 1, -1, etc.)
<a id="jc.utils.convert_to_float"></a>
### convert_to_float
```python
def convert_to_float(value: object) -> Optional[float]
```
Converts string and int input to float. Strips all non-numeric
characters from strings.
Parameters:
value: (string/integer) Input value
Returns:
float/None Float if successful conversion, otherwise None
<a id="jc.utils.convert_to_int"></a>
### convert_to_int
```python
def convert_to_int(value: object) -> Optional[int]
```
Converts string and float input to int. Strips all non-numeric
characters from strings.
Parameters:
value: (string/float) Input value
Returns:
integer/None Integer if successful conversion, otherwise None
<a id="jc.utils.error_message"></a>
### error_message
```python
def error_message(message_lines: List[str]) -> None
```
Prints an error message to `STDERR` for fatal issues. The first line is
prepended with 'jc: Error - ' and subsequent lines are indented.
Wraps text as needed based on the terminal width.
Parameters:
message: (list) list of string lines
Returns:
None - just prints output to STDERR
<a id="jc.utils.has_data"></a>
### has_data
```python
def has_data(data: Union[str, bytes]) -> bool
```
Checks if the string input contains data. If there are any
non-whitespace characters then return `True`, else return `False`.
For bytes, returns True if there is any data.
Parameters:
data: (string, bytes) input to check whether it contains data
Returns:
Boolean True if input string (data) contains non-whitespace
characters, otherwise False. For bytes data, returns
True if there is any data, otherwise False.
<a id="jc.utils.input_type_check"></a> <a id="jc.utils.input_type_check"></a>
### input\_type\_check ### input_type_check
```python ```python
def input_type_check(data: object) -> None def input_type_check(data: object) -> None
@ -279,16 +203,26 @@ def input_type_check(data: object) -> None
Ensure input data is a string. Raises `TypeError` if not. Ensure input data is a string. Raises `TypeError` if not.
<a id="jc.utils.is_compatible"></a>
### is_compatible
```python
def is_compatible(compatible: List[str]) -> bool
```
Returns True if the parser is compatible with the running OS platform.
<a id="jc.utils.line_slice"></a> <a id="jc.utils.line_slice"></a>
### line\_slice ### line_slice
```python ```python
def line_slice( def line_slice(
data: Union[str, Iterable[str], TextIO, bytes, None], data: Union[str, Iterable[str], TextIO, bytes, NoneType],
slice_start: Optional[int] = None, slice_start: Optional[int] = None,
slice_end: Optional[int] = None slice_end: Optional[int] = None
) -> Union[str, Iterable[str], TextIO, bytes, None] ) -> Union[str, Iterable[str], TextIO, bytes, NoneType]
``` ```
Slice input data by lines - lazily, if possible. Slice input data by lines - lazily, if possible.
@ -310,51 +244,69 @@ Returns:
string if input is a string. string if input is a string.
iterable of strings if input is an iterable (for streaming parsers) iterable of strings if input is an iterable (for streaming parsers)
<a id="jc.utils.timestamp"></a> <a id="jc.utils.normalize_key"></a>
### timestamp Objects ### normalize_key
```python ```python
class timestamp() def normalize_key(data: str) -> str
``` ```
<a id="jc.utils.timestamp.__init__"></a> Normalize a key name by shifting to lower-case and converting special
characters to underscores.
### \_\_init\_\_ Special characters are defined as `space` and the following:
```python !"#$%&'()*+,-./:;<=>?@[\]^`{|}~
def __init__(datetime_string: Optional[str],
format_hint: Optional[Iterable[int]] = None) -> None
```
Input a datetime text string of several formats and convert to a This is a lossy algorithm. Repeating and trailing underscores are
naive or timezone-aware epoch timestamp in UTC. removed.
Parameters: Parameters:
datetime_string (str): a string representation of a data: (string) Input value
datetime in several supported formats
format_hint (iterable): an optional iterable of format ID Returns:
integers to instruct the timestamp object to try those
formats first in the order given. Other formats will be
tried after the format hint list is exhausted. This can
speed up timestamp conversion so several different formats
don't have to be tried in brute-force fashion.
Returns a timestamp object with the following attributes: string
string (str): the input datetime string <a id="jc.utils.remove_quotes"></a>
format (int | None): the format rule that was used to decode ### remove_quotes
the datetime string. None if conversion fails.
naive (int | None): timestamp based on locally configured ```python
timezone. None if conversion fails. def remove_quotes(data: str) -> str
```
utc (int | None): aware timestamp only if UTC timezone Remove single or double quotes surrounding a string. If no quotes are
detected in datetime string. None if conversion fails. found then the string is returned unmodified.
Parameters:
data: (string) Input value
Returns:
string
<a id="jc.utils.warning_message"></a>
### warning_message
```python
def warning_message(message_lines: List[str]) -> None
```
Prints warning message to `STDERR` for non-fatal issues. The first line
is prepended with 'jc: Warning - ' and subsequent lines are indented.
Wraps text as needed based on the terminal width.
Parameters:
message: (list) list of string lines
Returns:
None - just prints output to STDERR
iso (str | None): ISO string - timezone information is output
only if UTC timezone is detected in the datetime string.

10
jc/parsers/ver.py
View File

@ -89,7 +89,7 @@ import jc.utils
class info(): class info():
"""Provides parser metadata (version, author, etc.)""" """Provides parser metadata (version, author, etc.)"""
version = '1.1' version = '1.2'
description = 'Version string parser' description = 'Version string parser'
author = 'Kelly Brazil' author = 'Kelly Brazil'
author_email = 'kellyjonbrazil@gmail.com' author_email = 'kellyjonbrazil@gmail.com'
@ -132,7 +132,7 @@ def _process(proc_data: JSONDictType) -> JSONDictType:
return proc_data return proc_data
def strict_parse(vstring): def _strict_parse(vstring):
version_re = re.compile(r'^(\d+) \. (\d+) (\. (\d+))? ([ab](\d+))?$', re.VERBOSE) version_re = re.compile(r'^(\d+) \. (\d+) (\. (\d+))? ([ab](\d+))?$', re.VERBOSE)
match = version_re.match(vstring) match = version_re.match(vstring)
if not match: if not match:
@ -158,7 +158,7 @@ def strict_parse(vstring):
} }
def loose_parse(vstring): def _loose_parse(vstring):
component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE)
components = [x for x in component_re.split(vstring) if x and x != '.'] components = [x for x in component_re.split(vstring) if x and x != '.']
@ -197,10 +197,10 @@ def parse(
data = data.strip() data = data.strip()
try: try:
raw_output = strict_parse(data) raw_output = _strict_parse(data)
except ValueError: except ValueError:
raw_output['components'] = loose_parse(data) raw_output['components'] = _loose_parse(data)
strict = False strict = False
if raw_output: if raw_output:

2
man/jc.1
View File

@ -1,4 +1,4 @@
.TH jc 1 2024-03-01 1.25.2 "JSON Convert" .TH jc 1 2024-03-14 1.25.2 "JSON Convert"
.SH NAME .SH NAME
\fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types, \fBjc\fP \- JSON Convert JSONifies the output of many CLI tools, file-types,
and strings and strings